Fix test runner crash when not in a git repo
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 import argparse
8 import signal
9 import sys
10 import os
11 import io
12 import shutil
13 import tempfile
14 import time
15 import re
16 import traceback
17
18 # We don't actually need subprocess in runtests.py, but:
19 # * We do need it in testlibs.py
20 # * We can't import testlibs.py until after we have imported ctypes
21 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
22 # says "Aborted" and we fail with exit code 134.
23 # So we import it here first, so that the testsuite doesn't appear to fail.
24 import subprocess
25
26 from testutil import getStdout, Watcher, str_warn, str_info
27 from testglobals import getConfig, ghc_env, getTestRun, TestOptions, brokens
28 from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty
29 from junit import junit
30 import cpu_features
31
32 # Readline sometimes spews out ANSI escapes for some values of TERM,
33 # which result in test failures. Thus set TERM to a nice, simple, safe
34 # value.
35 os.environ['TERM'] = 'vt100'
36 ghc_env['TERM'] = 'vt100'
37
38 global config
39 config = getConfig() # get it from testglobals
40
41 def signal_handler(signal, frame):
42 stopNow()
43
44 # -----------------------------------------------------------------------------
45 # cmd-line options
46
47 parser = argparse.ArgumentParser(description="GHC's testsuite driver")
48 perf_group = parser.add_mutually_exclusive_group()
49
50 parser.add_argument("-e", action='append', help="A string to execute from the command line.")
51 parser.add_argument("--config-file", action="append", help="config file")
52 parser.add_argument("--config", action='append', help="config field")
53 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
54 parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
55 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
56 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
57 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
58 parser.add_argument("--way", action="append", help="just this way")
59 parser.add_argument("--skipway", action="append", help="skip this way")
60 parser.add_argument("--threads", type=int, help="threads to run simultaneously")
61 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
62 parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
63 parser.add_argument("--test-env", default='local', help="Override default chosen test-env.")
64 perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
65 perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")
66
67 args = parser.parse_args()
68
69 if args.e:
70 for e in args.e:
71 exec(e)
72
73 if args.config_file:
74 for arg in args.config_file:
75 exec(open(arg).read())
76
77 if args.config:
78 for arg in args.config:
79 field, value = arg.split('=', 1)
80 setattr(config, field, value)
81
82 all_ways = config.run_ways+config.compile_ways+config.other_ways
83
84 if args.rootdir:
85 config.rootdirs = args.rootdir
86
87 config.metrics_file = args.metrics_file
88 hasMetricsFile = bool(config.metrics_file)
89 config.summary_file = args.summary_file
90 config.no_print_summary = args.no_print_summary
91
92 if args.only:
93 config.only = args.only
94 config.run_only_some_tests = True
95
96 if args.way:
97 for way in args.way:
98 if way not in all_ways:
99 print('WARNING: Unknown WAY %s in --way' % way)
100 else:
101 config.cmdline_ways += [way]
102 if way in config.other_ways:
103 config.run_ways += [way]
104 config.compile_ways += [way]
105
106 if args.skipway:
107 for way in args.skipway:
108 if way not in all_ways:
109 print('WARNING: Unknown WAY %s in --skipway' % way)
110
111 config.other_ways = [w for w in config.other_ways if w not in args.skipway]
112 config.run_ways = [w for w in config.run_ways if w not in args.skipway]
113 config.compile_ways = [w for w in config.compile_ways if w not in args.skipway]
114
115 if args.threads:
116 config.threads = args.threads
117 config.use_threads = True
118
119 if args.verbose is not None:
120 config.verbose = args.verbose
121
122 # Note force skip perf tests: skip if this is not a git repo (estimated with inside_git_repo)
123 # and no metrics file is given. In this case there is no way to read the previous commit's
124 # perf test results, nor a way to store new perf test results.
125 forceSkipPerfTests = not hasMetricsFile and not inside_git_repo()
126 config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
127 config.only_perf_tests = args.only_perf_tests
128
129 if args.test_env:
130 config.test_env = args.test_env
131
132 config.cygwin = False
133 config.msys = False
134
135 if windows:
136 h = os.popen('uname -s', 'r')
137 v = h.read()
138 h.close()
139 if v.startswith("CYGWIN"):
140 config.cygwin = True
141 elif v.startswith("MINGW") or v.startswith("MSYS"):
142 # msys gives "MINGW32"
143 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
144 config.msys = True
145 else:
146 raise Exception("Can't detect Windows terminal type")
147
148 # Try to use UTF8
149 if windows:
150 import ctypes
151 # Windows and mingw* Python provide windll, msys2 python provides cdll.
152 if hasattr(ctypes, 'WinDLL'):
153 mydll = ctypes.WinDLL
154 else:
155 mydll = ctypes.CDLL
156
157 # This actually leaves the terminal in codepage 65001 (UTF8) even
158 # after python terminates. We ought really remember the old codepage
159 # and set it back.
160 kernel32 = mydll('kernel32.dll')
161 if kernel32.SetConsoleCP(65001) == 0:
162 raise Exception("Failure calling SetConsoleCP(65001)")
163 if kernel32.SetConsoleOutputCP(65001) == 0:
164 raise Exception("Failure calling SetConsoleOutputCP(65001)")
165
166 # register the interrupt handler
167 signal.signal(signal.SIGINT, signal_handler)
168 else:
169 # Try and find a utf8 locale to use
170 # First see if we already have a UTF8 locale
171 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
172 v = h.read()
173 h.close()
174 if v == '':
175 # We don't, so now see if 'locale -a' works
176 h = os.popen('locale -a | grep -F .', 'r')
177 v = h.read()
178 h.close()
179 if v != '':
180 # If it does then use the first utf8 locale that is available
181 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
182 v = h.readline().strip()
183 h.close()
184 if v != '':
185 os.environ['LC_ALL'] = v
186 ghc_env['LC_ALL'] = v
187 print("setting LC_ALL to", v)
188 else:
189 print('WARNING: No UTF8 locale found.')
190 print('You may get some spurious test failures.')
191
192 # This has to come after arg parsing as the args can change the compiler
193 get_compiler_info()
194
195 # Can't import this earlier as we need to know if threading will be
196 # enabled or not
197 from testlib import *
198
199 def format_path(path):
200 if windows:
201 if os.pathsep == ':':
202 # If using msys2 python instead of mingw we have to change the drive
203 # letter representation. Otherwise it thinks we're adding two env
204 # variables E and /Foo when we add E:/Foo.
205 path = re.sub('([a-zA-Z]):', '/\\1', path)
206 if config.cygwin:
207 # On cygwin we can't put "c:\foo" in $PATH, as : is a
208 # field separator. So convert to /cygdrive/c/foo instead.
209 # Other pythons use ; as the separator, so no problem.
210 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
211 path = re.sub('\\\\', '/', path)
212 return path
213
214 # On Windows we need to set $PATH to include the paths to all the DLLs
215 # in order for the dynamic library tests to work.
216 if windows or darwin:
217 pkginfo = getStdout([config.ghc_pkg, 'dump'])
218 topdir = config.libdir
219 if windows:
220 mingw = os.path.abspath(os.path.join(topdir, '../mingw/bin'))
221 mingw = format_path(mingw)
222 ghc_env['PATH'] = os.pathsep.join([ghc_env.get("PATH", ""), mingw])
223 for line in pkginfo.split('\n'):
224 if line.startswith('library-dirs:'):
225 path = line.rstrip()
226 path = re.sub('^library-dirs: ', '', path)
227 # Use string.replace instead of re.sub, because re.sub
228 # interprets backslashes in the replacement string as
229 # escape sequences.
230 path = path.replace('$topdir', topdir)
231 if path.startswith('"'):
232 path = re.sub('^"(.*)"$', '\\1', path)
233 path = re.sub('\\\\(.)', '\\1', path)
234 if windows:
235 path = format_path(path)
236 ghc_env['PATH'] = os.pathsep.join([path, ghc_env.get("PATH", "")])
237 else:
238 # darwin
239 ghc_env['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, ghc_env.get("DYLD_LIBRARY_PATH", "")])
240
241 testopts_local.x = TestOptions()
242
243 # if timeout == -1 then we try to calculate a sensible value
244 if config.timeout == -1:
245 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
246
247 print('Timeout is ' + str(config.timeout))
248 print('Known ways: ' + ', '.join(config.other_ways))
249 print('Run ways: ' + ', '.join(config.run_ways))
250 print('Compile ways: ' + ', '.join(config.compile_ways))
251
252 # Try get allowed performance changes from the git commit.
253 try:
254 config.allowed_perf_changes = Perf.get_allowed_perf_changes()
255 except subprocess.CalledProcessError:
256 print('Failed to get allowed metric changes from the HEAD git commit message.')
257
258 print('Allowing performance changes in: ' + ', '.join(config.allowed_perf_changes.keys()))
259
260 # -----------------------------------------------------------------------------
261 # The main dude
262
263 if config.rootdirs == []:
264 config.rootdirs = ['.']
265
266 t_files = list(findTFiles(config.rootdirs))
267
268 print('Found', len(t_files), '.T files...')
269
270 t = getTestRun()
271
272 # Avoid cmd.exe built-in 'date' command on Windows
273 t.start_time = time.localtime()
274
275 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
276
277 # For reference
278 try:
279 print('Detected CPU features: ', cpu_features.get_cpu_features())
280 except Exception as e:
281 print('Failed to detect CPU features: ', e)
282
283 sys.stdout.flush()
284 # we output text, which cannot be unbuffered
285 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
286
287 if config.local:
288 tempdir = ''
289 else:
290 # See note [Running tests in /tmp]
291 tempdir = tempfile.mkdtemp('', 'ghctest-')
292
293 # opts.testdir should be quoted when used, to make sure the testsuite
294 # keeps working when it contains backward slashes, for example from
295 # using os.path.join. Windows native and mingw* python
296 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
297 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
298 # `os.path.sep = '/'`.
299 # To catch usage of unquoted opts.testdir early, insert some spaces into
300 # tempdir.
301 tempdir = os.path.join(tempdir, 'test spaces')
302
303 def cleanup_and_exit(exitcode):
304 if config.cleanup and tempdir:
305 shutil.rmtree(tempdir, ignore_errors=True)
306 exit(exitcode)
307
308 # First collect all the tests to be run
309 t_files_ok = True
310 for file in t_files:
311 if_verbose(2, '====> Scanning %s' % file)
312 newTestDir(tempdir, os.path.dirname(file))
313 try:
314 with io.open(file, encoding='utf8') as f:
315 src = f.read()
316
317 exec(src)
318 except Exception as e:
319 traceback.print_exc()
320 framework_fail(file, '', str(e))
321 t_files_ok = False
322
323 for name in config.only:
324 if t_files_ok:
325 # See Note [Mutating config.only]
326 framework_fail(name, '', 'test not found')
327 else:
328 # Let user fix .T file errors before reporting on unfound tests.
329 # The reason the test can not be found is likely because of those
330 # .T file errors.
331 pass
332
333 if config.list_broken:
334 print('')
335 print('Broken tests:')
336 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
337 print('')
338
339 if t.framework_failures:
340 print('WARNING:', len(t.framework_failures), 'framework failures!')
341 print('')
342 else:
343 # completion watcher
344 watcher = Watcher(len(parallelTests))
345
346 # Now run all the tests
347 try:
348 for oneTest in parallelTests:
349 if stopping():
350 break
351 oneTest(watcher)
352
353 # wait for parallel tests to finish
354 if not stopping():
355 watcher.wait()
356
357 # Run the following tests purely sequential
358 config.use_threads = False
359 for oneTest in aloneTests:
360 if stopping():
361 break
362 oneTest(watcher)
363 except KeyboardInterrupt:
364 pass
365
366 # flush everything before we continue
367 sys.stdout.flush()
368
369 # Warn if had to force skip perf tests (see Note force skip perf tests).
370 spacing = " "
371 if forceSkipPerfTests and not args.skip_perf_tests:
372 print()
373 print(str_warn('Skipping All Performance Tests') + ' `git` exited with non-zero exit code.')
374 print(spacing + 'Git is required because performance test results are compared with ancestor git commits\' results (stored with git notes).')
375 print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')
376
377 # Warn of new metrics.
378 new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
379 if any(new_metrics):
380 if inside_git_repo():
381 reason = 'a baseline (expected value) cannot be recovered from' + \
382 ' previous git commits. This may be due to HEAD having' + \
383 ' new tests or having expected changes, the presence of' + \
384 ' expected changes since the last run of the tests, and/or' + \
385 ' the latest test run being too old.'
386 fix = 'If the tests exist on the previous' + \
387 ' commit (And are configured to run with the same ways),' + \
388 ' then check out that commit and run the tests to generate' + \
389 ' the missing metrics. Alternatively, a baseline may be' + \
390 ' recovered from ci results once fetched:\n\n' + \
391 spacing + 'git fetch ' + \
392 'https://gitlab.haskell.org/ghc/ghc-performance-notes.git' + \
393 ' refs/notes/perf:refs/notes/' + Perf.CiNamespace
394 else:
395 reason = "this is not a git repo so the previous git commit's" + \
396 " metrics cannot be loaded from git notes:"
397 fix = ""
398 print()
399 print(str_warn('Missing Baseline Metrics') + \
400 ' these metrics trivially pass because ' + reason)
401 print(spacing + (' ').join(set([metric.test for metric in new_metrics])))
402 if fix != "":
403 print()
404 print(fix)
405
406 # Inform of how to accept metric changes.
407 if (len(t.unexpected_stat_failures) > 0):
408 print()
409 print(str_info("Some stats have changed") + " If this is expected, " + \
410 "allow changes by appending the git commit message with this:")
411 print('-' * 25)
412 print(Perf.allow_changes_string(t.metrics))
413 print('-' * 25)
414
415 summary(t, sys.stdout, config.no_print_summary, True)
416
417 # Write perf stats if any exist or if a metrics file is specified.
418 stats = [stat for (_, stat) in t.metrics]
419 if hasMetricsFile:
420 print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
421 with open(config.metrics_file, 'a') as file:
422 file.write("\n" + Perf.format_perf_stat(stats))
423 elif inside_git_repo() and any(stats):
424 if is_worktree_dirty():
425 print()
426 print(str_warn('Performance Metrics NOT Saved') + \
427 ' working tree is dirty. Commit changes or use ' + \
428 '--metrics-file to save metrics to a file.')
429 else:
430 Perf.append_perf_stat(stats)
431
432 # Write summary
433 if config.summary_file:
434 with open(config.summary_file, 'w') as file:
435 summary(t, file)
436
437 if args.junit:
438 junit(t).write(args.junit)
439
440 if len(t.unexpected_failures) > 0 or \
441 len(t.unexpected_stat_failures) > 0 or \
442 len(t.framework_failures) > 0:
443 exitcode = 1
444 else:
445 exitcode = 0
446
447 cleanup_and_exit(exitcode)
448
449 # Note [Running tests in /tmp]
450 #
451 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
452 # the source directory without copying them to the test directory first.
453 #
454 # As an example, take a run_command test with a Makefile containing
455 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
456 # Foo.hi in the source directory. There are 2 problems with this:
457 # * Output files in the source directory won't get cleaned up automatically.
458 # * Two tests might (over)write the same output file.
459 #
460 # Tests that only fail when run concurrently with other tests are the
461 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
462 #
463 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
464 # making changes to quite a few tests. The problem is that
465 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
466 # `module Main where` does not produce Foo.o, as it would without
467 # -outputdir, but Main.o. See [1].
468 #
469 # Using -outputdir='.' is not foolproof anyway, since it does not change
470 # the destination of the final executable (Foo.exe).
471 #
472 # Another hardening method that could be tried is to `chmod -w` the
473 # source directory.
474 #
475 # By default we set LOCAL=1, because it makes it easier to inspect the
476 # test directory while working on a new test.
477 #
478 # [1]
479 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files