Revert "Batch merge"
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 import argparse
8 import signal
9 import sys
10 import os
11 import io
12 import shutil
13 import tempfile
14 import time
15 import re
16 import traceback
17
18 # We don't actually need subprocess in runtests.py, but:
19 # * We do need it in testlibs.py
20 # * We can't import testlibs.py until after we have imported ctypes
21 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
22 # says "Aborted" and we fail with exit code 134.
23 # So we import it here first, so that the testsuite doesn't appear to fail.
24 import subprocess
25
26 from testutil import getStdout, Watcher, str_warn, str_info
27 from testglobals import getConfig, ghc_env, getTestRun, TestOptions, brokens
28 from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty
29 from junit import junit
30 import cpu_features
31
32 # Readline sometimes spews out ANSI escapes for some values of TERM,
33 # which result in test failures. Thus set TERM to a nice, simple, safe
34 # value.
35 os.environ['TERM'] = 'vt100'
36 ghc_env['TERM'] = 'vt100'
37
38 global config
39 config = getConfig() # get it from testglobals
40
41 def signal_handler(signal, frame):
42 stopNow()
43
44 # -----------------------------------------------------------------------------
45 # cmd-line options
46
47 parser = argparse.ArgumentParser(description="GHC's testsuite driver")
48 perf_group = parser.add_mutually_exclusive_group()
49
50 parser.add_argument("-e", action='append', help="A string to execute from the command line.")
51 parser.add_argument("--config-file", action="append", help="config file")
52 parser.add_argument("--config", action='append', help="config field")
53 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
54 parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
55 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
56 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
57 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
58 parser.add_argument("--way", action="append", help="just this way")
59 parser.add_argument("--skipway", action="append", help="skip this way")
60 parser.add_argument("--threads", type=int, help="threads to run simultaneously")
61 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
62 parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
63 parser.add_argument("--test-env", default='local', help="Override default chosen test-env.")
64 perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
65 perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")
66
67 args = parser.parse_args()
68
69 if args.e:
70 for e in args.e:
71 exec(e)
72
73 if args.config_file:
74 for arg in args.config_file:
75 exec(open(arg).read())
76
77 if args.config:
78 for arg in args.config:
79 field, value = arg.split('=', 1)
80 setattr(config, field, value)
81
82 all_ways = config.run_ways+config.compile_ways+config.other_ways
83
84 if args.rootdir:
85 config.rootdirs = args.rootdir
86
87 config.metrics_file = args.metrics_file
88 hasMetricsFile = bool(config.metrics_file)
89 config.summary_file = args.summary_file
90 config.no_print_summary = args.no_print_summary
91
92 if args.only:
93 config.only = args.only
94 config.run_only_some_tests = True
95
96 if args.way:
97 for way in args.way:
98 if way not in all_ways:
99 print('WARNING: Unknown WAY %s in --way' % way)
100 else:
101 config.cmdline_ways += [way]
102 if way in config.other_ways:
103 config.run_ways += [way]
104 config.compile_ways += [way]
105
106 if args.skipway:
107 for way in args.skipway:
108 if way not in all_ways:
109 print('WARNING: Unknown WAY %s in --skipway' % way)
110
111 config.other_ways = [w for w in config.other_ways if w not in args.skipway]
112 config.run_ways = [w for w in config.run_ways if w not in args.skipway]
113 config.compile_ways = [w for w in config.compile_ways if w not in args.skipway]
114
115 if args.threads:
116 config.threads = args.threads
117 config.use_threads = True
118
119 if args.verbose is not None:
120 config.verbose = args.verbose
121
122 # Note force skip perf tests: skip if this is not a git repo (estimated with inside_git_repo)
123 # and no metrics file is given. In this case there is no way to read the previous commit's
124 # perf test results, nor a way to store new perf test results.
125 canGitStatus = inside_git_repo()
126 forceSkipPerfTests = not hasMetricsFile and not canGitStatus
127 config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
128 config.only_perf_tests = args.only_perf_tests
129
130 if args.test_env:
131 config.test_env = args.test_env
132
133 config.cygwin = False
134 config.msys = False
135
136 if windows:
137 h = os.popen('uname -s', 'r')
138 v = h.read()
139 h.close()
140 if v.startswith("CYGWIN"):
141 config.cygwin = True
142 elif v.startswith("MINGW") or v.startswith("MSYS"):
143 # msys gives "MINGW32"
144 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
145 config.msys = True
146 else:
147 raise Exception("Can't detect Windows terminal type")
148
149 # Try to use UTF8
150 if windows:
151 import ctypes
152 # Windows and mingw* Python provide windll, msys2 python provides cdll.
153 if hasattr(ctypes, 'WinDLL'):
154 mydll = ctypes.WinDLL
155 else:
156 mydll = ctypes.CDLL
157
158 # This actually leaves the terminal in codepage 65001 (UTF8) even
159 # after python terminates. We ought really remember the old codepage
160 # and set it back.
161 kernel32 = mydll('kernel32.dll')
162 if kernel32.SetConsoleCP(65001) == 0:
163 raise Exception("Failure calling SetConsoleCP(65001)")
164 if kernel32.SetConsoleOutputCP(65001) == 0:
165 raise Exception("Failure calling SetConsoleOutputCP(65001)")
166
167 # register the interrupt handler
168 signal.signal(signal.SIGINT, signal_handler)
169 else:
170 # Try and find a utf8 locale to use
171 # First see if we already have a UTF8 locale
172 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
173 v = h.read()
174 h.close()
175 if v == '':
176 # We don't, so now see if 'locale -a' works
177 h = os.popen('locale -a | grep -F .', 'r')
178 v = h.read()
179 h.close()
180 if v != '':
181 # If it does then use the first utf8 locale that is available
182 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
183 v = h.readline().strip()
184 h.close()
185 if v != '':
186 os.environ['LC_ALL'] = v
187 ghc_env['LC_ALL'] = v
188 print("setting LC_ALL to", v)
189 else:
190 print('WARNING: No UTF8 locale found.')
191 print('You may get some spurious test failures.')
192
193 # This has to come after arg parsing as the args can change the compiler
194 get_compiler_info()
195
196 # Can't import this earlier as we need to know if threading will be
197 # enabled or not
198 from testlib import *
199
200 def format_path(path):
201 if windows:
202 if os.pathsep == ':':
203 # If using msys2 python instead of mingw we have to change the drive
204 # letter representation. Otherwise it thinks we're adding two env
205 # variables E and /Foo when we add E:/Foo.
206 path = re.sub('([a-zA-Z]):', '/\\1', path)
207 if config.cygwin:
208 # On cygwin we can't put "c:\foo" in $PATH, as : is a
209 # field separator. So convert to /cygdrive/c/foo instead.
210 # Other pythons use ; as the separator, so no problem.
211 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
212 path = re.sub('\\\\', '/', path)
213 return path
214
215 # On Windows we need to set $PATH to include the paths to all the DLLs
216 # in order for the dynamic library tests to work.
217 if windows or darwin:
218 pkginfo = getStdout([config.ghc_pkg, 'dump'])
219 topdir = config.libdir
220 if windows:
221 mingw = os.path.abspath(os.path.join(topdir, '../mingw/bin'))
222 mingw = format_path(mingw)
223 ghc_env['PATH'] = os.pathsep.join([ghc_env.get("PATH", ""), mingw])
224 for line in pkginfo.split('\n'):
225 if line.startswith('library-dirs:'):
226 path = line.rstrip()
227 path = re.sub('^library-dirs: ', '', path)
228 # Use string.replace instead of re.sub, because re.sub
229 # interprets backslashes in the replacement string as
230 # escape sequences.
231 path = path.replace('$topdir', topdir)
232 if path.startswith('"'):
233 path = re.sub('^"(.*)"$', '\\1', path)
234 path = re.sub('\\\\(.)', '\\1', path)
235 if windows:
236 path = format_path(path)
237 ghc_env['PATH'] = os.pathsep.join([path, ghc_env.get("PATH", "")])
238 else:
239 # darwin
240 ghc_env['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, ghc_env.get("DYLD_LIBRARY_PATH", "")])
241
242 testopts_local.x = TestOptions()
243
244 # if timeout == -1 then we try to calculate a sensible value
245 if config.timeout == -1:
246 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
247
248 print('Timeout is ' + str(config.timeout))
249 print('Known ways: ' + ', '.join(config.other_ways))
250 print('Run ways: ' + ', '.join(config.run_ways))
251 print('Compile ways: ' + ', '.join(config.compile_ways))
252
253 # Try get allowed performance changes from the git commit.
254 try:
255 config.allowed_perf_changes = Perf.get_allowed_perf_changes()
256 except subprocess.CalledProcessError:
257 print('Failed to get allowed metric changes from the HEAD git commit message.')
258
259 print('Allowing performance changes in: ' + ', '.join(config.allowed_perf_changes.keys()))
260
261 # -----------------------------------------------------------------------------
262 # The main dude
263
264 if config.rootdirs == []:
265 config.rootdirs = ['.']
266
267 t_files = list(findTFiles(config.rootdirs))
268
269 print('Found', len(t_files), '.T files...')
270
271 t = getTestRun()
272
273 # Avoid cmd.exe built-in 'date' command on Windows
274 t.start_time = time.localtime()
275
276 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
277
278 # For reference
279 try:
280 print('Detected CPU features: ', cpu_features.get_cpu_features())
281 except Exception as e:
282 print('Failed to detect CPU features: ', e)
283
284 sys.stdout.flush()
285 # we output text, which cannot be unbuffered
286 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
287
288 if config.local:
289 tempdir = ''
290 else:
291 # See note [Running tests in /tmp]
292 tempdir = tempfile.mkdtemp('', 'ghctest-')
293
294 # opts.testdir should be quoted when used, to make sure the testsuite
295 # keeps working when it contains backward slashes, for example from
296 # using os.path.join. Windows native and mingw* python
297 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
298 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
299 # `os.path.sep = '/'`.
300 # To catch usage of unquoted opts.testdir early, insert some spaces into
301 # tempdir.
302 tempdir = os.path.join(tempdir, 'test spaces')
303
304 def cleanup_and_exit(exitcode):
305 if config.cleanup and tempdir:
306 shutil.rmtree(tempdir, ignore_errors=True)
307 exit(exitcode)
308
309 # First collect all the tests to be run
310 t_files_ok = True
311 for file in t_files:
312 if_verbose(2, '====> Scanning %s' % file)
313 newTestDir(tempdir, os.path.dirname(file))
314 try:
315 with io.open(file, encoding='utf8') as f:
316 src = f.read()
317
318 exec(src)
319 except Exception as e:
320 traceback.print_exc()
321 framework_fail(file, '', str(e))
322 t_files_ok = False
323
324 for name in config.only:
325 if t_files_ok:
326 # See Note [Mutating config.only]
327 framework_fail(name, '', 'test not found')
328 else:
329 # Let user fix .T file errors before reporting on unfound tests.
330 # The reason the test can not be found is likely because of those
331 # .T file errors.
332 pass
333
334 if config.list_broken:
335 print('')
336 print('Broken tests:')
337 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
338 print('')
339
340 if t.framework_failures:
341 print('WARNING:', len(t.framework_failures), 'framework failures!')
342 print('')
343 else:
344 # completion watcher
345 watcher = Watcher(len(parallelTests))
346
347 # Now run all the tests
348 try:
349 for oneTest in parallelTests:
350 if stopping():
351 break
352 oneTest(watcher)
353
354 # wait for parallel tests to finish
355 if not stopping():
356 watcher.wait()
357
358 # Run the following tests purely sequential
359 config.use_threads = False
360 for oneTest in aloneTests:
361 if stopping():
362 break
363 oneTest(watcher)
364 except KeyboardInterrupt:
365 pass
366
367 # flush everything before we continue
368 sys.stdout.flush()
369
370 # Warn if had to force skip perf tests (see Note force skip perf tests).
371 spacing = " "
372 if forceSkipPerfTests and not args.skip_perf_tests:
373 print()
374 print(str_warn('Skipping All Performance Tests') + ' `git status` exited with non-zero exit code.')
375 print(spacing + 'Git is required because performance test results are compared with the previous git commit\'s results (stored with git notes).')
376 print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')
377
378 # Warn of new metrics.
379 new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
380 if any(new_metrics):
381 if canGitStatus:
382 reason = 'the previous git commit doesn\'t have recorded metrics for the following tests.' + \
383 ' If the tests exist on the previous commit, then check it out and run the tests to generate the missing metrics.'
384 else:
385 reason = 'this is not a git repo so the previous git commit\'s metrics cannot be loaded from git notes:'
386 print()
387 print(str_warn('New Metrics') + ' these metrics trivially pass because ' + reason)
388 print(spacing + ('\n' + spacing).join(set([metric.test for metric in new_metrics])))
389
390 # Inform of how to accept metric changes.
391 if (len(t.unexpected_stat_failures) > 0):
392 print()
393 print(str_info("Some stats have changed") + " If this is expected, allow changes by appending the git commit message with this:")
394 print('-' * 25)
395 print(Perf.allow_changes_string(t.metrics))
396 print('-' * 25)
397
398 summary(t, sys.stdout, config.no_print_summary, True)
399
400 # Write perf stats if any exist or if a metrics file is specified.
401 stats = [stat for (_, stat) in t.metrics]
402 if hasMetricsFile:
403 print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
404 with open(config.metrics_file, 'a') as file:
405 file.write("\n" + Perf.format_perf_stat(stats))
406 elif canGitStatus and any(stats):
407 if is_worktree_dirty():
408 print()
409 print(str_warn('Working Tree is Dirty') + ' performance metrics will not be saved.' + \
410 ' Commit changes or use --metrics-file to save metrics to a file.')
411 else:
412 Perf.append_perf_stat(stats)
413
414 # Write summary
415 if config.summary_file:
416 with open(config.summary_file, 'w') as file:
417 summary(t, file)
418
419 if args.junit:
420 junit(t).write(args.junit)
421
422 if len(t.unexpected_failures) > 0 or \
423 len(t.unexpected_stat_failures) > 0 or \
424 len(t.framework_failures) > 0:
425 exitcode = 1
426 else:
427 exitcode = 0
428
429 cleanup_and_exit(exitcode)
430
431 # Note [Running tests in /tmp]
432 #
433 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
434 # the source directory without copying them to the test directory first.
435 #
436 # As an example, take a run_command test with a Makefile containing
437 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
438 # Foo.hi in the source directory. There are 2 problems with this:
439 # * Output files in the source directory won't get cleaned up automatically.
440 # * Two tests might (over)write the same output file.
441 #
442 # Tests that only fail when run concurrently with other tests are the
443 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
444 #
445 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
446 # making changes to quite a few tests. The problem is that
447 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
448 # `module Main where` does not produce Foo.o, as it would without
449 # -outputdir, but Main.o. See [1].
450 #
451 # Using -outputdir='.' is not foolproof anyway, since it does not change
452 # the destination of the final executable (Foo.exe).
453 #
454 # Another hardening method that could be tried is to `chmod -w` the
455 # source directory.
456 #
457 # By default we set LOCAL=1, because it makes it easier to inspect the
458 # test directory while working on a new test.
459 #
460 # [1]
461 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files