622e2ea5506c27133fd336a4beec9c3c7df983e1
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 from __future__ import print_function
8
9 import argparse
10 import signal
11 import sys
12 import os
13 import string
14 import shutil
15 import tempfile
16 import time
17 import re
18
19 # We don't actually need subprocess in runtests.py, but:
20 # * We do need it in testlibs.py
21 # * We can't import testlibs.py until after we have imported ctypes
22 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
23 # says "Aborted" and we fail with exit code 134.
24 # So we import it here first, so that the testsuite doesn't appear to fail.
25 import subprocess
26
27 from testutil import *
28 from testglobals import *
29 from junit import junit
30
31 # Readline sometimes spews out ANSI escapes for some values of TERM,
32 # which result in test failures. Thus set TERM to a nice, simple, safe
33 # value.
34 os.environ['TERM'] = 'vt100'
35 ghc_env['TERM'] = 'vt100'
36
37 global config
38 config = getConfig() # get it from testglobals
39
40 def signal_handler(signal, frame):
41 stopNow()
42
43 # -----------------------------------------------------------------------------
44 # cmd-line options
45
46 parser = argparse.ArgumentParser(description="GHC's testsuite driver")
47
48 parser.add_argument("-e", action='append', help="A string to execute from the command line.")
49 parser.add_argument("--config-file", action="append", help="config file")
50 parser.add_argument("--config", action='append', help="config field")
51 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
52 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
53 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
54 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
55 parser.add_argument("--way", action="append", help="just this way")
56 parser.add_argument("--skipway", action="append", help="skip this way")
57 parser.add_argument("--threads", type=int, help="threads to run simultaneously")
58 parser.add_argument("--check-files-written", help="check files aren't written by multiple tests") # NOTE: This doesn't seem to exist?
59 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
60 parser.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
61 parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
62
63 args = parser.parse_args()
64
65 for e in args.e:
66 exec(e)
67
68 for arg in args.config_file:
69 exec(open(arg).read())
70
71 for arg in args.config:
72 field, value = arg.split('=', 1)
73 setattr(config, field, value)
74
75 all_ways = config.run_ways+config.compile_ways+config.other_ways
76 config.rootdirs = args.rootdir
77 config.summary_file = args.summary_file
78 config.no_print_summary = args.no_print_summary
79
80 if args.only:
81 config.only = args.only
82 config.run_only_some_tests = True
83
84 if args.way:
85 for way in args.way:
86 if way not in all_ways:
87 print('WARNING: Unknown WAY %s in --way' % way)
88 else:
89 config.cmdline_ways += [way]
90 if way in config.other_ways:
91 config.run_ways += [way]
92 config.compile_ways += [way]
93
94 if args.skipway:
95 for way in args.skipway:
96 if way not in all_ways:
97 print('WARNING: Unknown WAY %s in --skipway' % way)
98
99 config.other_ways = [w for w in config.other_ways if w not in args.skipway]
100 config.run_ways = [w for w in config.run_ways if w not in args.skipway]
101 config.compile_ways = [w for w in config.compile_ways if w not in args.skipway]
102
103 if args.threads:
104 config.threads = args.threads
105 config.use_threads = True
106
107 if args.verbose:
108 config.verbose = args.verbose
109 config.skip_perf_tests = args.skip_perf_tests
110
111 config.cygwin = False
112 config.msys = False
113
114 if windows:
115 h = os.popen('uname -s', 'r')
116 v = h.read()
117 h.close()
118 if v.startswith("CYGWIN"):
119 config.cygwin = True
120 elif v.startswith("MINGW") or v.startswith("MSYS"):
121 # msys gives "MINGW32"
122 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
123 config.msys = True
124 else:
125 raise Exception("Can't detect Windows terminal type")
126
127 # Try to use UTF8
128 if windows:
129 import ctypes
130 # Windows and mingw* Python provide windll, msys2 python provides cdll.
131 if hasattr(ctypes, 'WinDLL'):
132 mydll = ctypes.WinDLL
133 else:
134 mydll = ctypes.CDLL
135
136 # This actually leaves the terminal in codepage 65001 (UTF8) even
137 # after python terminates. We ought really remember the old codepage
138 # and set it back.
139 kernel32 = mydll('kernel32.dll')
140 if kernel32.SetConsoleCP(65001) == 0:
141 raise Exception("Failure calling SetConsoleCP(65001)")
142 if kernel32.SetConsoleOutputCP(65001) == 0:
143 raise Exception("Failure calling SetConsoleOutputCP(65001)")
144
145 # register the interrupt handler
146 signal.signal(signal.SIGINT, signal_handler)
147 else:
148 # Try and find a utf8 locale to use
149 # First see if we already have a UTF8 locale
150 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
151 v = h.read()
152 h.close()
153 if v == '':
154 # We don't, so now see if 'locale -a' works
155 h = os.popen('locale -a | grep -F .', 'r')
156 v = h.read()
157 h.close()
158 if v != '':
159 # If it does then use the first utf8 locale that is available
160 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
161 v = h.readline().strip()
162 h.close()
163 if v != '':
164 os.environ['LC_ALL'] = v
165 ghc_env['LC_ALL'] = v
166 print("setting LC_ALL to", v)
167 else:
168 print('WARNING: No UTF8 locale found.')
169 print('You may get some spurious test failures.')
170
171 # This has to come after arg parsing as the args can change the compiler
172 get_compiler_info()
173
174 # Can't import this earlier as we need to know if threading will be
175 # enabled or not
176 from testlib import *
177
178 def format_path(path):
179 if windows:
180 if os.pathsep == ':':
181 # If using msys2 python instead of mingw we have to change the drive
182 # letter representation. Otherwise it thinks we're adding two env
183 # variables E and /Foo when we add E:/Foo.
184 path = re.sub('([a-zA-Z]):', '/\\1', path)
185 if config.cygwin:
186 # On cygwin we can't put "c:\foo" in $PATH, as : is a
187 # field separator. So convert to /cygdrive/c/foo instead.
188 # Other pythons use ; as the separator, so no problem.
189 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
190 path = re.sub('\\\\', '/', path)
191 return path
192
193 # On Windows we need to set $PATH to include the paths to all the DLLs
194 # in order for the dynamic library tests to work.
195 if windows or darwin:
196 pkginfo = str(getStdout([config.ghc_pkg, 'dump']))
197 topdir = config.libdir
198 if windows:
199 mingw = os.path.abspath(os.path.join(topdir, '../mingw/bin'))
200 mingw = format_path(mingw)
201 ghc_env['PATH'] = os.pathsep.join([ghc_env.get("PATH", ""), mingw])
202 for line in pkginfo.split('\n'):
203 if line.startswith('library-dirs:'):
204 path = line.rstrip()
205 path = re.sub('^library-dirs: ', '', path)
206 # Use string.replace instead of re.sub, because re.sub
207 # interprets backslashes in the replacement string as
208 # escape sequences.
209 path = path.replace('$topdir', topdir)
210 if path.startswith('"'):
211 path = re.sub('^"(.*)"$', '\\1', path)
212 path = re.sub('\\\\(.)', '\\1', path)
213 if windows:
214 path = format_path(path)
215 ghc_env['PATH'] = os.pathsep.join([path, ghc_env.get("PATH", "")])
216 else:
217 # darwin
218 ghc_env['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, ghc_env.get("DYLD_LIBRARY_PATH", "")])
219
220 global testopts_local
221 testopts_local.x = TestOptions()
222
223 # if timeout == -1 then we try to calculate a sensible value
224 if config.timeout == -1:
225 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
226
227 print('Timeout is ' + str(config.timeout))
228
229 # -----------------------------------------------------------------------------
230 # The main dude
231
232 if config.rootdirs == []:
233 config.rootdirs = ['.']
234
235 t_files = list(findTFiles(config.rootdirs))
236
237 print('Found', len(t_files), '.T files...')
238
239 t = getTestRun()
240
241 # Avoid cmd.exe built-in 'date' command on Windows
242 t.start_time = time.localtime()
243
244 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
245
246 sys.stdout.flush()
247 # we output text, which cannot be unbuffered
248 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
249
250 if config.local:
251 tempdir = ''
252 else:
253 # See note [Running tests in /tmp]
254 tempdir = tempfile.mkdtemp('', 'ghctest-')
255
256 # opts.testdir should be quoted when used, to make sure the testsuite
257 # keeps working when it contains backward slashes, for example from
258 # using os.path.join. Windows native and mingw* python
259 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
260 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
261 # `os.path.sep = '/'`.
262 # To catch usage of unquoted opts.testdir early, insert some spaces into
263 # tempdir.
264 tempdir = os.path.join(tempdir, 'test spaces')
265
266 def cleanup_and_exit(exitcode):
267 if config.cleanup and tempdir:
268 shutil.rmtree(tempdir, ignore_errors=True)
269 exit(exitcode)
270
271 # First collect all the tests to be run
272 t_files_ok = True
273 for file in t_files:
274 if_verbose(2, '====> Scanning %s' % file)
275 newTestDir(tempdir, os.path.dirname(file))
276 try:
277 with io.open(file, encoding='utf8') as f:
278 src = f.read()
279
280 exec(src)
281 except Exception as e:
282 traceback.print_exc()
283 framework_fail(file, '', str(e))
284 t_files_ok = False
285
286 for name in config.only:
287 if t_files_ok:
288 # See Note [Mutating config.only]
289 framework_fail(name, '', 'test not found')
290 else:
291 # Let user fix .T file errors before reporting on unfound tests.
292 # The reson the test can not be found is likely because of those
293 # .T file errors.
294 pass
295
296 if config.list_broken:
297 global brokens
298 print('')
299 print('Broken tests:')
300 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
301 print('')
302
303 if t.framework_failures:
304 print('WARNING:', len(framework_failures), 'framework failures!')
305 print('')
306 else:
307 # completion watcher
308 watcher = Watcher(len(parallelTests))
309
310 # Now run all the tests
311 for oneTest in parallelTests:
312 if stopping():
313 break
314 oneTest(watcher)
315
316 # wait for parallel tests to finish
317 if not stopping():
318 watcher.wait()
319
320 # Run the following tests purely sequential
321 config.use_threads = False
322 for oneTest in aloneTests:
323 if stopping():
324 break
325 oneTest(watcher)
326
327 # flush everything before we continue
328 sys.stdout.flush()
329
330 summary(t, sys.stdout, config.no_print_summary)
331
332 if config.summary_file:
333 with open(config.summary_file, 'w') as file:
334 summary(t, file)
335
336 if args.junit:
337 junit(t).write(args.junit)
338
339 if len(t.unexpected_failures) > 0 or \
340 len(t.unexpected_stat_failures) > 0 or \
341 len(t.framework_failures) > 0:
342 exitcode = 1
343 else:
344 exitcode = 0
345
346 cleanup_and_exit(exitcode)
347
348 # Note [Running tests in /tmp]
349 #
350 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
351 # the source directory without copying them to the test directory first.
352 #
353 # As an example, take a run_command test with a Makefile containing
354 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
355 # Foo.hi in the source directory. There are 2 problems with this:
356 # * Output files in the source directory won't get cleaned up automatically.
357 # * Two tests might (over)write the same output file.
358 #
359 # Tests that only fail when run concurrently with other tests are the
360 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
361 #
362 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
363 # making changes to quite a few tests. The problem is that
364 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
365 # `module Main where` does not produce Foo.o, as it would without
366 # -outputdir, but Main.o. See [1].
367 #
368 # Using -outputdir='.' is not foolproof anyway, since it does not change
369 # the destination of the final executable (Foo.exe).
370 #
371 # Another hardening method that could be tried is to `chmod -w` the
372 # source directory.
373 #
374 # By default we set LOCAL=1, because it makes it easier to inspect the
375 # test directory while working on a new test.
376 #
377 # [1]
378 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files