Switched out optparse for argparse in runtests.py
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 from __future__ import print_function
8
9 import argparse
10 import signal
11 import sys
12 import os
13 import string
14 import shutil
15 import tempfile
16 import time
17 import re
18
19 # We don't actually need subprocess in runtests.py, but:
20 # * We do need it in testlibs.py
21 # * We can't import testlibs.py until after we have imported ctypes
22 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
23 # says "Aborted" and we fail with exit code 134.
24 # So we import it here first, so that the testsuite doesn't appear to fail.
25 import subprocess
26
27 from testutil import *
28 from testglobals import *
29
30 # Readline sometimes spews out ANSI escapes for some values of TERM,
31 # which result in test failures. Thus set TERM to a nice, simple, safe
32 # value.
33 os.environ['TERM'] = 'vt100'
34
35 global config
36 config = getConfig() # get it from testglobals
37
38 def signal_handler(signal, frame):
39 stopNow()
40
41 # -----------------------------------------------------------------------------
42 # cmd-line options
43
44 parser = argparse.ArgumentParser(description="GHC's testsuite driver",
45 allow_abbrev=False)
46
47 parser.add_argument("-e", action='append', help="A string to execute from the command line.")
48 parser.add_argument("--config-file", action="append", help="config file")
49 parser.add_argument("--config", action='append', help="config field")
50 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
51 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
52 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
53 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
54 parser.add_argument("--way", choices=config.run_ways+config.compile_ways+config.other_ways, help="just this way")
55 parser.add_argument("--skipway", action="append", choices=config.run_ways+config.compile_ways+config.other_ways, help="skip this way")
56 parser.add_argument("--threads", type=int, help="threads to run simultaneously")
57 parser.add_argument("--check-files-written", help="check files aren't written by multiple tests") # NOTE: This doesn't seem to exist?
58 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
59 parser.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
60
61 args = parser.parse_args()
62
63 for e in args.e:
64 exec(e)
65
66 for arg in args.config_file:
67 exec(open(arg).read())
68
69 for arg in args.config:
70 field, value = arg.split('=', 1)
71 setattr(config, field, value)
72
73 config.rootdirs = args.rootdir
74 config.summary_file = args.summary_file
75 config.no_print_summary = args.no_print_summary
76
77 if args.only:
78 config.only = args.only
79 config.run_only_some_tests = True
80
81 if args.way:
82 config.cmdline_ways = [args.way] + config.cmdline_ways
83 if (args.way in config.other_ways):
84 config.run_ways = [args.way] + config.run_ways
85 config.compile_ways = [args.way] + config.compile_ways
86
87 if args.skipway:
88 config.other_ways = [w for w in config.other_ways if w != args.skipway]
89 config.run_ways = [w for w in config.run_ways if w != args.skipway]
90 config.compile_ways = [w for w in config.compile_ways if w != args.skipway]
91
92 if args.threads:
93 config.threads = args.threads
94 config.use_threads = True
95
96 if args.verbose:
97 config.verbose = args.verbose
98 config.skip_perf_tests = args.skip_perf_tests
99
100 config.cygwin = False
101 config.msys = False
102
103 if windows:
104 h = os.popen('uname -s', 'r')
105 v = h.read()
106 h.close()
107 if v.startswith("CYGWIN"):
108 config.cygwin = True
109 elif v.startswith("MINGW") or v.startswith("MSYS"):
110 # msys gives "MINGW32"
111 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
112 config.msys = True
113 else:
114 raise Exception("Can't detect Windows terminal type")
115
116 # Try to use UTF8
117 if windows:
118 import ctypes
119 # Windows and mingw* Python provide windll, msys2 python provides cdll.
120 if hasattr(ctypes, 'WinDLL'):
121 mydll = ctypes.WinDLL
122 else:
123 mydll = ctypes.CDLL
124
125 # This actually leaves the terminal in codepage 65001 (UTF8) even
126 # after python terminates. We ought really remember the old codepage
127 # and set it back.
128 kernel32 = mydll('kernel32.dll')
129 if kernel32.SetConsoleCP(65001) == 0:
130 raise Exception("Failure calling SetConsoleCP(65001)")
131 if kernel32.SetConsoleOutputCP(65001) == 0:
132 raise Exception("Failure calling SetConsoleOutputCP(65001)")
133
134 # register the interrupt handler
135 signal.signal(signal.SIGINT, signal_handler)
136 else:
137 # Try and find a utf8 locale to use
138 # First see if we already have a UTF8 locale
139 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
140 v = h.read()
141 h.close()
142 if v == '':
143 # We don't, so now see if 'locale -a' works
144 h = os.popen('locale -a', 'r')
145 v = h.read()
146 h.close()
147 if v != '':
148 # If it does then use the first utf8 locale that is available
149 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
150 v = h.readline().strip()
151 h.close()
152 if v != '':
153 os.environ['LC_ALL'] = v
154 print("setting LC_ALL to", v)
155 else:
156 print('WARNING: No UTF8 locale found.')
157 print('You may get some spurious test failures.')
158
159 # This has to come after arg parsing as the args can change the compiler
160 get_compiler_info()
161
162 # Can't import this earlier as we need to know if threading will be
163 # enabled or not
164 from testlib import *
165
166 # On Windows we need to set $PATH to include the paths to all the DLLs
167 # in order for the dynamic library tests to work.
168 if windows or darwin:
169 pkginfo = str(getStdout([config.ghc_pkg, 'dump']))
170 topdir = config.libdir
171 if windows:
172 mingw = os.path.join(topdir, '../mingw/bin')
173 os.environ['PATH'] = os.pathsep.join([os.environ.get("PATH", ""), mingw])
174 for line in pkginfo.split('\n'):
175 if line.startswith('library-dirs:'):
176 path = line.rstrip()
177 path = re.sub('^library-dirs: ', '', path)
178 # Use string.replace instead of re.sub, because re.sub
179 # interprets backslashes in the replacement string as
180 # escape sequences.
181 path = path.replace('$topdir', topdir)
182 if path.startswith('"'):
183 path = re.sub('^"(.*)"$', '\\1', path)
184 path = re.sub('\\\\(.)', '\\1', path)
185 if windows:
186 if config.cygwin:
187 # On cygwin we can't put "c:\foo" in $PATH, as : is a
188 # field separator. So convert to /cygdrive/c/foo instead.
189 # Other pythons use ; as the separator, so no problem.
190 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
191 path = re.sub('\\\\', '/', path)
192 os.environ['PATH'] = os.pathsep.join([path, os.environ.get("PATH", "")])
193 else:
194 # darwin
195 os.environ['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, os.environ.get("DYLD_LIBRARY_PATH", "")])
196
197 global testopts_local
198 testopts_local.x = TestOptions()
199
200 # if timeout == -1 then we try to calculate a sensible value
201 if config.timeout == -1:
202 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
203
204 print('Timeout is ' + str(config.timeout))
205
206 # -----------------------------------------------------------------------------
207 # The main dude
208
209 if config.rootdirs == []:
210 config.rootdirs = ['.']
211
212 t_files = list(findTFiles(config.rootdirs))
213
214 print('Found', len(t_files), '.T files...')
215
216 t = getTestRun()
217
218 # Avoid cmd.exe built-in 'date' command on Windows
219 t.start_time = time.localtime()
220
221 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
222
223 sys.stdout.flush()
224 # we output text, which cannot be unbuffered
225 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
226
227 if config.local:
228 tempdir = ''
229 else:
230 # See note [Running tests in /tmp]
231 tempdir = tempfile.mkdtemp('', 'ghctest-')
232
233 # opts.testdir should be quoted when used, to make sure the testsuite
234 # keeps working when it contains backward slashes, for example from
235 # using os.path.join. Windows native and mingw* python
236 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
237 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
238 # `os.path.sep = '/'`.
239 # To catch usage of unquoted opts.testdir early, insert some spaces into
240 # tempdir.
241 tempdir = os.path.join(tempdir, 'test spaces')
242
243 def cleanup_and_exit(exitcode):
244 if config.cleanup and tempdir:
245 shutil.rmtree(tempdir, ignore_errors=True)
246 exit(exitcode)
247
248 # First collect all the tests to be run
249 t_files_ok = True
250 for file in t_files:
251 if_verbose(2, '====> Scanning %s' % file)
252 newTestDir(tempdir, os.path.dirname(file))
253 try:
254 with io.open(file, encoding='utf8') as f:
255 src = f.read()
256
257 exec(src)
258 except Exception as e:
259 traceback.print_exc()
260 framework_fail(file, '', str(e))
261 t_files_ok = False
262
263 for name in config.only:
264 if t_files_ok:
265 # See Note [Mutating config.only]
266 framework_fail(name, '', 'test not found')
267 else:
268 # Let user fix .T file errors before reporting on unfound tests.
269 # The reson the test can not be found is likely because of those
270 # .T file errors.
271 pass
272
273 if config.list_broken:
274 global brokens
275 print('')
276 print('Broken tests:')
277 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
278 print('')
279
280 if t.framework_failures:
281 print('WARNING:', len(framework_failures), 'framework failures!')
282 print('')
283 else:
284 # completion watcher
285 watcher = Watcher(len(parallelTests))
286
287 # Now run all the tests
288 for oneTest in parallelTests:
289 if stopping():
290 break
291 oneTest(watcher)
292
293 # wait for parallel tests to finish
294 if not stopping():
295 watcher.wait()
296
297 # Run the following tests purely sequential
298 config.use_threads = False
299 for oneTest in aloneTests:
300 if stopping():
301 break
302 oneTest(watcher)
303
304 # flush everything before we continue
305 sys.stdout.flush()
306
307 summary(t, sys.stdout, config.no_print_summary)
308
309 if config.summary_file:
310 with open(config.summary_file, 'w') as file:
311 summary(t, file)
312
313 cleanup_and_exit(0)
314
315 # Note [Running tests in /tmp]
316 #
317 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
318 # the source directory without copying them to the test directory first.
319 #
320 # As an example, take a run_command test with a Makefile containing
321 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
322 # Foo.hi in the source directory. There are 2 problems with this:
323 # * Output files in the source directory won't get cleaned up automatically.
324 # * Two tests might (over)write the same output file.
325 #
326 # Tests that only fail when run concurrently with other tests are the
327 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
328 #
329 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
330 # making changes to quite a few tests. The problem is that
331 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
332 # `module Main where` does not produce Foo.o, as it would without
333 # -outputdir, but Main.o. See [1].
334 #
335 # Using -outputdir='.' is not foolproof anyway, since it does not change
336 # the destination of the final executable (Foo.exe).
337 #
338 # Another hardening method that could be tried is to `chmod -w` the
339 # source directory.
340 #
341 # By default we set LOCAL=1, because it makes it easier to inspect the
342 # test directory while working on a new test.
343 #
344 # [1]
345 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files