7e4f375a2c2422e8c12abc431764fb3d66c495ab
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 from __future__ import print_function
8
9 import signal
10 import sys
11 import os
12 import string
13 import getopt
14 import shutil
15 import tempfile
16 import time
17 import re
18
19 # We don't actually need subprocess in runtests.py, but:
20 # * We do need it in testlibs.py
21 # * We can't import testlibs.py until after we have imported ctypes
22 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
23 # says "Aborted" and we fail with exit code 134.
24 # So we import it here first, so that the testsuite doesn't appear to fail.
25 import subprocess
26
27 from testutil import *
28 from testglobals import *
29
30 # Readline sometimes spews out ANSI escapes for some values of TERM,
31 # which result in test failures. Thus set TERM to a nice, simple, safe
32 # value.
33 os.environ['TERM'] = 'vt100'
34
35 global config
36 config = getConfig() # get it from testglobals
37
38 def signal_handler(signal, frame):
39 stopNow()
40
41 # -----------------------------------------------------------------------------
42 # cmd-line options
43
44 long_options = [
45 "configfile=", # config file
46 "config=", # config field
47 "rootdir=", # root of tree containing tests (default: .)
48 "summary-file=", # file in which to save the (human-readable) summary
49 "no-print-summary=", # should we print the summary?
50 "only=", # just this test (can be give multiple --only= flags)
51 "way=", # just this way
52 "skipway=", # skip this way
53 "threads=", # threads to run simultaneously
54 "check-files-written", # check files aren't written by multiple tests
55 "verbose=", # verbose (0,1,2 so far)
56 "skip-perf-tests", # skip performance tests
57 ]
58
59 opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
60
61 for opt,arg in opts:
62 if opt == '--configfile':
63 exec(open(arg).read())
64
65 # -e is a string to execute from the command line. For example:
66 # testframe -e 'config.compiler=ghc-5.04'
67 if opt == '-e':
68 exec(arg)
69
70 if opt == '--config':
71 field, value = arg.split('=', 1)
72 setattr(config, field, value)
73
74 if opt == '--rootdir':
75 config.rootdirs.append(arg)
76
77 if opt == '--summary-file':
78 config.summary_file = arg
79
80 if opt == '--no-print-summary':
81 config.no_print_summary = True
82
83 if opt == '--only':
84 config.run_only_some_tests = True
85 config.only.add(arg)
86
87 if opt == '--way':
88 if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
89 sys.stderr.write("ERROR: requested way \'" +
90 arg + "\' does not exist\n")
91 sys.exit(1)
92 config.cmdline_ways = [arg] + config.cmdline_ways
93 if (arg in config.other_ways):
94 config.run_ways = [arg] + config.run_ways
95 config.compile_ways = [arg] + config.compile_ways
96
97 if opt == '--skipway':
98 if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
99 sys.stderr.write("ERROR: requested way \'" +
100 arg + "\' does not exist\n")
101 sys.exit(1)
102 config.other_ways = [w for w in config.other_ways if w != arg]
103 config.run_ways = [w for w in config.run_ways if w != arg]
104 config.compile_ways = [w for w in config.compile_ways if w != arg]
105
106 if opt == '--threads':
107 config.threads = int(arg)
108 config.use_threads = 1
109
110 if opt == '--skip-perf-tests':
111 config.skip_perf_tests = True
112
113 if opt == '--verbose':
114 if arg not in ["0","1","2","3","4","5"]:
115 sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2,3,4 or 5" % arg)
116 sys.exit(1)
117 config.verbose = int(arg)
118
119
120 config.cygwin = False
121 config.msys = False
122
123 if windows:
124 h = os.popen('uname -s', 'r')
125 v = h.read()
126 h.close()
127 if v.startswith("CYGWIN"):
128 config.cygwin = True
129 elif v.startswith("MINGW") or v.startswith("MSYS"):
130 # msys gives "MINGW32"
131 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
132 config.msys = True
133 else:
134 raise Exception("Can't detect Windows terminal type")
135
136 # Try to use UTF8
137 if windows:
138 import ctypes
139 # Windows and mingw* Python provide windll, msys2 python provides cdll.
140 if hasattr(ctypes, 'WinDLL'):
141 mydll = ctypes.WinDLL
142 else:
143 mydll = ctypes.CDLL
144
145 # This actually leaves the terminal in codepage 65001 (UTF8) even
146 # after python terminates. We ought really remember the old codepage
147 # and set it back.
148 kernel32 = mydll('kernel32.dll')
149 if kernel32.SetConsoleCP(65001) == 0:
150 raise Exception("Failure calling SetConsoleCP(65001)")
151 if kernel32.SetConsoleOutputCP(65001) == 0:
152 raise Exception("Failure calling SetConsoleOutputCP(65001)")
153
154 # register the interrupt handler
155 signal.signal(signal.SIGINT, signal_handler)
156 else:
157 # Try and find a utf8 locale to use
158 # First see if we already have a UTF8 locale
159 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
160 v = h.read()
161 h.close()
162 if v == '':
163 # We don't, so now see if 'locale -a' works
164 h = os.popen('locale -a', 'r')
165 v = h.read()
166 h.close()
167 if v != '':
168 # If it does then use the first utf8 locale that is available
169 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
170 v = h.readline().strip()
171 h.close()
172 if v != '':
173 os.environ['LC_ALL'] = v
174 print("setting LC_ALL to", v)
175 else:
176 print('WARNING: No UTF8 locale found.')
177 print('You may get some spurious test failures.')
178
179 # This has to come after arg parsing as the args can change the compiler
180 get_compiler_info()
181
182 # Can't import this earlier as we need to know if threading will be
183 # enabled or not
184 from testlib import *
185
186 # On Windows we need to set $PATH to include the paths to all the DLLs
187 # in order for the dynamic library tests to work.
188 if windows or darwin:
189 pkginfo = str(getStdout([config.ghc_pkg, 'dump']))
190 topdir = config.libdir
191 if windows:
192 mingw = os.path.join(topdir, '../mingw/bin')
193 os.environ['PATH'] = os.pathsep.join([os.environ.get("PATH", ""), mingw])
194 for line in pkginfo.split('\n'):
195 if line.startswith('library-dirs:'):
196 path = line.rstrip()
197 path = re.sub('^library-dirs: ', '', path)
198 # Use string.replace instead of re.sub, because re.sub
199 # interprets backslashes in the replacement string as
200 # escape sequences.
201 path = path.replace('$topdir', topdir)
202 if path.startswith('"'):
203 path = re.sub('^"(.*)"$', '\\1', path)
204 path = re.sub('\\\\(.)', '\\1', path)
205 if windows:
206 if config.cygwin:
207 # On cygwin we can't put "c:\foo" in $PATH, as : is a
208 # field separator. So convert to /cygdrive/c/foo instead.
209 # Other pythons use ; as the separator, so no problem.
210 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
211 path = re.sub('\\\\', '/', path)
212 os.environ['PATH'] = os.pathsep.join([path, os.environ.get("PATH", "")])
213 else:
214 # darwin
215 os.environ['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, os.environ.get("DYLD_LIBRARY_PATH", "")])
216
217 global testopts_local
218 testopts_local.x = TestOptions()
219
220 # if timeout == -1 then we try to calculate a sensible value
221 if config.timeout == -1:
222 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
223
224 print('Timeout is ' + str(config.timeout))
225
226 # -----------------------------------------------------------------------------
227 # The main dude
228
229 if config.rootdirs == []:
230 config.rootdirs = ['.']
231
232 t_files = list(findTFiles(config.rootdirs))
233
234 print('Found', len(t_files), '.T files...')
235
236 t = getTestRun()
237
238 # Avoid cmd.exe built-in 'date' command on Windows
239 t.start_time = time.localtime()
240
241 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
242
243 sys.stdout.flush()
244 # we output text, which cannot be unbuffered
245 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
246
247 if config.local:
248 tempdir = ''
249 else:
250 # See note [Running tests in /tmp]
251 tempdir = tempfile.mkdtemp('', 'ghctest-')
252
253 # opts.testdir should be quoted when used, to make sure the testsuite
254 # keeps working when it contains backward slashes, for example from
255 # using os.path.join. Windows native and mingw* python
256 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
257 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
258 # `os.path.sep = '/'`.
259 # To catch usage of unquoted opts.testdir early, insert some spaces into
260 # tempdir.
261 tempdir = os.path.join(tempdir, 'test spaces')
262
263 def cleanup_and_exit(exitcode):
264 if config.cleanup and tempdir:
265 shutil.rmtree(tempdir, ignore_errors=True)
266 exit(exitcode)
267
268 # First collect all the tests to be run
269 t_files_ok = True
270 for file in t_files:
271 if_verbose(2, '====> Scanning %s' % file)
272 newTestDir(tempdir, os.path.dirname(file))
273 try:
274 with io.open(file, encoding='utf8') as f:
275 src = f.read()
276
277 exec(src)
278 except Exception as e:
279 traceback.print_exc()
280 framework_fail(file, '', str(e))
281 t_files_ok = False
282
283 for name in config.only:
284 if t_files_ok:
285 # See Note [Mutating config.only]
286 framework_fail(name, '', 'test not found')
287 else:
288 # Let user fix .T file errors before reporting on unfound tests.
289 # The reson the test can not be found is likely because of those
290 # .T file errors.
291 pass
292
293 if config.list_broken:
294 global brokens
295 print('')
296 print('Broken tests:')
297 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
298 print('')
299
300 if t.framework_failures:
301 print('WARNING:', len(framework_failures), 'framework failures!')
302 print('')
303 else:
304 # completion watcher
305 watcher = Watcher(len(parallelTests))
306
307 # Now run all the tests
308 for oneTest in parallelTests:
309 if stopping():
310 break
311 oneTest(watcher)
312
313 # wait for parallel tests to finish
314 if not stopping():
315 watcher.wait()
316
317 # Run the following tests purely sequential
318 config.use_threads = False
319 for oneTest in aloneTests:
320 if stopping():
321 break
322 oneTest(watcher)
323
324 # flush everything before we continue
325 sys.stdout.flush()
326
327 summary(t, sys.stdout, config.no_print_summary)
328
329 if config.summary_file != '':
330 with open(config.summary_file, 'w') as file:
331 summary(t, file)
332
333 cleanup_and_exit(0)
334
335 # Note [Running tests in /tmp]
336 #
337 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
338 # the source directory without copying them to the test directory first.
339 #
340 # As an example, take a run_command test with a Makefile containing
341 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
342 # Foo.hi in the source directory. There are 2 problems with this:
343 # * Output files in the source directory won't get cleaned up automatically.
344 # * Two tests might (over)write the same output file.
345 #
346 # Tests that only fail when run concurrently with other tests are the
347 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
348 #
349 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
350 # making changes to quite a few tests. The problem is that
351 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
352 # `module Main where` does not produce Foo.o, as it would without
353 # -outputdir, but Main.o. See [1].
354 #
355 # Using -outputdir='.' is not foolproof anyway, since it does not change
356 # the destination of the final executable (Foo.exe).
357 #
358 # Another hardening method that could be tried is to `chmod -w` the
359 # source directory.
360 #
361 # By default we set LOCAL=1, because it makes it easier to inspect the
362 # test directory while working on a new test.
363 #
364 # [1]
365 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files