testsuite: Remove Unicode literals from driver
[ghc.git] / testsuite / driver / runtests.py
1 #!/usr/bin/env python3
2
3 #
4 # (c) Simon Marlow 2002
5 #
6
7 from __future__ import print_function
8
9 import signal
10 import sys
11 import os
12 import string
13 import getopt
14 import platform
15 import shutil
16 import tempfile
17 import time
18 import re
19
20 # We don't actually need subprocess in runtests.py, but:
21 # * We do need it in testlibs.py
22 # * We can't import testlibs.py until after we have imported ctypes
23 # * If we import ctypes before subprocess on cygwin, then sys.exit(0)
24 # says "Aborted" and we fail with exit code 134.
25 # So we import it here first, so that the testsuite doesn't appear to fail.
26 import subprocess
27
28 PYTHON3 = sys.version_info >= (3, 0)
29
30 from testutil import *
31 from testglobals import *
32
33 # Readline sometimes spews out ANSI escapes for some values of TERM,
34 # which result in test failures. Thus set TERM to a nice, simple, safe
35 # value.
36 os.environ['TERM'] = 'vt100'
37
38 global config
39 config = getConfig() # get it from testglobals
40
41 def signal_handler(signal, frame):
42 stopNow()
43
44 # -----------------------------------------------------------------------------
45 # cmd-line options
46
47 long_options = [
48 "configfile=", # config file
49 "config=", # config field
50 "rootdir=", # root of tree containing tests (default: .)
51 "summary-file=", # file in which to save the (human-readable) summary
52 "no-print-summary=", # should we print the summary?
53 "only=", # just this test (can be give multiple --only= flags)
54 "way=", # just this way
55 "skipway=", # skip this way
56 "threads=", # threads to run simultaneously
57 "check-files-written", # check files aren't written by multiple tests
58 "verbose=", # verbose (0,1,2 so far)
59 "skip-perf-tests", # skip performance tests
60 ]
61
62 opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
63
64 for opt,arg in opts:
65 if opt == '--configfile':
66 exec(open(arg).read())
67
68 # -e is a string to execute from the command line. For example:
69 # testframe -e 'config.compiler=ghc-5.04'
70 if opt == '-e':
71 exec(arg)
72
73 if opt == '--config':
74 field, value = arg.split('=', 1)
75 setattr(config, field, value)
76
77 if opt == '--rootdir':
78 config.rootdirs.append(arg)
79
80 if opt == '--summary-file':
81 config.summary_file = arg
82
83 if opt == '--no-print-summary':
84 config.no_print_summary = True
85
86 if opt == '--only':
87 config.run_only_some_tests = True
88 config.only.add(arg)
89
90 if opt == '--way':
91 if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
92 sys.stderr.write("ERROR: requested way \'" +
93 arg + "\' does not exist\n")
94 sys.exit(1)
95 config.cmdline_ways = [arg] + config.cmdline_ways
96 if (arg in config.other_ways):
97 config.run_ways = [arg] + config.run_ways
98 config.compile_ways = [arg] + config.compile_ways
99
100 if opt == '--skipway':
101 if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
102 sys.stderr.write("ERROR: requested way \'" +
103 arg + "\' does not exist\n")
104 sys.exit(1)
105 config.other_ways = [w for w in config.other_ways if w != arg]
106 config.run_ways = [w for w in config.run_ways if w != arg]
107 config.compile_ways = [w for w in config.compile_ways if w != arg]
108
109 if opt == '--threads':
110 config.threads = int(arg)
111 config.use_threads = 1
112
113 if opt == '--skip-perf-tests':
114 config.skip_perf_tests = True
115
116 if opt == '--verbose':
117 if arg not in ["0","1","2","3","4"]:
118 sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2,3 or 4" % arg)
119 sys.exit(1)
120 config.verbose = int(arg)
121
122
123 if config.use_threads == 1:
124 # Trac #1558 says threads don't work in python 2.4.4, but do
125 # in 2.5.2. Probably >= 2.5 is sufficient, but let's be
126 # conservative here.
127 # Some versions of python have things like '1c1' for some of
128 # these components (see trac #3091), but int() chokes on the
129 # 'c1', so we drop it.
130 (maj, min, pat) = platform.python_version_tuple()
131 # We wrap maj, min, and pat in str() to work around a bug in python
132 # 2.6.1
133 maj = int(re.sub('[^0-9].*', '', str(maj)))
134 min = int(re.sub('[^0-9].*', '', str(min)))
135 pat = int(re.sub('[^0-9].*', '', str(pat)))
136 if (maj, min) < (2, 6):
137 print("Python < 2.6 is not supported")
138 sys.exit(1)
139 # We also need to disable threads for python 2.7.2, because of
140 # this bug: http://bugs.python.org/issue13817
141 elif (maj, min, pat) == (2, 7, 2):
142 print("Warning: Ignoring request to use threads as python version is 2.7.2")
143 print("See http://bugs.python.org/issue13817 for details.")
144 config.use_threads = 0
145
146 config.cygwin = False
147 config.msys = False
148
149 if windows:
150 h = os.popen('uname -s', 'r')
151 v = h.read()
152 h.close()
153 if v.startswith("CYGWIN"):
154 config.cygwin = True
155 elif v.startswith("MINGW") or v.startswith("MSYS"):
156 # msys gives "MINGW32"
157 # msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
158 config.msys = True
159 else:
160 raise Exception("Can't detect Windows terminal type")
161
162 # Try to use UTF8
163 if windows:
164 import ctypes
165 # Windows and mingw* Python provide windll, msys2 python provides cdll.
166 if hasattr(ctypes, 'WinDLL'):
167 mydll = ctypes.WinDLL
168 else:
169 mydll = ctypes.CDLL
170
171 # This actually leaves the terminal in codepage 65001 (UTF8) even
172 # after python terminates. We ought really remember the old codepage
173 # and set it back.
174 kernel32 = mydll('kernel32.dll')
175 if kernel32.SetConsoleCP(65001) == 0:
176 raise Exception("Failure calling SetConsoleCP(65001)")
177 if kernel32.SetConsoleOutputCP(65001) == 0:
178 raise Exception("Failure calling SetConsoleOutputCP(65001)")
179
180 # register the interrupt handler
181 signal.signal(signal.SIGINT, signal_handler)
182 else:
183 # Try and find a utf8 locale to use
184 # First see if we already have a UTF8 locale
185 h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
186 v = h.read()
187 h.close()
188 if v == '':
189 # We don't, so now see if 'locale -a' works
190 h = os.popen('locale -a', 'r')
191 v = h.read()
192 h.close()
193 if v != '':
194 # If it does then use the first utf8 locale that is available
195 h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
196 v = h.readline().strip()
197 h.close()
198 if v != '':
199 os.environ['LC_ALL'] = v
200 print("setting LC_ALL to", v)
201 else:
202 print('WARNING: No UTF8 locale found.')
203 print('You may get some spurious test failures.')
204
205 # This has to come after arg parsing as the args can change the compiler
206 get_compiler_info()
207
208 # Can't import this earlier as we need to know if threading will be
209 # enabled or not
210 from testlib import *
211
212 # On Windows we need to set $PATH to include the paths to all the DLLs
213 # in order for the dynamic library tests to work.
214 if windows or darwin:
215 pkginfo = str(getStdout([config.ghc_pkg, 'dump']))
216 topdir = config.libdir
217 if windows:
218 mingw = os.path.join(topdir, '../mingw/bin')
219 os.environ['PATH'] = os.pathsep.join([os.environ.get("PATH", ""), mingw])
220 for line in pkginfo.split('\n'):
221 if line.startswith('library-dirs:'):
222 path = line.rstrip()
223 path = re.sub('^library-dirs: ', '', path)
224 # Use string.replace instead of re.sub, because re.sub
225 # interprets backslashes in the replacement string as
226 # escape sequences.
227 path = path.replace('$topdir', topdir)
228 if path.startswith('"'):
229 path = re.sub('^"(.*)"$', '\\1', path)
230 path = re.sub('\\\\(.)', '\\1', path)
231 if windows:
232 if config.cygwin:
233 # On cygwin we can't put "c:\foo" in $PATH, as : is a
234 # field separator. So convert to /cygdrive/c/foo instead.
235 # Other pythons use ; as the separator, so no problem.
236 path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
237 path = re.sub('\\\\', '/', path)
238 os.environ['PATH'] = os.pathsep.join([path, os.environ.get("PATH", "")])
239 else:
240 # darwin
241 os.environ['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, os.environ.get("DYLD_LIBRARY_PATH", "")])
242
243 global testopts_local
244 testopts_local.x = TestOptions()
245
246 # if timeout == -1 then we try to calculate a sensible value
247 if config.timeout == -1:
248 config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
249
250 print('Timeout is ' + str(config.timeout))
251
252 # -----------------------------------------------------------------------------
253 # The main dude
254
255 if config.rootdirs == []:
256 config.rootdirs = ['.']
257
258 t_files = list(findTFiles(config.rootdirs))
259
260 print('Found', len(t_files), '.T files...')
261
262 t = getTestRun()
263
264 # Avoid cmd.exe built-in 'date' command on Windows
265 t.start_time = time.localtime()
266
267 print('Beginning test run at', time.strftime("%c %Z",t.start_time))
268
269 sys.stdout.flush()
270 if PYTHON3:
271 # in Python 3, we output text, which cannot be unbuffered
272 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
273 else:
274 # set stdout to unbuffered (is this the best way to do it?)
275 sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0)
276
277 if config.local:
278 tempdir = ''
279 else:
280 # See note [Running tests in /tmp]
281 tempdir = tempfile.mkdtemp('', 'ghctest-')
282
283 # opts.testdir should be quoted when used, to make sure the testsuite
284 # keeps working when it contains backward slashes, for example from
285 # using os.path.join. Windows native and mingw* python
286 # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
287 # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
288 # `os.path.sep = '/'`.
289 # To catch usage of unquoted opts.testdir early, insert some spaces into
290 # tempdir.
291 tempdir = os.path.join(tempdir, 'test spaces')
292
293 def cleanup_and_exit(exitcode):
294 if config.cleanup and tempdir:
295 shutil.rmtree(tempdir, ignore_errors=True)
296 exit(exitcode)
297
298 # First collect all the tests to be run
299 t_files_ok = True
300 for file in t_files:
301 if_verbose(2, '====> Scanning %s' % file)
302 newTestDir(tempdir, os.path.dirname(file))
303 try:
304 if PYTHON3:
305 with io.open(file, encoding='utf8') as f:
306 src = f.read()
307 else:
308 with open(file) as f:
309 src = f.read()
310
311 exec(src)
312 except Exception as e:
313 traceback.print_exc()
314 framework_fail(file, '', str(e))
315 t_files_ok = False
316
317 for name in config.only:
318 if t_files_ok:
319 # See Note [Mutating config.only]
320 framework_fail(name, '', 'test not found')
321 else:
322 # Let user fix .T file errors before reporting on unfound tests.
323 # The reson the test can not be found is likely because of those
324 # .T file errors.
325 pass
326
327 if config.list_broken:
328 global brokens
329 print('')
330 print('Broken tests:')
331 print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
332 print('')
333
334 if t.framework_failures:
335 print('WARNING:', len(framework_failures), 'framework failures!')
336 print('')
337 else:
338 # completion watcher
339 watcher = Watcher(len(parallelTests))
340
341 # Now run all the tests
342 for oneTest in parallelTests:
343 if stopping():
344 break
345 oneTest(watcher)
346
347 # wait for parallel tests to finish
348 if not stopping():
349 watcher.wait()
350
351 # Run the following tests purely sequential
352 config.use_threads = False
353 for oneTest in aloneTests:
354 if stopping():
355 break
356 oneTest(watcher)
357
358 # flush everything before we continue
359 sys.stdout.flush()
360
361 summary(t, sys.stdout, config.no_print_summary)
362
363 if config.summary_file != '':
364 with open(config.summary_file, 'w') as file:
365 summary(t, file)
366
367 cleanup_and_exit(0)
368
369 # Note [Running tests in /tmp]
370 #
371 # Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
372 # the source directory without copying them to the test directory first.
373 #
374 # As an example, take a run_command test with a Makefile containing
375 # `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
376 # Foo.hi in the source directory. There are 2 problems with this:
377 # * Output files in the source directory won't get cleaned up automatically.
378 # * Two tests might (over)write the same output file.
379 #
380 # Tests that only fail when run concurrently with other tests are the
381 # worst, so we try to catch them early by enabling LOCAL=0 in validate.
382 #
383 # Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
384 # making changes to quite a few tests. The problem is that
385 # `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
386 # `module Main where` does not produce Foo.o, as it would without
387 # -outputdir, but Main.o. See [1].
388 #
389 # Using -outputdir='.' is not foolproof anyway, since it does not change
390 # the destination of the final executable (Foo.exe).
391 #
392 # Another hardening method that could be tried is to `chmod -w` the
393 # source directory.
394 #
395 # By default we set LOCAL=1, because it makes it easier to inspect the
396 # test directory while working on a new test.
397 #
398 # [1]
399 # https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files