Render \t as 8 spaces in caret diagnostics
[ghc.git] / testsuite / driver / testlib.py
1 # coding=utf8
2 #
3 # (c) Simon Marlow 2002
4 #
5
6 from __future__ import print_function
7
8 import io
9 import shutil
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 import sys
20 from math import ceil, trunc
21 import collections
22 import subprocess
23
24 from testglobals import *
25 from testutil import *
26 extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
27
28 if config.use_threads:
29 import threading
30 try:
31 import thread
32 except ImportError: # Python 3
33 import _thread as thread
34
35 global wantToStop
36 wantToStop = False
37
38 global pool_sema
39 if config.use_threads:
40 pool_sema = threading.BoundedSemaphore(value=config.threads)
41
42 def stopNow():
43 global wantToStop
44 wantToStop = True
45 def stopping():
46 return wantToStop
47
48 # Options valid for the current test only (these get reset to
49 # testdir_testopts after each test).
50
51 global testopts_local
52 if config.use_threads:
53 testopts_local = threading.local()
54 else:
55 class TestOpts_Local:
56 pass
57 testopts_local = TestOpts_Local()
58
59 def getTestOpts():
60 return testopts_local.x
61
62 def setLocalTestOpts(opts):
63 global testopts_local
64 testopts_local.x=opts
65
66 def isStatsTest():
67 opts = getTestOpts()
68 return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
69
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 global thisdir_settings
75 thisdir_settings = [thisdir_settings, f]
76
77 # -----------------------------------------------------------------------------
78 # Canned setup functions for common cases. eg. for a test you might say
79 #
80 # test('test001', normal, compile, [''])
81 #
82 # to run it without any options, but change it to
83 #
84 # test('test001', expect_fail, compile, [''])
85 #
86 # to expect failure for this test.
87 #
88 # type TestOpt = (name :: String, opts :: Object) -> IO ()
89
90 def normal( name, opts ):
91 return;
92
93 def skip( name, opts ):
94 opts.skip = 1
95
96 def expect_fail( name, opts ):
97 # The compiler, testdriver, OS or platform is missing a certain
98 # feature, and we don't plan to or can't fix it now or in the
99 # future.
100 opts.expect = 'fail';
101
102 def reqlib( lib ):
103 return lambda name, opts, l=lib: _reqlib (name, opts, l )
104
105 def stage1(name, opts):
106 # See Note [Why is there no stage1 setup function?]
107 framework_fail(name, 'stage1 setup function does not exist',
108 'add your test to testsuite/tests/stage1 instead')
109
110 # Note [Why is there no stage1 setup function?]
111 #
112 # Presumably a stage1 setup function would signal that the stage1
113 # compiler should be used to compile a test.
114 #
115 # Trouble is, the path to the compiler + the `ghc --info` settings for
116 # that compiler are currently passed in from the `make` part of the
117 # testsuite driver.
118 #
119 # Switching compilers in the Python part would be entirely too late, as
120 # all ghc_with_* settings would be wrong. See config/ghc for possible
121 # consequences (for example, config.run_ways would still be
122 # based on the default compiler, quite likely causing ./validate --slow
123 # to fail).
124 #
125 # It would be possible to let the Python part of the testsuite driver
126 # make the call to `ghc --info`, but doing so would require quite some
127 # work. Care has to be taken to not affect the run_command tests for
128 # example, as they also use the `ghc --info` settings:
129 # quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
130 #
131 # If you want a test to run using the stage1 compiler, add it to the
132 # testsuite/tests/stage1 directory. Validate runs the tests in that
133 # directory with `make stage=1`.
134
135 # Cache the results of looking to see if we have a library or not.
136 # This makes quite a difference, especially on Windows.
137 have_lib = {}
138
139 def _reqlib( name, opts, lib ):
140 if lib in have_lib:
141 got_it = have_lib[lib]
142 else:
143 cmd = strip_quotes(config.ghc_pkg)
144 p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
145 stdout=subprocess.PIPE,
146 stderr=subprocess.PIPE)
147 # read from stdout and stderr to avoid blocking due to
148 # buffers filling
149 p.communicate()
150 r = p.wait()
151 got_it = r == 0
152 have_lib[lib] = got_it
153
154 if not got_it:
155 opts.expect = 'missing-lib'
156
157 def req_haddock( name, opts ):
158 if not config.haddock:
159 opts.expect = 'missing-lib'
160
161 def req_profiling( name, opts ):
162 '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
163 if not config.have_profiling:
164 opts.expect = 'fail'
165
166 def req_shared_libs( name, opts ):
167 if not config.have_shared_libs:
168 opts.expect = 'fail'
169
170 def req_interp( name, opts ):
171 if not config.have_interp:
172 opts.expect = 'fail'
173
174 def req_smp( name, opts ):
175 if not config.have_smp:
176 opts.expect = 'fail'
177
178 def ignore_stdout(name, opts):
179 opts.ignore_stdout = True
180
181 def ignore_stderr(name, opts):
182 opts.ignore_stderr = True
183
184 def combined_output( name, opts ):
185 opts.combined_output = True
186
187 # -----
188
189 def expect_fail_for( ways ):
190 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
191
192 def _expect_fail_for( name, opts, ways ):
193 opts.expect_fail_for = ways
194
195 def expect_broken( bug ):
196 # This test is a expected not to work due to the indicated trac bug
197 # number.
198 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
199
200 def _expect_broken( name, opts, bug ):
201 record_broken(name, opts, bug)
202 opts.expect = 'fail';
203
204 def expect_broken_for( bug, ways ):
205 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
206
207 def _expect_broken_for( name, opts, bug, ways ):
208 record_broken(name, opts, bug)
209 opts.expect_fail_for = ways
210
211 def record_broken(name, opts, bug):
212 global brokens
213 me = (bug, opts.testdir, name)
214 if not me in brokens:
215 brokens.append(me)
216
217 def _expect_pass(way):
218 # Helper function. Not intended for use in .T files.
219 opts = getTestOpts()
220 return opts.expect == 'pass' and way not in opts.expect_fail_for
221
222 # -----
223
224 def omit_ways( ways ):
225 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
226
227 def _omit_ways( name, opts, ways ):
228 opts.omit_ways = ways
229
230 # -----
231
232 def only_ways( ways ):
233 return lambda name, opts, w=ways: _only_ways( name, opts, w )
234
235 def _only_ways( name, opts, ways ):
236 opts.only_ways = ways
237
238 # -----
239
240 def extra_ways( ways ):
241 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
242
243 def _extra_ways( name, opts, ways ):
244 opts.extra_ways = ways
245
246 # -----
247
248 def set_stdin( file ):
249 return lambda name, opts, f=file: _set_stdin(name, opts, f);
250
251 def _set_stdin( name, opts, f ):
252 opts.stdin = f
253
254 # -----
255
256 def exit_code( val ):
257 return lambda name, opts, v=val: _exit_code(name, opts, v);
258
259 def _exit_code( name, opts, v ):
260 opts.exit_code = v
261
262 def signal_exit_code( val ):
263 if opsys('solaris2'):
264 return exit_code( val );
265 else:
266 # When application running on Linux receives fatal error
267 # signal, then its exit code is encoded as 128 + signal
268 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
269 # I assume that Mac OS X behaves in the same way at least Mac
270 # OS X builder behavior suggests this.
271 return exit_code( val+128 );
272
273 # -----
274
275 def compile_timeout_multiplier( val ):
276 return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
277
278 def _compile_timeout_multiplier( name, opts, v ):
279 opts.compile_timeout_multiplier = v
280
281 def run_timeout_multiplier( val ):
282 return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)
283
284 def _run_timeout_multiplier( name, opts, v ):
285 opts.run_timeout_multiplier = v
286
287 # -----
288
289 def extra_run_opts( val ):
290 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
291
292 def _extra_run_opts( name, opts, v ):
293 opts.extra_run_opts = v
294
295 # -----
296
297 def extra_hc_opts( val ):
298 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
299
300 def _extra_hc_opts( name, opts, v ):
301 opts.extra_hc_opts = v
302
303 # -----
304
305 def extra_clean( files ):
306 # TODO. Remove all calls to extra_clean.
307 return lambda _name, _opts: None
308
309 def extra_files(files):
310 return lambda name, opts: _extra_files(name, opts, files)
311
312 def _extra_files(name, opts, files):
313 opts.extra_files.extend(files)
314
315 # -----
316
317 def stats_num_field( field, expecteds ):
318 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
319
320 def _stats_num_field( name, opts, field, expecteds ):
321 if field in opts.stats_range_fields:
322 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
323
324 if type(expecteds) is list:
325 for (b, expected, dev) in expecteds:
326 if b:
327 opts.stats_range_fields[field] = (expected, dev)
328 return
329 framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
330
331 else:
332 (expected, dev) = expecteds
333 opts.stats_range_fields[field] = (expected, dev)
334
335 def compiler_stats_num_field( field, expecteds ):
336 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
337
338 def _compiler_stats_num_field( name, opts, field, expecteds ):
339 if field in opts.compiler_stats_range_fields:
340 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
341
342 # Compiler performance numbers change when debugging is on, making the results
343 # useless and confusing. Therefore, skip if debugging is on.
344 if compiler_debugged():
345 skip(name, opts)
346
347 for (b, expected, dev) in expecteds:
348 if b:
349 opts.compiler_stats_range_fields[field] = (expected, dev)
350 return
351
352 framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
353
354 # -----
355
356 def when(b, f):
357 # When list_brokens is on, we want to see all expect_broken calls,
358 # so we always do f
359 if b or config.list_broken:
360 return f
361 else:
362 return normal
363
364 def unless(b, f):
365 return when(not b, f)
366
367 def doing_ghci():
368 return 'ghci' in config.run_ways
369
370 def ghc_dynamic():
371 return config.ghc_dynamic
372
373 def fast():
374 return config.speed == 2
375
376 def platform( plat ):
377 return config.platform == plat
378
379 def opsys( os ):
380 return config.os == os
381
382 def arch( arch ):
383 return config.arch == arch
384
385 def wordsize( ws ):
386 return config.wordsize == str(ws)
387
388 def msys( ):
389 return config.msys
390
391 def cygwin( ):
392 return config.cygwin
393
394 def have_vanilla( ):
395 return config.have_vanilla
396
397 def have_dynamic( ):
398 return config.have_dynamic
399
400 def have_profiling( ):
401 return config.have_profiling
402
403 def in_tree_compiler( ):
404 return config.in_tree_compiler
405
406 def unregisterised( ):
407 return config.unregisterised
408
409 def compiler_profiled( ):
410 return config.compiler_profiled
411
412 def compiler_debugged( ):
413 return config.compiler_debugged
414
415 # ---
416
417 def high_memory_usage(name, opts):
418 opts.alone = True
419
420 # If a test is for a multi-CPU race, then running the test alone
421 # increases the chance that we'll actually see it.
422 def multi_cpu_race(name, opts):
423 opts.alone = True
424
425 # ---
426 def literate( name, opts ):
427 opts.literate = 1;
428
429 def c_src( name, opts ):
430 opts.c_src = 1;
431
432 def objc_src( name, opts ):
433 opts.objc_src = 1;
434
435 def objcpp_src( name, opts ):
436 opts.objcpp_src = 1;
437
438 def cmm_src( name, opts ):
439 opts.cmm_src = 1;
440
441 def outputdir( odir ):
442 return lambda name, opts, d=odir: _outputdir(name, opts, d)
443
444 def _outputdir( name, opts, odir ):
445 opts.outputdir = odir;
446
447 # ----
448
449 def pre_cmd( cmd ):
450 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
451
452 def _pre_cmd( name, opts, cmd ):
453 opts.pre_cmd = cmd
454
455 # ----
456
457 def clean_cmd( cmd ):
458 # TODO. Remove all calls to clean_cmd.
459 return lambda _name, _opts: None
460
461 # ----
462
463 def cmd_prefix( prefix ):
464 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
465
466 def _cmd_prefix( name, opts, prefix ):
467 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
468
469 # ----
470
471 def cmd_wrapper( fun ):
472 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
473
474 def _cmd_wrapper( name, opts, fun ):
475 opts.cmd_wrapper = fun
476
477 # ----
478
479 def compile_cmd_prefix( prefix ):
480 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
481
482 def _compile_cmd_prefix( name, opts, prefix ):
483 opts.compile_cmd_prefix = prefix
484
485 # ----
486
487 def check_stdout( f ):
488 return lambda name, opts, f=f: _check_stdout(name, opts, f)
489
490 def _check_stdout( name, opts, f ):
491 opts.check_stdout = f
492
493 def no_check_hp(name, opts):
494 opts.check_hp = False
495
496 # ----
497
498 def filter_stdout_lines( regex ):
499 """ Filter lines of stdout with the given regular expression """
500 import re
501 def f( name, opts ):
502 _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
503 return f
504
505 def normalise_slashes( name, opts ):
506 _normalise_fun(name, opts, normalise_slashes_)
507
508 def normalise_exe( name, opts ):
509 _normalise_fun(name, opts, normalise_exe_)
510
511 def normalise_fun( *fs ):
512 return lambda name, opts: _normalise_fun(name, opts, fs)
513
514 def _normalise_fun( name, opts, *fs ):
515 opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
516
517 def normalise_errmsg_fun( *fs ):
518 return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
519
520 def _normalise_errmsg_fun( name, opts, *fs ):
521 opts.extra_errmsg_normaliser = join_normalisers(opts.extra_errmsg_normaliser, fs)
522
523 def normalise_whitespace_fun(f):
524 return lambda name, opts: _normalise_whitespace_fun(name, opts, f)
525
526 def _normalise_whitespace_fun(name, opts, f):
527 opts.whitespace_normaliser = f
528
529 def normalise_version_( *pkgs ):
530 def normalise_version__( str ):
531 return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
532 '\\1-<VERSION>', str)
533 return normalise_version__
534
535 def normalise_version( *pkgs ):
536 def normalise_version__( name, opts ):
537 _normalise_fun(name, opts, normalise_version_(*pkgs))
538 _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
539 return normalise_version__
540
541 def normalise_drive_letter(name, opts):
542 # Windows only. Change D:\\ to C:\\.
543 _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))
544
545 def keep_prof_callstacks(name, opts):
546 """Keep profiling callstacks.
547
548 Use together with `only_ways(prof_ways)`.
549 """
550 opts.keep_prof_callstacks = True
551
552 def join_normalisers(*a):
553 """
554 Compose functions, flattening sequences.
555
556 join_normalisers(f1,[f2,f3],f4)
557
558 is the same as
559
560 lambda x: f1(f2(f3(f4(x))))
561 """
562
563 def flatten(l):
564 """
565 Taken from http://stackoverflow.com/a/2158532/946226
566 """
567 for el in l:
568 if (isinstance(el, collections.Iterable)
569 and not isinstance(el, (bytes, str))):
570 for sub in flatten(el):
571 yield sub
572 else:
573 yield el
574
575 a = flatten(a)
576
577 fn = lambda x:x # identity function
578 for f in a:
579 assert callable(f)
580 fn = lambda x,f=f,fn=fn: fn(f(x))
581 return fn
582
583 # ----
584 # Function for composing two opt-fns together
585
586 def executeSetups(fs, name, opts):
587 if type(fs) is list:
588 # If we have a list of setups, then execute each one
589 for f in fs:
590 executeSetups(f, name, opts)
591 else:
592 # fs is a single function, so just apply it
593 fs(name, opts)
594
595 # -----------------------------------------------------------------------------
596 # The current directory of tests
597
598 def newTestDir(tempdir, dir):
599
600 global thisdir_settings
601 # reset the options for this test directory
602 def settings(name, opts, tempdir=tempdir, dir=dir):
603 return _newTestDir(name, opts, tempdir, dir)
604 thisdir_settings = settings
605
606 # Should be equal to entry in toplevel .gitignore.
607 testdir_suffix = '.run'
608
609 def _newTestDir(name, opts, tempdir, dir):
610 opts.srcdir = os.path.join(os.getcwd(), dir)
611 opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
612 opts.compiler_always_flags = config.compiler_always_flags
613
614 # -----------------------------------------------------------------------------
615 # Actually doing tests
616
617 parallelTests = []
618 aloneTests = []
619 allTestNames = set([])
620
621 def runTest(watcher, opts, name, func, args):
622 if config.use_threads:
623 pool_sema.acquire()
624 t = threading.Thread(target=test_common_thread,
625 name=name,
626 args=(watcher, name, opts, func, args))
627 t.daemon = False
628 t.start()
629 else:
630 test_common_work(watcher, name, opts, func, args)
631
632 # name :: String
633 # setup :: [TestOpt] -> IO ()
634 def test(name, setup, func, args):
635 global aloneTests
636 global parallelTests
637 global allTestNames
638 global thisdir_settings
639 if name in allTestNames:
640 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
641 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
642 framework_fail(name, 'bad_name', 'This test has an invalid name')
643
644 if config.run_only_some_tests:
645 if name not in config.only:
646 return
647 else:
648 # Note [Mutating config.only]
649 # config.only is initially the set of tests requested by
650 # the user (via 'make TEST='). We then remove all tests that
651 # we've already seen (in .T files), so that we can later
652 # report on any tests we couldn't find and error out.
653 config.only.remove(name)
654
655 # Make a deep copy of the default_testopts, as we need our own copy
656 # of any dictionaries etc inside it. Otherwise, if one test modifies
657 # them, all tests will see the modified version!
658 myTestOpts = copy.deepcopy(default_testopts)
659
660 executeSetups([thisdir_settings, setup], name, myTestOpts)
661
662 thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
663 if myTestOpts.alone:
664 aloneTests.append(thisTest)
665 else:
666 parallelTests.append(thisTest)
667 allTestNames.add(name)
668
669 if config.use_threads:
670 def test_common_thread(watcher, name, opts, func, args):
671 try:
672 test_common_work(watcher, name, opts, func, args)
673 finally:
674 pool_sema.release()
675
676 def get_package_cache_timestamp():
677 if config.package_conf_cache_file == '':
678 return 0.0
679 else:
680 try:
681 return os.stat(config.package_conf_cache_file).st_mtime
682 except:
683 return 0.0
684
685 do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
686
687 def test_common_work(watcher, name, opts, func, args):
688 try:
689 t.total_tests += 1
690 setLocalTestOpts(opts)
691
692 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
693
694 # All the ways we might run this test
695 if func == compile or func == multimod_compile:
696 all_ways = config.compile_ways
697 elif func == compile_and_run or func == multimod_compile_and_run:
698 all_ways = config.run_ways
699 elif func == ghci_script:
700 if 'ghci' in config.run_ways:
701 all_ways = ['ghci']
702 else:
703 all_ways = []
704 else:
705 all_ways = ['normal']
706
707 # A test itself can request extra ways by setting opts.extra_ways
708 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
709
710 t.total_test_cases += len(all_ways)
711
712 ok_way = lambda way: \
713 not getTestOpts().skip \
714 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
715 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
716 and (not (config.skip_perf_tests and isStatsTest())) \
717 and way not in getTestOpts().omit_ways
718
719 # Which ways we are asked to skip
720 do_ways = list(filter (ok_way,all_ways))
721
722 # Only run all ways in slow mode.
723 # See Note [validate and testsuite speed] in toplevel Makefile.
724 if config.accept:
725 # Only ever run one way
726 do_ways = do_ways[:1]
727 elif config.speed > 0:
728 # However, if we EXPLICITLY asked for a way (with extra_ways)
729 # please test it!
730 explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
731 other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
732 do_ways = other_ways[:1] + explicit_ways
733
734 # Find all files in the source directory that this test
735 # depends on. Do this only once for all ways.
736 # Generously add all filenames that start with the name of
737 # the test to this set, as a convenience to test authors.
738 # They will have to use the `extra_files` setup function to
739 # specify all other files that their test depends on (but
740 # this seems to be necessary for only about 10% of all
741 # tests).
742 files = set(f for f in os.listdir(opts.srcdir)
743 if f.startswith(name) and not f == name and
744 not f.endswith(testdir_suffix) and
745 not os.path.splitext(f)[1] in do_not_copy)
746 for filename in (opts.extra_files + extra_src_files.get(name, [])):
747 if filename.startswith('/'):
748 framework_fail(name, 'whole-test',
749 'no absolute paths in extra_files please: ' + filename)
750
751 elif '*' in filename:
752 # Don't use wildcards in extra_files too much, as
753 # globbing is slow.
754 files.update((os.path.relpath(f, opts.srcdir)
755 for f in glob.iglob(in_srcdir(filename))))
756
757 elif filename:
758 files.add(filename)
759
760 else:
761 framework_fail(name, 'whole-test', 'extra_file is empty string')
762
763 # Run the required tests...
764 for way in do_ways:
765 if stopping():
766 break
767 try:
768 do_test(name, way, func, args, files)
769 except KeyboardInterrupt:
770 stopNow()
771 except Exception as e:
772 framework_fail(name, way, str(e))
773 traceback.print_exc()
774
775 t.n_tests_skipped += len(set(all_ways) - set(do_ways))
776
777 if config.cleanup and do_ways:
778 try:
779 cleanup()
780 except Exception as e:
781 framework_fail(name, 'runTest', 'Unhandled exception during cleanup: ' + str(e))
782
783 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
784
785 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
786 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
787
788 except Exception as e:
789 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
790 finally:
791 watcher.notify()
792
793 def do_test(name, way, func, args, files):
794 opts = getTestOpts()
795
796 full_name = name + '(' + way + ')'
797
798 if_verbose(2, "=====> {0} {1} of {2} {3}".format(
799 full_name, t.total_tests, len(allTestNames),
800 [len(t.unexpected_passes),
801 len(t.unexpected_failures),
802 len(t.framework_failures)]))
803
804 # Clean up prior to the test, so that we can't spuriously conclude
805 # that it passed on the basis of old run outputs.
806 cleanup()
807 os.makedirs(opts.testdir)
808
809 # Link all source files for this test into a new directory in
810 # /tmp, and run the test in that directory. This makes it
811 # possible to run tests in parallel, without modification, that
812 # would otherwise (accidentally) write to the same output file.
813 # It also makes it easier to keep the testsuite clean.
814
815 for extra_file in files:
816 src = in_srcdir(extra_file)
817 dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
818 if os.path.isfile(src):
819 link_or_copy_file(src, dst)
820 elif os.path.isdir(src):
821 os.mkdir(dst)
822 lndir(src, dst)
823 else:
824 if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
825 # When using a ghc built without haddock support, .t
826 # files are rightfully missing. Don't
827 # framework_fail. Test will be skipped later.
828 pass
829 else:
830 framework_fail(name, way,
831 'extra_file does not exist: ' + extra_file)
832
833 if func.__name__ == 'run_command' or opts.pre_cmd:
834 # When running 'MAKE' make sure 'TOP' still points to the
835 # root of the testsuite.
836 src_makefile = in_srcdir('Makefile')
837 dst_makefile = in_testdir('Makefile')
838 if os.path.exists(src_makefile):
839 with io.open(src_makefile, 'r', encoding='utf8') as src:
840 makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
841 with io.open(dst_makefile, 'w', encoding='utf8') as dst:
842 dst.write(makefile)
843
844 if opts.pre_cmd:
845 exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, override_options(opts.pre_cmd)),
846 stderr = subprocess.STDOUT,
847 print_output = config.verbose >= 3)
848
849 if exit_code != 0:
850 framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
851
852 result = func(*[name,way] + args)
853
854 if opts.expect not in ['pass', 'fail', 'missing-lib']:
855 framework_fail(name, way, 'bad expected ' + opts.expect)
856
857 try:
858 passFail = result['passFail']
859 except (KeyError, TypeError):
860 passFail = 'No passFail found'
861
862 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
863
864 if passFail == 'pass':
865 if _expect_pass(way):
866 t.n_expected_passes += 1
867 else:
868 if_verbose(1, '*** unexpected pass for %s' % full_name)
869 t.unexpected_passes.append((directory, name, 'unexpected', way))
870 elif passFail == 'fail':
871 if _expect_pass(way):
872 reason = result['reason']
873 tag = result.get('tag')
874 if tag == 'stat':
875 if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
876 t.unexpected_stat_failures.append((directory, name, reason, way))
877 else:
878 if_verbose(1, '*** unexpected failure for %s' % full_name)
879 t.unexpected_failures.append((directory, name, reason, way))
880 else:
881 if opts.expect == 'missing-lib':
882 t.missing_libs.append((directory, name, 'missing-lib', way))
883 else:
884 t.n_expected_failures += 1
885 else:
886 framework_fail(name, way, 'bad result ' + passFail)
887
888 # Make is often invoked with -s, which means if it fails, we get
889 # no feedback at all. This is annoying. So let's remove the option
890 # if found and instead have the testsuite decide on what to do
891 # with the output.
892 def override_options(pre_cmd):
893 if config.verbose >= 5 and bool(re.match('\$make', pre_cmd, re.I)):
894 return pre_cmd.replace('-s' , '') \
895 .replace('--silent', '') \
896 .replace('--quiet' , '')
897
898 return pre_cmd
899
900 def framework_fail(name, way, reason):
901 opts = getTestOpts()
902 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
903 full_name = name + '(' + way + ')'
904 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
905 t.framework_failures.append((directory, name, way, reason))
906
907 def framework_warn(name, way, reason):
908 opts = getTestOpts()
909 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
910 full_name = name + '(' + way + ')'
911 if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
912 t.framework_warnings.append((directory, name, way, reason))
913
914 def badResult(result):
915 try:
916 if result['passFail'] == 'pass':
917 return False
918 return True
919 except (KeyError, TypeError):
920 return True
921
922 def passed():
923 return {'passFail': 'pass'}
924
925 def failBecause(reason, tag=None):
926 return {'passFail': 'fail', 'reason': reason, 'tag': tag}
927
928 # -----------------------------------------------------------------------------
929 # Generic command tests
930
931 # A generic command test is expected to run and exit successfully.
932 #
933 # The expected exit code can be changed via exit_code() as normal, and
934 # the expected stdout/stderr are stored in <testname>.stdout and
935 # <testname>.stderr. The output of the command can be ignored
936 # altogether by using the setup function ignore_stdout instead of
937 # run_command.
938
939 def run_command( name, way, cmd ):
940 return simple_run( name, '', override_options(cmd), '' )
941
942 # -----------------------------------------------------------------------------
943 # GHCi tests
944
945 def ghci_script( name, way, script):
946 flags = ' '.join(get_compiler_flags())
947 way_flags = ' '.join(config.way_flags[way])
948
949 # We pass HC and HC_OPTS as environment variables, so that the
950 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
951 cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
952 ).format(flags=flags, way_flags=way_flags)
953
954 getTestOpts().stdin = script
955 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
956
957 # -----------------------------------------------------------------------------
958 # Compile-only tests
959
960 def compile( name, way, extra_hc_opts ):
961 return do_compile( name, way, 0, '', [], extra_hc_opts )
962
963 def compile_fail( name, way, extra_hc_opts ):
964 return do_compile( name, way, 1, '', [], extra_hc_opts )
965
966 def backpack_typecheck( name, way, extra_hc_opts ):
967 return do_compile( name, way, 0, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
968
969 def backpack_typecheck_fail( name, way, extra_hc_opts ):
970 return do_compile( name, way, 1, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
971
972 def backpack_compile( name, way, extra_hc_opts ):
973 return do_compile( name, way, 0, '', [], extra_hc_opts, backpack=1 )
974
975 def backpack_compile_fail( name, way, extra_hc_opts ):
976 return do_compile( name, way, 1, '', [], extra_hc_opts, backpack=1 )
977
978 def backpack_run( name, way, extra_hc_opts ):
979 return compile_and_run__( name, way, '', [], extra_hc_opts, backpack=1 )
980
981 def multimod_compile( name, way, top_mod, extra_hc_opts ):
982 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
983
984 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
985 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
986
987 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
988 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
989
990 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
991 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
992
993 def do_compile(name, way, should_fail, top_mod, extra_mods, extra_hc_opts, **kwargs):
994 # print 'Compile only, extra args = ', extra_hc_opts
995
996 result = extras_build( way, extra_mods, extra_hc_opts )
997 if badResult(result):
998 return result
999 extra_hc_opts = result['hc_opts']
1000
1001 result = simple_build(name, way, extra_hc_opts, should_fail, top_mod, 0, 1, **kwargs)
1002
1003 if badResult(result):
1004 return result
1005
1006 # the actual stderr should always match the expected, regardless
1007 # of whether we expected the compilation to fail or not (successful
1008 # compilations may generate warnings).
1009
1010 expected_stderr_file = find_expected_file(name, 'stderr')
1011 actual_stderr_file = add_suffix(name, 'comp.stderr')
1012
1013 if not compare_outputs(way, 'stderr',
1014 join_normalisers(getTestOpts().extra_errmsg_normaliser,
1015 normalise_errmsg),
1016 expected_stderr_file, actual_stderr_file,
1017 whitespace_normaliser=getattr(getTestOpts(),
1018 "whitespace_normaliser",
1019 normalise_whitespace)):
1020 return failBecause('stderr mismatch')
1021
1022 # no problems found, this test passed
1023 return passed()
1024
1025 def compile_cmp_asm( name, way, extra_hc_opts ):
1026 print('Compile only, extra args = ', extra_hc_opts)
1027 result = simple_build(name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0)
1028
1029 if badResult(result):
1030 return result
1031
1032 # the actual stderr should always match the expected, regardless
1033 # of whether we expected the compilation to fail or not (successful
1034 # compilations may generate warnings).
1035
1036 expected_asm_file = find_expected_file(name, 'asm')
1037 actual_asm_file = add_suffix(name, 's')
1038
1039 if not compare_outputs(way, 'asm',
1040 join_normalisers(normalise_errmsg, normalise_asm),
1041 expected_asm_file, actual_asm_file):
1042 return failBecause('asm mismatch')
1043
1044 # no problems found, this test passed
1045 return passed()
1046
1047 # -----------------------------------------------------------------------------
1048 # Compile-and-run tests
1049
1050 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts, backpack=0 ):
1051 # print 'Compile and run, extra args = ', extra_hc_opts
1052
1053 result = extras_build( way, extra_mods, extra_hc_opts )
1054 if badResult(result):
1055 return result
1056 extra_hc_opts = result['hc_opts']
1057
1058 if way.startswith('ghci'): # interpreted...
1059 return interpreter_run(name, way, extra_hc_opts, top_mod)
1060 else: # compiled...
1061 result = simple_build(name, way, extra_hc_opts, 0, top_mod, 1, 1, backpack = backpack)
1062 if badResult(result):
1063 return result
1064
1065 cmd = './' + name;
1066
1067 # we don't check the compiler's stderr for a compile-and-run test
1068 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1069
1070 def compile_and_run( name, way, extra_hc_opts ):
1071 return compile_and_run__( name, way, '', [], extra_hc_opts)
1072
1073 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1074 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1075
1076 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1077 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1078
1079 def stats( name, way, stats_file ):
1080 opts = getTestOpts()
1081 return checkStats(name, way, stats_file, opts.stats_range_fields)
1082
1083 # -----------------------------------------------------------------------------
1084 # Check -t stats info
1085
1086 def checkStats(name, way, stats_file, range_fields):
1087 full_name = name + '(' + way + ')'
1088
1089 result = passed()
1090 if range_fields:
1091 try:
1092 f = open(in_testdir(stats_file))
1093 except IOError as e:
1094 return failBecause(str(e))
1095 contents = f.read()
1096 f.close()
1097
1098 for (field, (expected, dev)) in range_fields.items():
1099 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1100 if m == None:
1101 print('Failed to find field: ', field)
1102 result = failBecause('no such stats field')
1103 val = int(m.group(1))
1104
1105 lowerBound = trunc( expected * ((100 - float(dev))/100))
1106 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1107
1108 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1109
1110 if val < lowerBound:
1111 print(field, 'value is too low:')
1112 print('(If this is because you have improved GHC, please')
1113 print('update the test so that GHC doesn\'t regress again)')
1114 result = failBecause('stat too good', tag='stat')
1115 if val > upperBound:
1116 print(field, 'value is too high:')
1117 result = failBecause('stat not good enough', tag='stat')
1118
1119 if val < lowerBound or val > upperBound or config.verbose >= 4:
1120 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1121
1122 def display(descr, val, extra):
1123 print(descr, str(val).rjust(length), extra)
1124
1125 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1126 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1127 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1128 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1129 if val != expected:
1130 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1131
1132 return result
1133
1134 # -----------------------------------------------------------------------------
1135 # Build a single-module program
1136
1137 def extras_build( way, extra_mods, extra_hc_opts ):
1138 for mod, opts in extra_mods:
1139 result = simple_build(mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0)
1140 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1141 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1142 if badResult(result):
1143 return result
1144
1145 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1146
1147 def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, backpack = False):
1148 opts = getTestOpts()
1149
1150 # Redirect stdout and stderr to the same file
1151 stdout = in_testdir(name, 'comp.stderr')
1152 stderr = subprocess.STDOUT
1153
1154 if top_mod != '':
1155 srcname = top_mod
1156 elif addsuf:
1157 if backpack:
1158 srcname = add_suffix(name, 'bkp')
1159 else:
1160 srcname = add_hs_lhs_suffix(name)
1161 else:
1162 srcname = name
1163
1164 if top_mod != '':
1165 to_do = '--make '
1166 if link:
1167 to_do = to_do + '-o ' + name
1168 elif backpack:
1169 if link:
1170 to_do = '-o ' + name + ' '
1171 else:
1172 to_do = ''
1173 to_do = to_do + '--backpack '
1174 elif link:
1175 to_do = '-o ' + name
1176 else:
1177 to_do = '-c' # just compile
1178
1179 stats_file = name + '.comp.stats'
1180 if opts.compiler_stats_range_fields:
1181 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1182 if backpack:
1183 extra_hc_opts += ' -outputdir ' + name + '.out'
1184
1185 # Required by GHC 7.3+, harmless for earlier versions:
1186 if (getTestOpts().c_src or
1187 getTestOpts().objc_src or
1188 getTestOpts().objcpp_src or
1189 getTestOpts().cmm_src):
1190 extra_hc_opts += ' -no-hs-main '
1191
1192 if getTestOpts().compile_cmd_prefix == '':
1193 cmd_prefix = ''
1194 else:
1195 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1196
1197 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1198
1199 cmd = ('cd "{opts.testdir}" && {cmd_prefix} '
1200 '{{compiler}} {to_do} {srcname} {flags} {extra_hc_opts}'
1201 ).format(**locals())
1202
1203 exit_code = runCmd(cmd, None, stdout, stderr, opts.compile_timeout_multiplier)
1204
1205 if exit_code != 0 and not should_fail:
1206 if config.verbose >= 1 and _expect_pass(way):
1207 print('Compile failed (exit code {0}) errors were:'.format(exit_code))
1208 actual_stderr_path = in_testdir(name, 'comp.stderr')
1209 if_verbose_dump(1, actual_stderr_path)
1210
1211 # ToDo: if the sub-shell was killed by ^C, then exit
1212
1213 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1214
1215 if badResult(statsResult):
1216 return statsResult
1217
1218 if should_fail:
1219 if exit_code == 0:
1220 return failBecause('exit code 0')
1221 else:
1222 if exit_code != 0:
1223 return failBecause('exit code non-0')
1224
1225 return passed()
1226
1227 # -----------------------------------------------------------------------------
1228 # Run a program and check its output
1229 #
1230 # If testname.stdin exists, route input from that, else
1231 # from /dev/null. Route output to testname.run.stdout and
1232 # testname.run.stderr. Returns the exit code of the run.
1233
1234 def simple_run(name, way, prog, extra_run_opts):
1235 opts = getTestOpts()
1236
1237 # figure out what to use for stdin
1238 if opts.stdin:
1239 stdin = in_testdir(opts.stdin)
1240 elif os.path.exists(in_testdir(name, 'stdin')):
1241 stdin = in_testdir(name, 'stdin')
1242 else:
1243 stdin = None
1244
1245 stdout = in_testdir(name, 'run.stdout')
1246 if opts.combined_output:
1247 stderr = subprocess.STDOUT
1248 else:
1249 stderr = in_testdir(name, 'run.stderr')
1250
1251 my_rts_flags = rts_flags(way)
1252
1253 stats_file = name + '.stats'
1254 if opts.stats_range_fields:
1255 stats_args = ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1256 else:
1257 stats_args = ''
1258
1259 # Put extra_run_opts last: extra_run_opts('+RTS foo') should work.
1260 cmd = prog + stats_args + ' ' + my_rts_flags + ' ' + extra_run_opts
1261
1262 if opts.cmd_wrapper != None:
1263 cmd = opts.cmd_wrapper(cmd)
1264
1265 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1266
1267 # run the command
1268 exit_code = runCmd(cmd, stdin, stdout, stderr, opts.run_timeout_multiplier)
1269
1270 # check the exit code
1271 if exit_code != opts.exit_code:
1272 if config.verbose >= 1 and _expect_pass(way):
1273 print('Wrong exit code for ' + name + '(' + way + ')' + '(expected', opts.exit_code, ', actual', exit_code, ')')
1274 dump_stdout(name)
1275 dump_stderr(name)
1276 return failBecause('bad exit code')
1277
1278 if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
1279 return failBecause('bad stderr')
1280 if not (opts.ignore_stdout or stdout_ok(name, way)):
1281 return failBecause('bad stdout')
1282
1283 check_hp = '-h' in my_rts_flags and opts.check_hp
1284 check_prof = '-p' in my_rts_flags
1285
1286 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1287 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1288 return failBecause('bad heap profile')
1289 if check_prof and not check_prof_ok(name, way):
1290 return failBecause('bad profile')
1291
1292 return checkStats(name, way, stats_file, opts.stats_range_fields)
1293
1294 def rts_flags(way):
1295 args = config.way_rts_flags.get(way, [])
1296 return '+RTS {0} -RTS'.format(' '.join(args)) if args else ''
1297
1298 # -----------------------------------------------------------------------------
1299 # Run a program in the interpreter and check its output
1300
1301 def interpreter_run(name, way, extra_hc_opts, top_mod):
1302 opts = getTestOpts()
1303
1304 stdout = in_testdir(name, 'interp.stdout')
1305 stderr = in_testdir(name, 'interp.stderr')
1306 script = in_testdir(name, 'genscript')
1307
1308 if opts.combined_output:
1309 framework_fail(name, 'unsupported',
1310 'WAY=ghci and combined_output together is not supported')
1311
1312 if (top_mod == ''):
1313 srcname = add_hs_lhs_suffix(name)
1314 else:
1315 srcname = top_mod
1316
1317 delimiter = '===== program output begins here\n'
1318
1319 with io.open(script, 'w', encoding='utf8') as f:
1320 # set the prog name and command-line args to match the compiled
1321 # environment.
1322 f.write(':set prog ' + name + '\n')
1323 f.write(':set args ' + opts.extra_run_opts + '\n')
1324 # Add marker lines to the stdout and stderr output files, so we
1325 # can separate GHCi's output from the program's.
1326 f.write(':! echo ' + delimiter)
1327 f.write(':! echo 1>&2 ' + delimiter)
1328 # Set stdout to be line-buffered to match the compiled environment.
1329 f.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1330 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1331 # in the event of an exception as for the compiled program.
1332 f.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1333
1334 stdin = in_testdir(opts.stdin if opts.stdin else add_suffix(name, 'stdin'))
1335 if os.path.exists(stdin):
1336 os.system('cat "{0}" >> "{1}"'.format(stdin, script))
1337
1338 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1339
1340 cmd = ('{{compiler}} {srcname} {flags} {extra_hc_opts}'
1341 ).format(**locals())
1342
1343 if getTestOpts().cmd_wrapper != None:
1344 cmd = opts.cmd_wrapper(cmd);
1345
1346 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1347
1348 exit_code = runCmd(cmd, script, stdout, stderr, opts.run_timeout_multiplier)
1349
1350 # split the stdout into compilation/program output
1351 split_file(stdout, delimiter,
1352 in_testdir(name, 'comp.stdout'),
1353 in_testdir(name, 'run.stdout'))
1354 split_file(stderr, delimiter,
1355 in_testdir(name, 'comp.stderr'),
1356 in_testdir(name, 'run.stderr'))
1357
1358 # check the exit code
1359 if exit_code != getTestOpts().exit_code:
1360 print('Wrong exit code for ' + name + '(' + way + ') (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1361 dump_stdout(name)
1362 dump_stderr(name)
1363 return failBecause('bad exit code')
1364
1365 # ToDo: if the sub-shell was killed by ^C, then exit
1366
1367 if not (opts.ignore_stderr or stderr_ok(name, way)):
1368 return failBecause('bad stderr')
1369 elif not (opts.ignore_stdout or stdout_ok(name, way)):
1370 return failBecause('bad stdout')
1371 else:
1372 return passed()
1373
1374 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1375 # See Note [Universal newlines].
1376 with io.open(in_fn, 'r', encoding='utf8', errors='replace', newline=None) as infile:
1377 with io.open(out1_fn, 'w', encoding='utf8', newline='') as out1:
1378 with io.open(out2_fn, 'w', encoding='utf8', newline='') as out2:
1379 line = infile.readline()
1380 while re.sub('^\s*','',line) != delimiter and line != '':
1381 out1.write(line)
1382 line = infile.readline()
1383
1384 line = infile.readline()
1385 while line != '':
1386 out2.write(line)
1387 line = infile.readline()
1388
1389 # -----------------------------------------------------------------------------
1390 # Utils
1391 def get_compiler_flags():
1392 opts = getTestOpts()
1393
1394 flags = copy.copy(opts.compiler_always_flags)
1395
1396 flags.append(opts.extra_hc_opts)
1397
1398 if opts.outputdir != None:
1399 flags.extend(["-outputdir", opts.outputdir])
1400
1401 return flags
1402
1403 def stdout_ok(name, way):
1404 actual_stdout_file = add_suffix(name, 'run.stdout')
1405 expected_stdout_file = find_expected_file(name, 'stdout')
1406
1407 extra_norm = join_normalisers(normalise_output, getTestOpts().extra_normaliser)
1408
1409 check_stdout = getTestOpts().check_stdout
1410 if check_stdout:
1411 actual_stdout_path = in_testdir(actual_stdout_file)
1412 return check_stdout(actual_stdout_path, extra_norm)
1413
1414 return compare_outputs(way, 'stdout', extra_norm,
1415 expected_stdout_file, actual_stdout_file)
1416
1417 def dump_stdout( name ):
1418 with open(in_testdir(name, 'run.stdout')) as f:
1419 str = f.read().strip()
1420 if str:
1421 print("Stdout (", name, "):")
1422 print(str)
1423
1424 def stderr_ok(name, way):
1425 actual_stderr_file = add_suffix(name, 'run.stderr')
1426 expected_stderr_file = find_expected_file(name, 'stderr')
1427
1428 return compare_outputs(way, 'stderr',
1429 join_normalisers(normalise_errmsg, getTestOpts().extra_errmsg_normaliser), \
1430 expected_stderr_file, actual_stderr_file,
1431 whitespace_normaliser=normalise_whitespace)
1432
1433 def dump_stderr( name ):
1434 with open(in_testdir(name, 'run.stderr')) as f:
1435 str = f.read().strip()
1436 if str:
1437 print("Stderr (", name, "):")
1438 print(str)
1439
1440 def read_no_crs(file):
1441 str = ''
1442 try:
1443 # See Note [Universal newlines].
1444 with io.open(file, 'r', encoding='utf8', errors='replace', newline=None) as h:
1445 str = h.read()
1446 except Exception:
1447 # On Windows, if the program fails very early, it seems the
1448 # files stdout/stderr are redirected to may not get created
1449 pass
1450 return str
1451
1452 def write_file(file, str):
1453 # See Note [Universal newlines].
1454 with io.open(file, 'w', encoding='utf8', newline='') as h:
1455 h.write(str)
1456
1457 # Note [Universal newlines]
1458 #
1459 # We don't want to write any Windows style line endings ever, because
1460 # it would mean that `make accept` would touch every line of the file
1461 # when switching between Linux and Windows.
1462 #
1463 # Furthermore, when reading a file, it is convenient to translate all
1464 # Windows style endings to '\n', as it simplifies searching or massaging
1465 # the content.
1466 #
1467 # Solution: use `io.open` instead of `open`
1468 # * when reading: use newline=None to translate '\r\n' to '\n'
1469 # * when writing: use newline='' to not translate '\n' to '\r\n'
1470 #
1471 # See https://docs.python.org/2/library/io.html#io.open.
1472 #
1473 # This should work with both python2 and python3, and with both mingw*
1474 # as msys2 style Python.
1475 #
1476 # Do note that io.open returns unicode strings. So we have to specify
1477 # the expected encoding. But there is at least one file which is not
1478 # valid utf8 (decodingerror002.stdout). Solution: use errors='replace'.
1479 # Another solution would be to open files in binary mode always, and
1480 # operate on bytes.
1481
1482 def check_hp_ok(name):
1483 opts = getTestOpts()
1484
1485 # do not qualify for hp2ps because we should be in the right directory
1486 hp2psCmd = 'cd "{opts.testdir}" && {{hp2ps}} {name}'.format(**locals())
1487
1488 hp2psResult = runCmd(hp2psCmd)
1489
1490 actual_ps_path = in_testdir(name, 'ps')
1491
1492 if hp2psResult == 0:
1493 if os.path.exists(actual_ps_path):
1494 if gs_working:
1495 gsResult = runCmd(genGSCmd(actual_ps_path))
1496 if (gsResult == 0):
1497 return (True)
1498 else:
1499 print("hp2ps output for " + name + "is not valid PostScript")
1500 else: return (True) # assume postscript is valid without ghostscript
1501 else:
1502 print("hp2ps did not generate PostScript for " + name)
1503 return (False)
1504 else:
1505 print("hp2ps error when processing heap profile for " + name)
1506 return(False)
1507
1508 def check_prof_ok(name, way):
1509 expected_prof_file = find_expected_file(name, 'prof.sample')
1510 expected_prof_path = in_testdir(expected_prof_file)
1511
1512 # Check actual prof file only if we have an expected prof file to
1513 # compare it with.
1514 if not os.path.exists(expected_prof_path):
1515 return True
1516
1517 actual_prof_file = add_suffix(name, 'prof')
1518 actual_prof_path = in_testdir(actual_prof_file)
1519
1520 if not os.path.exists(actual_prof_path):
1521 print(actual_prof_path + " does not exist")
1522 return(False)
1523
1524 if os.path.getsize(actual_prof_path) == 0:
1525 print(actual_prof_path + " is empty")
1526 return(False)
1527
1528 return compare_outputs(way, 'prof', normalise_prof,
1529 expected_prof_file, actual_prof_file,
1530 whitespace_normaliser=normalise_whitespace)
1531
1532 # Compare expected output to actual output, and optionally accept the
1533 # new output. Returns true if output matched or was accepted, false
1534 # otherwise. See Note [Output comparison] for the meaning of the
1535 # normaliser and whitespace_normaliser parameters.
1536 def compare_outputs(way, kind, normaliser, expected_file, actual_file,
1537 whitespace_normaliser=lambda x:x):
1538
1539 expected_path = in_srcdir(expected_file)
1540 actual_path = in_testdir(actual_file)
1541
1542 if os.path.exists(expected_path):
1543 expected_str = normaliser(read_no_crs(expected_path))
1544 # Create the .normalised file in the testdir, not in the srcdir.
1545 expected_normalised_file = add_suffix(expected_file, 'normalised')
1546 expected_normalised_path = in_testdir(expected_normalised_file)
1547 else:
1548 expected_str = ''
1549 expected_normalised_path = '/dev/null'
1550
1551 actual_raw = read_no_crs(actual_path)
1552 actual_str = normaliser(actual_raw)
1553
1554 # See Note [Output comparison].
1555 if whitespace_normaliser(expected_str) == whitespace_normaliser(actual_str):
1556 return 1
1557 else:
1558 if config.verbose >= 1 and _expect_pass(way):
1559 print('Actual ' + kind + ' output differs from expected:')
1560
1561 if expected_normalised_path != '/dev/null':
1562 write_file(expected_normalised_path, expected_str)
1563
1564 actual_normalised_path = add_suffix(actual_path, 'normalised')
1565 write_file(actual_normalised_path, actual_str)
1566
1567 if config.verbose >= 1 and _expect_pass(way):
1568 # See Note [Output comparison].
1569 r = runCmd('diff -uw "{0}" "{1}"'.format(expected_normalised_path,
1570 actual_normalised_path),
1571 print_output = 1)
1572
1573 # If for some reason there were no non-whitespace differences,
1574 # then do a full diff
1575 if r == 0:
1576 r = runCmd('diff -u "{0}" "{1}"'.format(expected_normalised_path,
1577 actual_normalised_path),
1578 print_output = 1)
1579
1580 if config.accept and (getTestOpts().expect == 'fail' or
1581 way in getTestOpts().expect_fail_for):
1582 if_verbose(1, 'Test is expected to fail. Not accepting new output.')
1583 return 0
1584 elif config.accept and actual_raw:
1585 if_verbose(1, 'Accepting new output.')
1586 write_file(expected_path, actual_raw)
1587 return 1
1588 elif config.accept:
1589 if_verbose(1, 'No output. Deleting "{0}".'.format(expected_path))
1590 os.remove(expected_path)
1591 return 1
1592 else:
1593 return 0
1594
1595 # Note [Output comparison]
1596 #
1597 # We do two types of output comparison:
1598 #
1599 # 1. To decide whether a test has failed. We apply a `normaliser` and an
1600 # optional `whitespace_normaliser` to the expected and the actual
1601 # output, before comparing the two.
1602 #
1603 # 2. To show as a diff to the user when the test indeed failed. We apply
1604 # the same `normaliser` function to the outputs, to make the diff as
1605 # small as possible (only showing the actual problem). But we don't
1606 # apply the `whitespace_normaliser` here, because it might completely
1607 # squash all whitespace, making the diff unreadable. Instead we rely
1608 # on the `diff` program to ignore whitespace changes as much as
1609 # possible (#10152).
1610
1611 def normalise_whitespace( str ):
1612 # Merge contiguous whitespace characters into a single space.
1613 return ' '.join(w for w in str.split())
1614
1615 callSite_re = re.compile(r', called at (.+):[\d]+:[\d]+ in [\w\-\.]+:')
1616
1617 def normalise_callstacks(s):
1618 opts = getTestOpts()
1619 def repl(matches):
1620 location = matches.group(1)
1621 location = normalise_slashes_(location)
1622 return ', called at {0}:<line>:<column> in <package-id>:'.format(location)
1623 # Ignore line number differences in call stacks (#10834).
1624 s = re.sub(callSite_re, repl, s)
1625 # Ignore the change in how we identify implicit call-stacks
1626 s = s.replace('from ImplicitParams', 'from HasCallStack')
1627 if not opts.keep_prof_callstacks:
1628 # Don't output prof callstacks. Test output should be
1629 # independent from the WAY we run the test.
1630 s = re.sub(r'CallStack \(from -prof\):(\n .*)*\n?', '', s)
1631 return s
1632
1633 tyCon_re = re.compile(r'TyCon\s*\d+L?\#\#\s*\d+L?\#\#\s*', flags=re.MULTILINE)
1634
1635 def normalise_type_reps(str):
1636 """ Normalise out fingerprints from Typeable TyCon representations """
1637 return re.sub(tyCon_re, 'TyCon FINGERPRINT FINGERPRINT ', str)
1638
1639 def normalise_errmsg( str ):
1640 """Normalise error-messages emitted via stderr"""
1641 # IBM AIX's `ld` is a bit chatty
1642 if opsys('aix'):
1643 str = str.replace('ld: 0706-027 The -x flag is ignored.\n', '')
1644 # remove " error:" and lower-case " Warning:" to make patch for
1645 # trac issue #10021 smaller
1646 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1647 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1648 str = normalise_callstacks(str)
1649 str = normalise_type_reps(str)
1650
1651 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1652 # the colon is there because it appears in error messages; this
1653 # hacky solution is used in place of more sophisticated filename
1654 # mangling
1655 str = re.sub('([^\\s])\\.exe', '\\1', str)
1656
1657 # normalise slashes, minimise Windows/Unix filename differences
1658 str = re.sub('\\\\', '/', str)
1659
1660 # The inplace ghc's are called ghc-stage[123] to avoid filename
1661 # collisions, so we need to normalise that to just "ghc"
1662 str = re.sub('ghc-stage[123]', 'ghc', str)
1663
1664 # Error messages simetimes contain integer implementation package
1665 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1666
1667 # Also filter out bullet characters. This is because bullets are used to
1668 # separate error sections, and tests shouldn't be sensitive to how the
1669 # the division happens.
1670 bullet = '•'.encode('utf8') if isinstance(str, bytes) else '•'
1671 str = str.replace(bullet, '')
1672
1673 # Windows only, this is a bug in hsc2hs but it is preventing
1674 # stable output for the testsuite. See Trac #9775. For now we filter out this
1675 # warning message to get clean output.
1676 if config.msys:
1677 str = re.sub('Failed to remove file (.*); error= (.*)$', '', str)
1678 str = re.sub('DeleteFile "(.+)": permission denied \(Access is denied\.\)(.*)$', '', str)
1679
1680 return str
1681
1682 # normalise a .prof file, so that we can reasonably compare it against
1683 # a sample. This doesn't compare any of the actual profiling data,
1684 # only the shape of the profile and the number of entries.
1685 def normalise_prof (str):
1686 # strip everything up to the line beginning "COST CENTRE"
1687 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1688
1689 # strip results for CAFs, these tend to change unpredictably
1690 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1691
1692 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1693 # sometimes under MAIN.
1694 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1695
1696 # We have something like this:
1697 #
1698 # MAIN MAIN <built-in> 53 0 0.0 0.2 0.0 100.0
1699 # CAF Main <entire-module> 105 0 0.0 0.3 0.0 62.5
1700 # readPrec Main Main_1.hs:7:13-16 109 1 0.0 0.6 0.0 0.6
1701 # readPrec Main Main_1.hs:4:13-16 107 1 0.0 0.6 0.0 0.6
1702 # main Main Main_1.hs:(10,1)-(20,20) 106 1 0.0 20.2 0.0 61.0
1703 # == Main Main_1.hs:7:25-26 114 1 0.0 0.0 0.0 0.0
1704 # == Main Main_1.hs:4:25-26 113 1 0.0 0.0 0.0 0.0
1705 # showsPrec Main Main_1.hs:7:19-22 112 2 0.0 1.2 0.0 1.2
1706 # showsPrec Main Main_1.hs:4:19-22 111 2 0.0 0.9 0.0 0.9
1707 # readPrec Main Main_1.hs:7:13-16 110 0 0.0 18.8 0.0 18.8
1708 # readPrec Main Main_1.hs:4:13-16 108 0 0.0 19.9 0.0 19.9
1709 #
1710 # then we remove all the specific profiling data, leaving only the cost
1711 # centre name, module, src, and entries, to end up with this: (modulo
1712 # whitespace between columns)
1713 #
1714 # MAIN MAIN <built-in> 0
1715 # readPrec Main Main_1.hs:7:13-16 1
1716 # readPrec Main Main_1.hs:4:13-16 1
1717 # == Main Main_1.hs:7:25-26 1
1718 # == Main Main_1.hs:4:25-26 1
1719 # showsPrec Main Main_1.hs:7:19-22 2
1720 # showsPrec Main Main_1.hs:4:19-22 2
1721 # readPrec Main Main_1.hs:7:13-16 0
1722 # readPrec Main Main_1.hs:4:13-16 0
1723
1724 # Split 9 whitespace-separated groups, take columns 1 (cost-centre), 2
1725 # (module), 3 (src), and 5 (entries). SCC names can't have whitespace, so
1726 # this works fine.
1727 str = re.sub(r'\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*',
1728 '\\1 \\2 \\3 \\5\n', str)
1729 return str
1730
1731 def normalise_slashes_( str ):
1732 str = re.sub('\\\\', '/', str)
1733 return str
1734
1735 def normalise_exe_( str ):
1736 str = re.sub('\.exe', '', str)
1737 return str
1738
1739 def normalise_output( str ):
1740 # remove " error:" and lower-case " Warning:" to make patch for
1741 # trac issue #10021 smaller
1742 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1743 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1744 # Remove a .exe extension (for Windows)
1745 # This can occur in error messages generated by the program.
1746 str = re.sub('([^\\s])\\.exe', '\\1', str)
1747 str = normalise_callstacks(str)
1748 str = normalise_type_reps(str)
1749 return str
1750
1751 def normalise_asm( str ):
1752 lines = str.split('\n')
1753 # Only keep instructions and labels not starting with a dot.
1754 metadata = re.compile('^[ \t]*\\..*$')
1755 out = []
1756 for line in lines:
1757 # Drop metadata directives (e.g. ".type")
1758 if not metadata.match(line):
1759 line = re.sub('@plt', '', line)
1760 instr = line.lstrip().split()
1761 # Drop empty lines.
1762 if not instr:
1763 continue
1764 # Drop operands, except for call instructions.
1765 elif instr[0] == 'call':
1766 out.append(instr[0] + ' ' + instr[1])
1767 else:
1768 out.append(instr[0])
1769 out = '\n'.join(out)
1770 return out
1771
1772 def if_verbose( n, s ):
1773 if config.verbose >= n:
1774 print(s)
1775
1776 def if_verbose_dump( n, f ):
1777 if config.verbose >= n:
1778 try:
1779 with io.open(f) as file:
1780 print(file.read())
1781 except Exception:
1782 print('')
1783
1784 def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, print_output=0):
1785 timeout_prog = strip_quotes(config.timeout_prog)
1786 timeout = str(int(ceil(config.timeout * timeout_multiplier)))
1787
1788 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1789 cmd = cmd.format(**config.__dict__)
1790 if_verbose(3, cmd + ('< ' + os.path.basename(stdin) if stdin else ''))
1791
1792 # declare the buffers to a default
1793 stdin_buffer = None
1794
1795 # ***** IMPORTANT *****
1796 # We have to treat input and output as
1797 # just binary data here. Don't try to decode
1798 # it to a string, since we have tests that actually
1799 # feed malformed utf-8 to see how GHC handles it.
1800 if stdin:
1801 with io.open(stdin, 'rb') as f:
1802 stdin_buffer = f.read()
1803
1804 stdout_buffer = b''
1805 stderr_buffer = b''
1806
1807 hStdErr = subprocess.PIPE
1808 if stderr is subprocess.STDOUT:
1809 hStdErr = subprocess.STDOUT
1810
1811 try:
1812 # cmd is a complex command in Bourne-shell syntax
1813 # e.g (cd . && 'C:/users/simonpj/HEAD/inplace/bin/ghc-stage2' ...etc)
1814 # Hence it must ultimately be run by a Bourne shell. It's timeout's job
1815 # to invoke the Bourne shell
1816
1817 r = subprocess.Popen([timeout_prog, timeout, cmd],
1818 stdin=subprocess.PIPE,
1819 stdout=subprocess.PIPE,
1820 stderr=hStdErr)
1821
1822 stdout_buffer, stderr_buffer = r.communicate(stdin_buffer)
1823 finally:
1824 if config.verbose >= 1 and print_output >= 1:
1825 if stdout_buffer:
1826 sys.stdout.buffer.write(stdout_buffer)
1827 if stderr_buffer:
1828 sys.stderr.buffer.write(stderr_buffer)
1829
1830 if stdout:
1831 with io.open(stdout, 'wb') as f:
1832 f.write(stdout_buffer)
1833 if stderr:
1834 if stderr is not subprocess.STDOUT:
1835 with io.open(stderr, 'wb') as f:
1836 f.write(stderr_buffer)
1837
1838 if r.returncode == 98:
1839 # The python timeout program uses 98 to signal that ^C was pressed
1840 stopNow()
1841 if r.returncode == 99 and getTestOpts().exit_code != 99:
1842 # Only print a message when timeout killed the process unexpectedly.
1843 if_verbose(1, 'Timeout happened...killed process "{0}"...\n'.format(cmd))
1844 return r.returncode
1845
1846 # -----------------------------------------------------------------------------
1847 # checking if ghostscript is available for checking the output of hp2ps
1848
1849 def genGSCmd(psfile):
1850 return '{{gs}} -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE "{0}"'.format(psfile)
1851
1852 def gsNotWorking():
1853 global gs_working
1854 print("GhostScript not available for hp2ps tests")
1855
1856 global gs_working
1857 gs_working = 0
1858 if config.have_profiling:
1859 if config.gs != '':
1860 resultGood = runCmd(genGSCmd(config.confdir + '/good.ps'));
1861 if resultGood == 0:
1862 resultBad = runCmd(genGSCmd(config.confdir + '/bad.ps') +
1863 ' >/dev/null 2>&1')
1864 if resultBad != 0:
1865 print("GhostScript available for hp2ps tests")
1866 gs_working = 1;
1867 else:
1868 gsNotWorking();
1869 else:
1870 gsNotWorking();
1871 else:
1872 gsNotWorking();
1873
1874 def add_suffix( name, suffix ):
1875 if suffix == '':
1876 return name
1877 else:
1878 return name + '.' + suffix
1879
1880 def add_hs_lhs_suffix(name):
1881 if getTestOpts().c_src:
1882 return add_suffix(name, 'c')
1883 elif getTestOpts().cmm_src:
1884 return add_suffix(name, 'cmm')
1885 elif getTestOpts().objc_src:
1886 return add_suffix(name, 'm')
1887 elif getTestOpts().objcpp_src:
1888 return add_suffix(name, 'mm')
1889 elif getTestOpts().literate:
1890 return add_suffix(name, 'lhs')
1891 else:
1892 return add_suffix(name, 'hs')
1893
1894 def replace_suffix( name, suffix ):
1895 base, suf = os.path.splitext(name)
1896 return base + '.' + suffix
1897
1898 def in_testdir(name, suffix=''):
1899 return os.path.join(getTestOpts().testdir, add_suffix(name, suffix))
1900
1901 def in_srcdir(name, suffix=''):
1902 return os.path.join(getTestOpts().srcdir, add_suffix(name, suffix))
1903
1904 # Finding the sample output. The filename is of the form
1905 #
1906 # <test>.stdout[-ws-<wordsize>][-<platform>]
1907 #
1908 def find_expected_file(name, suff):
1909 basename = add_suffix(name, suff)
1910
1911 files = [basename + ws + plat
1912 for plat in ['-' + config.platform, '-' + config.os, '']
1913 for ws in ['-ws-' + config.wordsize, '']]
1914
1915 for f in files:
1916 if os.path.exists(in_srcdir(f)):
1917 return f
1918
1919 return basename
1920
1921 if config.msys:
1922 import stat
1923 import time
1924 def cleanup():
1925 testdir = getTestOpts().testdir
1926 max_attempts = 5
1927 retries = max_attempts
1928 def on_error(function, path, excinfo):
1929 # At least one test (T11489) removes the write bit from a file it
1930 # produces. Windows refuses to delete read-only files with a
1931 # permission error. Try setting the write bit and try again.
1932 os.chmod(path, stat.S_IWRITE)
1933 function(path)
1934
1935 # On Windows we have to retry the delete a couple of times.
1936 # The reason for this is that a FileDelete command just marks a
1937 # file for deletion. The file is really only removed when the last
1938 # handle to the file is closed. Unfortunately there are a lot of
1939 # system services that can have a file temporarily opened using a shared
1940 # readonly lock, such as the built in AV and search indexer.
1941 #
1942 # We can't really guarantee that these are all off, so what we can do is
1943 # whenever after a rmtree the folder still exists to try again and wait a bit.
1944 #
1945 # Based on what I've seen from the tests on CI server, is that this is relatively rare.
1946 # So overall we won't be retrying a lot. If after a reasonable amount of time the folder is
1947 # still locked then abort the current test by throwing an exception, this so it won't fail
1948 # with an even more cryptic error.
1949 #
1950 # See Trac #13162
1951 exception = None
1952 while retries > 0 and os.path.exists(testdir):
1953 time.sleep((max_attempts-retries)*6)
1954 try:
1955 shutil.rmtree(testdir, onerror=on_error, ignore_errors=False)
1956 except Exception as e:
1957 exception = e
1958 retries -= 1
1959
1960 if retries == 0 and os.path.exists(testdir):
1961 raise Exception("Unable to remove folder '%s': %s\nUnable to start current test."
1962 % (testdir, exception))
1963 else:
1964 def cleanup():
1965 testdir = getTestOpts().testdir
1966 if os.path.exists(testdir):
1967 shutil.rmtree(testdir, ignore_errors=False)
1968
1969
1970 # -----------------------------------------------------------------------------
1971 # Return a list of all the files ending in '.T' below directories roots.
1972
1973 def findTFiles(roots):
1974 for root in roots:
1975 for path, dirs, files in os.walk(root, topdown=True):
1976 # Never pick up .T files in uncleaned .run directories.
1977 dirs[:] = [dir for dir in sorted(dirs)
1978 if not dir.endswith(testdir_suffix)]
1979 for filename in files:
1980 if filename.endswith('.T'):
1981 yield os.path.join(path, filename)
1982
1983 # -----------------------------------------------------------------------------
1984 # Output a test summary to the specified file object
1985
1986 def summary(t, file, short=False):
1987
1988 file.write('\n')
1989 printUnexpectedTests(file,
1990 [t.unexpected_passes, t.unexpected_failures,
1991 t.unexpected_stat_failures, t.framework_failures])
1992
1993 if short:
1994 # Only print the list of unexpected tests above.
1995 return
1996
1997 file.write('SUMMARY for test run started at '
1998 + time.strftime("%c %Z", t.start_time) + '\n'
1999 + str(datetime.timedelta(seconds=
2000 round(time.time() - time.mktime(t.start_time)))).rjust(8)
2001 + ' spent to go through\n'
2002 + repr(t.total_tests).rjust(8)
2003 + ' total tests, which gave rise to\n'
2004 + repr(t.total_test_cases).rjust(8)
2005 + ' test cases, of which\n'
2006 + repr(t.n_tests_skipped).rjust(8)
2007 + ' were skipped\n'
2008 + '\n'
2009 + repr(len(t.missing_libs)).rjust(8)
2010 + ' had missing libraries\n'
2011 + repr(t.n_expected_passes).rjust(8)
2012 + ' expected passes\n'
2013 + repr(t.n_expected_failures).rjust(8)
2014 + ' expected failures\n'
2015 + '\n'
2016 + repr(len(t.framework_failures)).rjust(8)
2017 + ' caused framework failures\n'
2018 + repr(len(t.framework_warnings)).rjust(8)
2019 + ' caused framework warnings\n'
2020 + repr(len(t.unexpected_passes)).rjust(8)
2021 + ' unexpected passes\n'
2022 + repr(len(t.unexpected_failures)).rjust(8)
2023 + ' unexpected failures\n'
2024 + repr(len(t.unexpected_stat_failures)).rjust(8)
2025 + ' unexpected stat failures\n'
2026 + '\n')
2027
2028 if t.unexpected_passes:
2029 file.write('Unexpected passes:\n')
2030 printTestInfosSummary(file, t.unexpected_passes)
2031
2032 if t.unexpected_failures:
2033 file.write('Unexpected failures:\n')
2034 printTestInfosSummary(file, t.unexpected_failures)
2035
2036 if t.unexpected_stat_failures:
2037 file.write('Unexpected stat failures:\n')
2038 printTestInfosSummary(file, t.unexpected_stat_failures)
2039
2040 if t.framework_failures:
2041 file.write('Framework failures:\n')
2042 printTestInfosSummary(file, t.framework_failures)
2043
2044 if t.framework_warnings:
2045 file.write('Framework warnings:\n')
2046 printTestInfosSummary(file, t.framework_warnings)
2047
2048 if stopping():
2049 file.write('WARNING: Testsuite run was terminated early\n')
2050
2051 def printUnexpectedTests(file, testInfoss):
2052 unexpected = set(name for testInfos in testInfoss
2053 for (_, name, _, _) in testInfos
2054 if not name.endswith('.T'))
2055 if unexpected:
2056 file.write('Unexpected results from:\n')
2057 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2058 file.write('\n')
2059
2060 def printTestInfosSummary(file, testInfos):
2061 maxDirLen = max(len(directory) for (directory, _, _, _) in testInfos)
2062 for (directory, name, reason, way) in testInfos:
2063 directory = directory.ljust(maxDirLen)
2064 file.write(' {directory} {name} [{reason}] ({way})\n'.format(**locals()))
2065 file.write('\n')
2066
2067 def modify_lines(s, f):
2068 s = '\n'.join([f(l) for l in s.splitlines()])
2069 if s and s[-1] != '\n':
2070 # Prevent '\ No newline at end of file' warnings when diffing.
2071 s += '\n'
2072 return s