1f08f5b38a3f6c84bb0ff06a7572c290da15d173
[ghc.git] / testsuite / driver / testlib.py
1 # coding=utf8
2 #
3 # (c) Simon Marlow 2002
4 #
5
6 from __future__ import print_function
7
8 import io
9 import shutil
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 import sys
20 from math import ceil, trunc
21 import collections
22 import subprocess
23
24 from testglobals import *
25 from testutil import *
26 extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
27
28 if config.use_threads:
29 import threading
30 try:
31 import thread
32 except ImportError: # Python 3
33 import _thread as thread
34
35 global wantToStop
36 wantToStop = False
37
38 global pool_sema
39 if config.use_threads:
40 pool_sema = threading.BoundedSemaphore(value=config.threads)
41
42 def stopNow():
43 global wantToStop
44 wantToStop = True
45 def stopping():
46 return wantToStop
47
48 # Options valid for the current test only (these get reset to
49 # testdir_testopts after each test).
50
51 global testopts_local
52 if config.use_threads:
53 testopts_local = threading.local()
54 else:
55 class TestOpts_Local:
56 pass
57 testopts_local = TestOpts_Local()
58
59 def getTestOpts():
60 return testopts_local.x
61
62 def setLocalTestOpts(opts):
63 global testopts_local
64 testopts_local.x=opts
65
66 def isStatsTest():
67 opts = getTestOpts()
68 return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
69
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 global thisdir_settings
75 thisdir_settings = [thisdir_settings, f]
76
77 # -----------------------------------------------------------------------------
78 # Canned setup functions for common cases. eg. for a test you might say
79 #
80 # test('test001', normal, compile, [''])
81 #
82 # to run it without any options, but change it to
83 #
84 # test('test001', expect_fail, compile, [''])
85 #
86 # to expect failure for this test.
87
88 def normal( name, opts ):
89 return;
90
91 def skip( name, opts ):
92 opts.skip = 1
93
94 def expect_fail( name, opts ):
95 # The compiler, testdriver, OS or platform is missing a certain
96 # feature, and we don't plan to or can't fix it now or in the
97 # future.
98 opts.expect = 'fail';
99
100 def reqlib( lib ):
101 return lambda name, opts, l=lib: _reqlib (name, opts, l )
102
103 def stage1(name, opts):
104 # See Note [Why is there no stage1 setup function?]
105 framework_fail(name, 'stage1 setup function does not exist',
106 'add your test to testsuite/tests/stage1 instead')
107
108 # Note [Why is there no stage1 setup function?]
109 #
110 # Presumably a stage1 setup function would signal that the stage1
111 # compiler should be used to compile a test.
112 #
113 # Trouble is, the path to the compiler + the `ghc --info` settings for
114 # that compiler are currently passed in from the `make` part of the
115 # testsuite driver.
116 #
117 # Switching compilers in the Python part would be entirely too late, as
118 # all ghc_with_* settings would be wrong. See config/ghc for possible
119 # consequences (for example, config.run_ways would still be
120 # based on the default compiler, quite likely causing ./validate --slow
121 # to fail).
122 #
123 # It would be possible to let the Python part of the testsuite driver
124 # make the call to `ghc --info`, but doing so would require quite some
125 # work. Care has to be taken to not affect the run_command tests for
126 # example, as they also use the `ghc --info` settings:
127 # quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
128 #
129 # If you want a test to run using the stage1 compiler, add it to the
130 # testsuite/tests/stage1 directory. Validate runs the tests in that
131 # directory with `make stage=1`.
132
133 # Cache the results of looking to see if we have a library or not.
134 # This makes quite a difference, especially on Windows.
135 have_lib = {}
136
137 def _reqlib( name, opts, lib ):
138 if lib in have_lib:
139 got_it = have_lib[lib]
140 else:
141 cmd = strip_quotes(config.ghc_pkg)
142 p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
143 stdout=subprocess.PIPE,
144 stderr=subprocess.PIPE)
145 # read from stdout and stderr to avoid blocking due to
146 # buffers filling
147 p.communicate()
148 r = p.wait()
149 got_it = r == 0
150 have_lib[lib] = got_it
151
152 if not got_it:
153 opts.expect = 'missing-lib'
154
155 def req_haddock( name, opts ):
156 if not config.haddock:
157 opts.expect = 'missing-lib'
158
159 def req_profiling( name, opts ):
160 '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
161 if not config.have_profiling:
162 opts.expect = 'fail'
163
164 def req_shared_libs( name, opts ):
165 if not config.have_shared_libs:
166 opts.expect = 'fail'
167
168 def req_interp( name, opts ):
169 if not config.have_interp:
170 opts.expect = 'fail'
171
172 def req_smp( name, opts ):
173 if not config.have_smp:
174 opts.expect = 'fail'
175
176 def ignore_stdout(name, opts):
177 opts.ignore_stdout = True
178
179 def ignore_stderr(name, opts):
180 opts.ignore_stderr = True
181
182 def combined_output( name, opts ):
183 opts.combined_output = True
184
185 # -----
186
187 def expect_fail_for( ways ):
188 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
189
190 def _expect_fail_for( name, opts, ways ):
191 opts.expect_fail_for = ways
192
193 def expect_broken( bug ):
194 # This test is a expected not to work due to the indicated trac bug
195 # number.
196 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
197
198 def _expect_broken( name, opts, bug ):
199 record_broken(name, opts, bug)
200 opts.expect = 'fail';
201
202 def expect_broken_for( bug, ways ):
203 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
204
205 def _expect_broken_for( name, opts, bug, ways ):
206 record_broken(name, opts, bug)
207 opts.expect_fail_for = ways
208
209 def record_broken(name, opts, bug):
210 global brokens
211 me = (bug, opts.testdir, name)
212 if not me in brokens:
213 brokens.append(me)
214
215 def _expect_pass(way):
216 # Helper function. Not intended for use in .T files.
217 opts = getTestOpts()
218 return opts.expect == 'pass' and way not in opts.expect_fail_for
219
220 # -----
221
222 def omit_ways( ways ):
223 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
224
225 def _omit_ways( name, opts, ways ):
226 opts.omit_ways = ways
227
228 # -----
229
230 def only_ways( ways ):
231 return lambda name, opts, w=ways: _only_ways( name, opts, w )
232
233 def _only_ways( name, opts, ways ):
234 opts.only_ways = ways
235
236 # -----
237
238 def extra_ways( ways ):
239 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
240
241 def _extra_ways( name, opts, ways ):
242 opts.extra_ways = ways
243
244 # -----
245
246 def set_stdin( file ):
247 return lambda name, opts, f=file: _set_stdin(name, opts, f);
248
249 def _set_stdin( name, opts, f ):
250 opts.stdin = f
251
252 # -----
253
254 def exit_code( val ):
255 return lambda name, opts, v=val: _exit_code(name, opts, v);
256
257 def _exit_code( name, opts, v ):
258 opts.exit_code = v
259
260 def signal_exit_code( val ):
261 if opsys('solaris2'):
262 return exit_code( val );
263 else:
264 # When application running on Linux receives fatal error
265 # signal, then its exit code is encoded as 128 + signal
266 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
267 # I assume that Mac OS X behaves in the same way at least Mac
268 # OS X builder behavior suggests this.
269 return exit_code( val+128 );
270
271 # -----
272
273 def compile_timeout_multiplier( val ):
274 return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
275
276 def _compile_timeout_multiplier( name, opts, v ):
277 opts.compile_timeout_multiplier = v
278
279 def run_timeout_multiplier( val ):
280 return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)
281
282 def _run_timeout_multiplier( name, opts, v ):
283 opts.run_timeout_multiplier = v
284
285 # -----
286
287 def extra_run_opts( val ):
288 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
289
290 def _extra_run_opts( name, opts, v ):
291 opts.extra_run_opts = v
292
293 # -----
294
295 def extra_hc_opts( val ):
296 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
297
298 def _extra_hc_opts( name, opts, v ):
299 opts.extra_hc_opts = v
300
301 # -----
302
303 def extra_clean( files ):
304 # TODO. Remove all calls to extra_clean.
305 return lambda _name, _opts: None
306
307 def extra_files(files):
308 return lambda name, opts: _extra_files(name, opts, files)
309
310 def _extra_files(name, opts, files):
311 opts.extra_files.extend(files)
312
313 # -----
314
315 def stats_num_field( field, expecteds ):
316 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
317
318 def _stats_num_field( name, opts, field, expecteds ):
319 if field in opts.stats_range_fields:
320 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
321
322 if type(expecteds) is list:
323 for (b, expected, dev) in expecteds:
324 if b:
325 opts.stats_range_fields[field] = (expected, dev)
326 return
327 framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
328
329 else:
330 (expected, dev) = expecteds
331 opts.stats_range_fields[field] = (expected, dev)
332
333 def compiler_stats_num_field( field, expecteds ):
334 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
335
336 def _compiler_stats_num_field( name, opts, field, expecteds ):
337 if field in opts.compiler_stats_range_fields:
338 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
339
340 # Compiler performance numbers change when debugging is on, making the results
341 # useless and confusing. Therefore, skip if debugging is on.
342 if compiler_debugged():
343 skip(name, opts)
344
345 for (b, expected, dev) in expecteds:
346 if b:
347 opts.compiler_stats_range_fields[field] = (expected, dev)
348 return
349
350 framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
351
352 # -----
353
354 def when(b, f):
355 # When list_brokens is on, we want to see all expect_broken calls,
356 # so we always do f
357 if b or config.list_broken:
358 return f
359 else:
360 return normal
361
362 def unless(b, f):
363 return when(not b, f)
364
365 def doing_ghci():
366 return 'ghci' in config.run_ways
367
368 def ghc_dynamic():
369 return config.ghc_dynamic
370
371 def fast():
372 return config.speed == 2
373
374 def platform( plat ):
375 return config.platform == plat
376
377 def opsys( os ):
378 return config.os == os
379
380 def arch( arch ):
381 return config.arch == arch
382
383 def wordsize( ws ):
384 return config.wordsize == str(ws)
385
386 def msys( ):
387 return config.msys
388
389 def cygwin( ):
390 return config.cygwin
391
392 def have_vanilla( ):
393 return config.have_vanilla
394
395 def have_dynamic( ):
396 return config.have_dynamic
397
398 def have_profiling( ):
399 return config.have_profiling
400
401 def in_tree_compiler( ):
402 return config.in_tree_compiler
403
404 def unregisterised( ):
405 return config.unregisterised
406
407 def compiler_profiled( ):
408 return config.compiler_profiled
409
410 def compiler_debugged( ):
411 return config.compiler_debugged
412
413 # ---
414
415 def high_memory_usage(name, opts):
416 opts.alone = True
417
418 # If a test is for a multi-CPU race, then running the test alone
419 # increases the chance that we'll actually see it.
420 def multi_cpu_race(name, opts):
421 opts.alone = True
422
423 # ---
424 def literate( name, opts ):
425 opts.literate = 1;
426
427 def c_src( name, opts ):
428 opts.c_src = 1;
429
430 def objc_src( name, opts ):
431 opts.objc_src = 1;
432
433 def objcpp_src( name, opts ):
434 opts.objcpp_src = 1;
435
436 def cmm_src( name, opts ):
437 opts.cmm_src = 1;
438
439 def outputdir( odir ):
440 return lambda name, opts, d=odir: _outputdir(name, opts, d)
441
442 def _outputdir( name, opts, odir ):
443 opts.outputdir = odir;
444
445 # ----
446
447 def pre_cmd( cmd ):
448 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
449
450 def _pre_cmd( name, opts, cmd ):
451 opts.pre_cmd = cmd
452
453 # ----
454
455 def clean_cmd( cmd ):
456 # TODO. Remove all calls to clean_cmd.
457 return lambda _name, _opts: None
458
459 # ----
460
461 def cmd_prefix( prefix ):
462 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
463
464 def _cmd_prefix( name, opts, prefix ):
465 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
466
467 # ----
468
469 def cmd_wrapper( fun ):
470 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
471
472 def _cmd_wrapper( name, opts, fun ):
473 opts.cmd_wrapper = fun
474
475 # ----
476
477 def compile_cmd_prefix( prefix ):
478 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
479
480 def _compile_cmd_prefix( name, opts, prefix ):
481 opts.compile_cmd_prefix = prefix
482
483 # ----
484
485 def check_stdout( f ):
486 return lambda name, opts, f=f: _check_stdout(name, opts, f)
487
488 def _check_stdout( name, opts, f ):
489 opts.check_stdout = f
490
491 def no_check_hp(name, opts):
492 opts.check_hp = False
493
494 # ----
495
496 def filter_stdout_lines( regex ):
497 """ Filter lines of stdout with the given regular expression """
498 import re
499 def f( name, opts ):
500 _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
501 return f
502
503 def normalise_slashes( name, opts ):
504 _normalise_fun(name, opts, normalise_slashes_)
505
506 def normalise_exe( name, opts ):
507 _normalise_fun(name, opts, normalise_exe_)
508
509 def normalise_fun( *fs ):
510 return lambda name, opts: _normalise_fun(name, opts, fs)
511
512 def _normalise_fun( name, opts, *fs ):
513 opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
514
515 def normalise_errmsg_fun( *fs ):
516 return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
517
518 def _normalise_errmsg_fun( name, opts, *fs ):
519 opts.extra_errmsg_normaliser = join_normalisers(opts.extra_errmsg_normaliser, fs)
520
521 def normalise_version_( *pkgs ):
522 def normalise_version__( str ):
523 return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
524 '\\1-<VERSION>', str)
525 return normalise_version__
526
527 def normalise_version( *pkgs ):
528 def normalise_version__( name, opts ):
529 _normalise_fun(name, opts, normalise_version_(*pkgs))
530 _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
531 return normalise_version__
532
533 def normalise_drive_letter(name, opts):
534 # Windows only. Change D:\\ to C:\\.
535 _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))
536
537 def keep_prof_callstacks(name, opts):
538 """Keep profiling callstacks.
539
540 Use together with `only_ways(prof_ways)`.
541 """
542 opts.keep_prof_callstacks = True
543
544 def join_normalisers(*a):
545 """
546 Compose functions, flattening sequences.
547
548 join_normalisers(f1,[f2,f3],f4)
549
550 is the same as
551
552 lambda x: f1(f2(f3(f4(x))))
553 """
554
555 def flatten(l):
556 """
557 Taken from http://stackoverflow.com/a/2158532/946226
558 """
559 for el in l:
560 if (isinstance(el, collections.Iterable)
561 and not isinstance(el, (bytes, str))):
562 for sub in flatten(el):
563 yield sub
564 else:
565 yield el
566
567 a = flatten(a)
568
569 fn = lambda x:x # identity function
570 for f in a:
571 assert callable(f)
572 fn = lambda x,f=f,fn=fn: fn(f(x))
573 return fn
574
575 # ----
576 # Function for composing two opt-fns together
577
578 def executeSetups(fs, name, opts):
579 if type(fs) is list:
580 # If we have a list of setups, then execute each one
581 for f in fs:
582 executeSetups(f, name, opts)
583 else:
584 # fs is a single function, so just apply it
585 fs(name, opts)
586
587 # -----------------------------------------------------------------------------
588 # The current directory of tests
589
590 def newTestDir(tempdir, dir):
591
592 global thisdir_settings
593 # reset the options for this test directory
594 def settings(name, opts, tempdir=tempdir, dir=dir):
595 return _newTestDir(name, opts, tempdir, dir)
596 thisdir_settings = settings
597
598 # Should be equal to entry in toplevel .gitignore.
599 testdir_suffix = '.run'
600
601 def _newTestDir(name, opts, tempdir, dir):
602 opts.srcdir = os.path.join(os.getcwd(), dir)
603 opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
604 opts.compiler_always_flags = config.compiler_always_flags
605
606 # -----------------------------------------------------------------------------
607 # Actually doing tests
608
609 parallelTests = []
610 aloneTests = []
611 allTestNames = set([])
612
613 def runTest(watcher, opts, name, func, args):
614 if config.use_threads:
615 pool_sema.acquire()
616 t = threading.Thread(target=test_common_thread,
617 name=name,
618 args=(watcher, name, opts, func, args))
619 t.daemon = False
620 t.start()
621 else:
622 test_common_work(watcher, name, opts, func, args)
623
624 # name :: String
625 # setup :: TestOpts -> IO ()
626 def test(name, setup, func, args):
627 global aloneTests
628 global parallelTests
629 global allTestNames
630 global thisdir_settings
631 if name in allTestNames:
632 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
633 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
634 framework_fail(name, 'bad_name', 'This test has an invalid name')
635
636 if config.run_only_some_tests:
637 if name not in config.only:
638 return
639 else:
640 # Note [Mutating config.only]
641 # config.only is initially the set of tests requested by
642 # the user (via 'make TEST='). We then remove all tests that
643 # we've already seen (in .T files), so that we can later
644 # report on any tests we couldn't find and error out.
645 config.only.remove(name)
646
647 # Make a deep copy of the default_testopts, as we need our own copy
648 # of any dictionaries etc inside it. Otherwise, if one test modifies
649 # them, all tests will see the modified version!
650 myTestOpts = copy.deepcopy(default_testopts)
651
652 executeSetups([thisdir_settings, setup], name, myTestOpts)
653
654 thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
655 if myTestOpts.alone:
656 aloneTests.append(thisTest)
657 else:
658 parallelTests.append(thisTest)
659 allTestNames.add(name)
660
661 if config.use_threads:
662 def test_common_thread(watcher, name, opts, func, args):
663 try:
664 test_common_work(watcher, name, opts, func, args)
665 finally:
666 pool_sema.release()
667
668 def get_package_cache_timestamp():
669 if config.package_conf_cache_file == '':
670 return 0.0
671 else:
672 try:
673 return os.stat(config.package_conf_cache_file).st_mtime
674 except:
675 return 0.0
676
677 do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
678
679 def test_common_work(watcher, name, opts, func, args):
680 try:
681 t.total_tests += 1
682 setLocalTestOpts(opts)
683
684 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
685
686 # All the ways we might run this test
687 if func == compile or func == multimod_compile:
688 all_ways = config.compile_ways
689 elif func == compile_and_run or func == multimod_compile_and_run:
690 all_ways = config.run_ways
691 elif func == ghci_script:
692 if 'ghci' in config.run_ways:
693 all_ways = ['ghci']
694 else:
695 all_ways = []
696 else:
697 all_ways = ['normal']
698
699 # A test itself can request extra ways by setting opts.extra_ways
700 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
701
702 t.total_test_cases += len(all_ways)
703
704 ok_way = lambda way: \
705 not getTestOpts().skip \
706 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
707 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
708 and (not (config.skip_perf_tests and isStatsTest())) \
709 and way not in getTestOpts().omit_ways
710
711 # Which ways we are asked to skip
712 do_ways = list(filter (ok_way,all_ways))
713
714 # Only run all ways in slow mode.
715 # See Note [validate and testsuite speed] in toplevel Makefile.
716 if config.accept:
717 # Only ever run one way
718 do_ways = do_ways[:1]
719 elif config.speed > 0:
720 # However, if we EXPLICITLY asked for a way (with extra_ways)
721 # please test it!
722 explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
723 other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
724 do_ways = other_ways[:1] + explicit_ways
725
726 # Find all files in the source directory that this test
727 # depends on. Do this only once for all ways.
728 # Generously add all filenames that start with the name of
729 # the test to this set, as a convenience to test authors.
730 # They will have to use the `extra_files` setup function to
731 # specify all other files that their test depends on (but
732 # this seems to be necessary for only about 10% of all
733 # tests).
734 files = set(f for f in os.listdir(opts.srcdir)
735 if f.startswith(name) and not f == name and
736 not f.endswith(testdir_suffix) and
737 not os.path.splitext(f)[1] in do_not_copy)
738 for filename in (opts.extra_files + extra_src_files.get(name, [])):
739 if filename.startswith('/'):
740 framework_fail(name, 'whole-test',
741 'no absolute paths in extra_files please: ' + filename)
742
743 elif '*' in filename:
744 # Don't use wildcards in extra_files too much, as
745 # globbing is slow.
746 files.update((os.path.relpath(f, opts.srcdir)
747 for f in glob.iglob(in_srcdir(filename))))
748
749 elif filename:
750 files.add(filename)
751
752 else:
753 framework_fail(name, 'whole-test', 'extra_file is empty string')
754
755 # Run the required tests...
756 for way in do_ways:
757 if stopping():
758 break
759 try:
760 do_test(name, way, func, args, files)
761 except KeyboardInterrupt:
762 stopNow()
763 except Exception as e:
764 framework_fail(name, way, str(e))
765 traceback.print_exc()
766
767 t.n_tests_skipped += len(set(all_ways) - set(do_ways))
768
769 if config.cleanup and do_ways:
770 try:
771 cleanup()
772 except Exception as e:
773 framework_fail(name, 'runTest', 'Unhandled exception during cleanup: ' + str(e))
774
775 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
776
777 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
778 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
779
780 except Exception as e:
781 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
782 finally:
783 watcher.notify()
784
785 def do_test(name, way, func, args, files):
786 opts = getTestOpts()
787
788 full_name = name + '(' + way + ')'
789
790 if_verbose(2, "=====> {0} {1} of {2} {3}".format(
791 full_name, t.total_tests, len(allTestNames),
792 [len(t.unexpected_passes),
793 len(t.unexpected_failures),
794 len(t.framework_failures)]))
795
796 # Clean up prior to the test, so that we can't spuriously conclude
797 # that it passed on the basis of old run outputs.
798 cleanup()
799 os.makedirs(opts.testdir)
800
801 # Link all source files for this test into a new directory in
802 # /tmp, and run the test in that directory. This makes it
803 # possible to run tests in parallel, without modification, that
804 # would otherwise (accidentally) write to the same output file.
805 # It also makes it easier to keep the testsuite clean.
806
807 for extra_file in files:
808 src = in_srcdir(extra_file)
809 dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
810 if os.path.isfile(src):
811 link_or_copy_file(src, dst)
812 elif os.path.isdir(src):
813 os.mkdir(dst)
814 lndir(src, dst)
815 else:
816 if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
817 # When using a ghc built without haddock support, .t
818 # files are rightfully missing. Don't
819 # framework_fail. Test will be skipped later.
820 pass
821 else:
822 framework_fail(name, way,
823 'extra_file does not exist: ' + extra_file)
824
825 if func.__name__ == 'run_command' or opts.pre_cmd:
826 # When running 'MAKE' make sure 'TOP' still points to the
827 # root of the testsuite.
828 src_makefile = in_srcdir('Makefile')
829 dst_makefile = in_testdir('Makefile')
830 if os.path.exists(src_makefile):
831 with io.open(src_makefile, 'r', encoding='utf8') as src:
832 makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
833 with io.open(dst_makefile, 'w', encoding='utf8') as dst:
834 dst.write(makefile)
835
836 if opts.pre_cmd:
837 exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, override_options(opts.pre_cmd)),
838 stderr = subprocess.STDOUT,
839 print_output = config.verbose >= 3)
840
841 if exit_code != 0:
842 framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
843
844 result = func(*[name,way] + args)
845
846 if opts.expect not in ['pass', 'fail', 'missing-lib']:
847 framework_fail(name, way, 'bad expected ' + opts.expect)
848
849 try:
850 passFail = result['passFail']
851 except (KeyError, TypeError):
852 passFail = 'No passFail found'
853
854 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
855
856 if passFail == 'pass':
857 if _expect_pass(way):
858 t.n_expected_passes += 1
859 else:
860 if_verbose(1, '*** unexpected pass for %s' % full_name)
861 t.unexpected_passes.append((directory, name, 'unexpected', way))
862 elif passFail == 'fail':
863 if _expect_pass(way):
864 reason = result['reason']
865 tag = result.get('tag')
866 if tag == 'stat':
867 if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
868 t.unexpected_stat_failures.append((directory, name, reason, way))
869 else:
870 if_verbose(1, '*** unexpected failure for %s' % full_name)
871 t.unexpected_failures.append((directory, name, reason, way))
872 else:
873 if opts.expect == 'missing-lib':
874 t.missing_libs.append((directory, name, 'missing-lib', way))
875 else:
876 t.n_expected_failures += 1
877 else:
878 framework_fail(name, way, 'bad result ' + passFail)
879
880 # Make is often invoked with -s, which means if it fails, we get
881 # no feedback at all. This is annoying. So let's remove the option
882 # if found and instead have the testsuite decide on what to do
883 # with the output.
884 def override_options(pre_cmd):
885 if config.verbose >= 5 and bool(re.match('\$make', pre_cmd, re.I)):
886 return pre_cmd.replace('-s' , '') \
887 .replace('--silent', '') \
888 .replace('--quiet' , '')
889
890 return pre_cmd
891
892 def framework_fail(name, way, reason):
893 opts = getTestOpts()
894 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
895 full_name = name + '(' + way + ')'
896 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
897 t.framework_failures.append((directory, name, way, reason))
898
899 def framework_warn(name, way, reason):
900 opts = getTestOpts()
901 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
902 full_name = name + '(' + way + ')'
903 if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
904 t.framework_warnings.append((directory, name, way, reason))
905
906 def badResult(result):
907 try:
908 if result['passFail'] == 'pass':
909 return False
910 return True
911 except (KeyError, TypeError):
912 return True
913
914 def passed():
915 return {'passFail': 'pass'}
916
917 def failBecause(reason, tag=None):
918 return {'passFail': 'fail', 'reason': reason, 'tag': tag}
919
920 # -----------------------------------------------------------------------------
921 # Generic command tests
922
923 # A generic command test is expected to run and exit successfully.
924 #
925 # The expected exit code can be changed via exit_code() as normal, and
926 # the expected stdout/stderr are stored in <testname>.stdout and
927 # <testname>.stderr. The output of the command can be ignored
928 # altogether by using the setup function ignore_stdout instead of
929 # run_command.
930
931 def run_command( name, way, cmd ):
932 return simple_run( name, '', override_options(cmd), '' )
933
934 # -----------------------------------------------------------------------------
935 # GHCi tests
936
937 def ghci_script( name, way, script):
938 flags = ' '.join(get_compiler_flags())
939 way_flags = ' '.join(config.way_flags[way])
940
941 # We pass HC and HC_OPTS as environment variables, so that the
942 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
943 cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
944 ).format(flags=flags, way_flags=way_flags)
945
946 getTestOpts().stdin = script
947 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
948
949 # -----------------------------------------------------------------------------
950 # Compile-only tests
951
952 def compile( name, way, extra_hc_opts ):
953 return do_compile( name, way, 0, '', [], extra_hc_opts )
954
955 def compile_fail( name, way, extra_hc_opts ):
956 return do_compile( name, way, 1, '', [], extra_hc_opts )
957
958 def backpack_typecheck( name, way, extra_hc_opts ):
959 return do_compile( name, way, 0, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
960
961 def backpack_typecheck_fail( name, way, extra_hc_opts ):
962 return do_compile( name, way, 1, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
963
964 def backpack_compile( name, way, extra_hc_opts ):
965 return do_compile( name, way, 0, '', [], extra_hc_opts, backpack=1 )
966
967 def backpack_compile_fail( name, way, extra_hc_opts ):
968 return do_compile( name, way, 1, '', [], extra_hc_opts, backpack=1 )
969
970 def backpack_run( name, way, extra_hc_opts ):
971 return compile_and_run__( name, way, '', [], extra_hc_opts, backpack=1 )
972
973 def multimod_compile( name, way, top_mod, extra_hc_opts ):
974 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
975
976 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
977 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
978
979 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
980 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
981
982 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
983 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
984
985 def do_compile(name, way, should_fail, top_mod, extra_mods, extra_hc_opts, **kwargs):
986 # print 'Compile only, extra args = ', extra_hc_opts
987
988 result = extras_build( way, extra_mods, extra_hc_opts )
989 if badResult(result):
990 return result
991 extra_hc_opts = result['hc_opts']
992
993 result = simple_build(name, way, extra_hc_opts, should_fail, top_mod, 0, 1, **kwargs)
994
995 if badResult(result):
996 return result
997
998 # the actual stderr should always match the expected, regardless
999 # of whether we expected the compilation to fail or not (successful
1000 # compilations may generate warnings).
1001
1002 expected_stderr_file = find_expected_file(name, 'stderr')
1003 actual_stderr_file = add_suffix(name, 'comp.stderr')
1004
1005 if not compare_outputs(way, 'stderr',
1006 join_normalisers(getTestOpts().extra_errmsg_normaliser,
1007 normalise_errmsg),
1008 expected_stderr_file, actual_stderr_file,
1009 whitespace_normaliser=normalise_whitespace):
1010 return failBecause('stderr mismatch')
1011
1012 # no problems found, this test passed
1013 return passed()
1014
1015 def compile_cmp_asm( name, way, extra_hc_opts ):
1016 print('Compile only, extra args = ', extra_hc_opts)
1017 result = simple_build(name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0)
1018
1019 if badResult(result):
1020 return result
1021
1022 # the actual stderr should always match the expected, regardless
1023 # of whether we expected the compilation to fail or not (successful
1024 # compilations may generate warnings).
1025
1026 expected_asm_file = find_expected_file(name, 'asm')
1027 actual_asm_file = add_suffix(name, 's')
1028
1029 if not compare_outputs(way, 'asm',
1030 join_normalisers(normalise_errmsg, normalise_asm),
1031 expected_asm_file, actual_asm_file):
1032 return failBecause('asm mismatch')
1033
1034 # no problems found, this test passed
1035 return passed()
1036
1037 # -----------------------------------------------------------------------------
1038 # Compile-and-run tests
1039
1040 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts, backpack=0 ):
1041 # print 'Compile and run, extra args = ', extra_hc_opts
1042
1043 result = extras_build( way, extra_mods, extra_hc_opts )
1044 if badResult(result):
1045 return result
1046 extra_hc_opts = result['hc_opts']
1047
1048 if way.startswith('ghci'): # interpreted...
1049 return interpreter_run(name, way, extra_hc_opts, top_mod)
1050 else: # compiled...
1051 result = simple_build(name, way, extra_hc_opts, 0, top_mod, 1, 1, backpack = backpack)
1052 if badResult(result):
1053 return result
1054
1055 cmd = './' + name;
1056
1057 # we don't check the compiler's stderr for a compile-and-run test
1058 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1059
1060 def compile_and_run( name, way, extra_hc_opts ):
1061 return compile_and_run__( name, way, '', [], extra_hc_opts)
1062
1063 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1064 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1065
1066 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1067 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1068
1069 def stats( name, way, stats_file ):
1070 opts = getTestOpts()
1071 return checkStats(name, way, stats_file, opts.stats_range_fields)
1072
1073 # -----------------------------------------------------------------------------
1074 # Check -t stats info
1075
1076 def checkStats(name, way, stats_file, range_fields):
1077 full_name = name + '(' + way + ')'
1078
1079 result = passed()
1080 if range_fields:
1081 try:
1082 f = open(in_testdir(stats_file))
1083 except IOError as e:
1084 return failBecause(str(e))
1085 contents = f.read()
1086 f.close()
1087
1088 for (field, (expected, dev)) in range_fields.items():
1089 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1090 if m == None:
1091 print('Failed to find field: ', field)
1092 result = failBecause('no such stats field')
1093 val = int(m.group(1))
1094
1095 lowerBound = trunc( expected * ((100 - float(dev))/100))
1096 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1097
1098 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1099
1100 if val < lowerBound:
1101 print(field, 'value is too low:')
1102 print('(If this is because you have improved GHC, please')
1103 print('update the test so that GHC doesn\'t regress again)')
1104 result = failBecause('stat too good', tag='stat')
1105 if val > upperBound:
1106 print(field, 'value is too high:')
1107 result = failBecause('stat not good enough', tag='stat')
1108
1109 if val < lowerBound or val > upperBound or config.verbose >= 4:
1110 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1111
1112 def display(descr, val, extra):
1113 print(descr, str(val).rjust(length), extra)
1114
1115 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1116 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1117 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1118 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1119 if val != expected:
1120 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1121
1122 return result
1123
1124 # -----------------------------------------------------------------------------
1125 # Build a single-module program
1126
1127 def extras_build( way, extra_mods, extra_hc_opts ):
1128 for mod, opts in extra_mods:
1129 result = simple_build(mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0)
1130 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1131 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1132 if badResult(result):
1133 return result
1134
1135 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1136
1137 def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, backpack = False):
1138 opts = getTestOpts()
1139
1140 # Redirect stdout and stderr to the same file
1141 stdout = in_testdir(name, 'comp.stderr')
1142 stderr = subprocess.STDOUT
1143
1144 if top_mod != '':
1145 srcname = top_mod
1146 elif addsuf:
1147 if backpack:
1148 srcname = add_suffix(name, 'bkp')
1149 else:
1150 srcname = add_hs_lhs_suffix(name)
1151 else:
1152 srcname = name
1153
1154 if top_mod != '':
1155 to_do = '--make '
1156 if link:
1157 to_do = to_do + '-o ' + name
1158 elif backpack:
1159 if link:
1160 to_do = '-o ' + name + ' '
1161 else:
1162 to_do = ''
1163 to_do = to_do + '--backpack '
1164 elif link:
1165 to_do = '-o ' + name
1166 else:
1167 to_do = '-c' # just compile
1168
1169 stats_file = name + '.comp.stats'
1170 if opts.compiler_stats_range_fields:
1171 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1172 if backpack:
1173 extra_hc_opts += ' -outputdir ' + name + '.out'
1174
1175 # Required by GHC 7.3+, harmless for earlier versions:
1176 if (getTestOpts().c_src or
1177 getTestOpts().objc_src or
1178 getTestOpts().objcpp_src or
1179 getTestOpts().cmm_src):
1180 extra_hc_opts += ' -no-hs-main '
1181
1182 if getTestOpts().compile_cmd_prefix == '':
1183 cmd_prefix = ''
1184 else:
1185 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1186
1187 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1188
1189 cmd = ('cd "{opts.testdir}" && {cmd_prefix} '
1190 '{{compiler}} {to_do} {srcname} {flags} {extra_hc_opts}'
1191 ).format(**locals())
1192
1193 exit_code = runCmd(cmd, None, stdout, stderr, opts.compile_timeout_multiplier)
1194
1195 if exit_code != 0 and not should_fail:
1196 if config.verbose >= 1 and _expect_pass(way):
1197 print('Compile failed (exit code {0}) errors were:'.format(exit_code))
1198 actual_stderr_path = in_testdir(name, 'comp.stderr')
1199 if_verbose_dump(1, actual_stderr_path)
1200
1201 # ToDo: if the sub-shell was killed by ^C, then exit
1202
1203 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1204
1205 if badResult(statsResult):
1206 return statsResult
1207
1208 if should_fail:
1209 if exit_code == 0:
1210 return failBecause('exit code 0')
1211 else:
1212 if exit_code != 0:
1213 return failBecause('exit code non-0')
1214
1215 return passed()
1216
1217 # -----------------------------------------------------------------------------
1218 # Run a program and check its output
1219 #
1220 # If testname.stdin exists, route input from that, else
1221 # from /dev/null. Route output to testname.run.stdout and
1222 # testname.run.stderr. Returns the exit code of the run.
1223
1224 def simple_run(name, way, prog, extra_run_opts):
1225 opts = getTestOpts()
1226
1227 # figure out what to use for stdin
1228 if opts.stdin:
1229 stdin = in_testdir(opts.stdin)
1230 elif os.path.exists(in_testdir(name, 'stdin')):
1231 stdin = in_testdir(name, 'stdin')
1232 else:
1233 stdin = None
1234
1235 stdout = in_testdir(name, 'run.stdout')
1236 if opts.combined_output:
1237 stderr = subprocess.STDOUT
1238 else:
1239 stderr = in_testdir(name, 'run.stderr')
1240
1241 my_rts_flags = rts_flags(way)
1242
1243 stats_file = name + '.stats'
1244 if opts.stats_range_fields:
1245 stats_args = ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1246 else:
1247 stats_args = ''
1248
1249 # Put extra_run_opts last: extra_run_opts('+RTS foo') should work.
1250 cmd = prog + stats_args + ' ' + my_rts_flags + ' ' + extra_run_opts
1251
1252 if opts.cmd_wrapper != None:
1253 cmd = opts.cmd_wrapper(cmd)
1254
1255 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1256
1257 # run the command
1258 exit_code = runCmd(cmd, stdin, stdout, stderr, opts.run_timeout_multiplier)
1259
1260 # check the exit code
1261 if exit_code != opts.exit_code:
1262 if config.verbose >= 1 and _expect_pass(way):
1263 print('Wrong exit code for ' + name + '(' + way + ')' + '(expected', opts.exit_code, ', actual', exit_code, ')')
1264 dump_stdout(name)
1265 dump_stderr(name)
1266 return failBecause('bad exit code')
1267
1268 if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
1269 return failBecause('bad stderr')
1270 if not (opts.ignore_stdout or stdout_ok(name, way)):
1271 return failBecause('bad stdout')
1272
1273 check_hp = '-h' in my_rts_flags and opts.check_hp
1274 check_prof = '-p' in my_rts_flags
1275
1276 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1277 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1278 return failBecause('bad heap profile')
1279 if check_prof and not check_prof_ok(name, way):
1280 return failBecause('bad profile')
1281
1282 return checkStats(name, way, stats_file, opts.stats_range_fields)
1283
1284 def rts_flags(way):
1285 args = config.way_rts_flags.get(way, [])
1286 return '+RTS {0} -RTS'.format(' '.join(args)) if args else ''
1287
1288 # -----------------------------------------------------------------------------
1289 # Run a program in the interpreter and check its output
1290
1291 def interpreter_run(name, way, extra_hc_opts, top_mod):
1292 opts = getTestOpts()
1293
1294 stdout = in_testdir(name, 'interp.stdout')
1295 stderr = in_testdir(name, 'interp.stderr')
1296 script = in_testdir(name, 'genscript')
1297
1298 if opts.combined_output:
1299 framework_fail(name, 'unsupported',
1300 'WAY=ghci and combined_output together is not supported')
1301
1302 if (top_mod == ''):
1303 srcname = add_hs_lhs_suffix(name)
1304 else:
1305 srcname = top_mod
1306
1307 delimiter = '===== program output begins here\n'
1308
1309 with io.open(script, 'w', encoding='utf8') as f:
1310 # set the prog name and command-line args to match the compiled
1311 # environment.
1312 f.write(':set prog ' + name + '\n')
1313 f.write(':set args ' + opts.extra_run_opts + '\n')
1314 # Add marker lines to the stdout and stderr output files, so we
1315 # can separate GHCi's output from the program's.
1316 f.write(':! echo ' + delimiter)
1317 f.write(':! echo 1>&2 ' + delimiter)
1318 # Set stdout to be line-buffered to match the compiled environment.
1319 f.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1320 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1321 # in the event of an exception as for the compiled program.
1322 f.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1323
1324 stdin = in_testdir(opts.stdin if opts.stdin else add_suffix(name, 'stdin'))
1325 if os.path.exists(stdin):
1326 os.system('cat "{0}" >> "{1}"'.format(stdin, script))
1327
1328 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1329
1330 cmd = ('{{compiler}} {srcname} {flags} {extra_hc_opts}'
1331 ).format(**locals())
1332
1333 if getTestOpts().cmd_wrapper != None:
1334 cmd = opts.cmd_wrapper(cmd);
1335
1336 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1337
1338 exit_code = runCmd(cmd, script, stdout, stderr, opts.run_timeout_multiplier)
1339
1340 # split the stdout into compilation/program output
1341 split_file(stdout, delimiter,
1342 in_testdir(name, 'comp.stdout'),
1343 in_testdir(name, 'run.stdout'))
1344 split_file(stderr, delimiter,
1345 in_testdir(name, 'comp.stderr'),
1346 in_testdir(name, 'run.stderr'))
1347
1348 # check the exit code
1349 if exit_code != getTestOpts().exit_code:
1350 print('Wrong exit code for ' + name + '(' + way + ') (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1351 dump_stdout(name)
1352 dump_stderr(name)
1353 return failBecause('bad exit code')
1354
1355 # ToDo: if the sub-shell was killed by ^C, then exit
1356
1357 if not (opts.ignore_stderr or stderr_ok(name, way)):
1358 return failBecause('bad stderr')
1359 elif not (opts.ignore_stdout or stdout_ok(name, way)):
1360 return failBecause('bad stdout')
1361 else:
1362 return passed()
1363
1364 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1365 # See Note [Universal newlines].
1366 with io.open(in_fn, 'r', encoding='utf8', errors='replace', newline=None) as infile:
1367 with io.open(out1_fn, 'w', encoding='utf8', newline='') as out1:
1368 with io.open(out2_fn, 'w', encoding='utf8', newline='') as out2:
1369 line = infile.readline()
1370 while re.sub('^\s*','',line) != delimiter and line != '':
1371 out1.write(line)
1372 line = infile.readline()
1373
1374 line = infile.readline()
1375 while line != '':
1376 out2.write(line)
1377 line = infile.readline()
1378
1379 # -----------------------------------------------------------------------------
1380 # Utils
1381 def get_compiler_flags():
1382 opts = getTestOpts()
1383
1384 flags = copy.copy(opts.compiler_always_flags)
1385
1386 flags.append(opts.extra_hc_opts)
1387
1388 if opts.outputdir != None:
1389 flags.extend(["-outputdir", opts.outputdir])
1390
1391 return flags
1392
1393 def stdout_ok(name, way):
1394 actual_stdout_file = add_suffix(name, 'run.stdout')
1395 expected_stdout_file = find_expected_file(name, 'stdout')
1396
1397 extra_norm = join_normalisers(normalise_output, getTestOpts().extra_normaliser)
1398
1399 check_stdout = getTestOpts().check_stdout
1400 if check_stdout:
1401 actual_stdout_path = in_testdir(actual_stdout_file)
1402 return check_stdout(actual_stdout_path, extra_norm)
1403
1404 return compare_outputs(way, 'stdout', extra_norm,
1405 expected_stdout_file, actual_stdout_file)
1406
1407 def dump_stdout( name ):
1408 with open(in_testdir(name, 'run.stdout')) as f:
1409 str = f.read().strip()
1410 if str:
1411 print("Stdout (", name, "):")
1412 print(str)
1413
1414 def stderr_ok(name, way):
1415 actual_stderr_file = add_suffix(name, 'run.stderr')
1416 expected_stderr_file = find_expected_file(name, 'stderr')
1417
1418 return compare_outputs(way, 'stderr',
1419 join_normalisers(normalise_errmsg, getTestOpts().extra_errmsg_normaliser), \
1420 expected_stderr_file, actual_stderr_file,
1421 whitespace_normaliser=normalise_whitespace)
1422
1423 def dump_stderr( name ):
1424 with open(in_testdir(name, 'run.stderr')) as f:
1425 str = f.read().strip()
1426 if str:
1427 print("Stderr (", name, "):")
1428 print(str)
1429
1430 def read_no_crs(file):
1431 str = ''
1432 try:
1433 # See Note [Universal newlines].
1434 with io.open(file, 'r', encoding='utf8', errors='replace', newline=None) as h:
1435 str = h.read()
1436 except Exception:
1437 # On Windows, if the program fails very early, it seems the
1438 # files stdout/stderr are redirected to may not get created
1439 pass
1440 return str
1441
1442 def write_file(file, str):
1443 # See Note [Universal newlines].
1444 with io.open(file, 'w', encoding='utf8', newline='') as h:
1445 h.write(str)
1446
1447 # Note [Universal newlines]
1448 #
1449 # We don't want to write any Windows style line endings ever, because
1450 # it would mean that `make accept` would touch every line of the file
1451 # when switching between Linux and Windows.
1452 #
1453 # Furthermore, when reading a file, it is convenient to translate all
1454 # Windows style endings to '\n', as it simplifies searching or massaging
1455 # the content.
1456 #
1457 # Solution: use `io.open` instead of `open`
1458 # * when reading: use newline=None to translate '\r\n' to '\n'
1459 # * when writing: use newline='' to not translate '\n' to '\r\n'
1460 #
1461 # See https://docs.python.org/2/library/io.html#io.open.
1462 #
1463 # This should work with both python2 and python3, and with both mingw*
1464 # as msys2 style Python.
1465 #
1466 # Do note that io.open returns unicode strings. So we have to specify
1467 # the expected encoding. But there is at least one file which is not
1468 # valid utf8 (decodingerror002.stdout). Solution: use errors='replace'.
1469 # Another solution would be to open files in binary mode always, and
1470 # operate on bytes.
1471
1472 def check_hp_ok(name):
1473 opts = getTestOpts()
1474
1475 # do not qualify for hp2ps because we should be in the right directory
1476 hp2psCmd = 'cd "{opts.testdir}" && {{hp2ps}} {name}'.format(**locals())
1477
1478 hp2psResult = runCmd(hp2psCmd)
1479
1480 actual_ps_path = in_testdir(name, 'ps')
1481
1482 if hp2psResult == 0:
1483 if os.path.exists(actual_ps_path):
1484 if gs_working:
1485 gsResult = runCmd(genGSCmd(actual_ps_path))
1486 if (gsResult == 0):
1487 return (True)
1488 else:
1489 print("hp2ps output for " + name + "is not valid PostScript")
1490 else: return (True) # assume postscript is valid without ghostscript
1491 else:
1492 print("hp2ps did not generate PostScript for " + name)
1493 return (False)
1494 else:
1495 print("hp2ps error when processing heap profile for " + name)
1496 return(False)
1497
1498 def check_prof_ok(name, way):
1499 expected_prof_file = find_expected_file(name, 'prof.sample')
1500 expected_prof_path = in_testdir(expected_prof_file)
1501
1502 # Check actual prof file only if we have an expected prof file to
1503 # compare it with.
1504 if not os.path.exists(expected_prof_path):
1505 return True
1506
1507 actual_prof_file = add_suffix(name, 'prof')
1508 actual_prof_path = in_testdir(actual_prof_file)
1509
1510 if not os.path.exists(actual_prof_path):
1511 print(actual_prof_path + " does not exist")
1512 return(False)
1513
1514 if os.path.getsize(actual_prof_path) == 0:
1515 print(actual_prof_path + " is empty")
1516 return(False)
1517
1518 return compare_outputs(way, 'prof', normalise_prof,
1519 expected_prof_file, actual_prof_file,
1520 whitespace_normaliser=normalise_whitespace)
1521
1522 # Compare expected output to actual output, and optionally accept the
1523 # new output. Returns true if output matched or was accepted, false
1524 # otherwise. See Note [Output comparison] for the meaning of the
1525 # normaliser and whitespace_normaliser parameters.
1526 def compare_outputs(way, kind, normaliser, expected_file, actual_file,
1527 whitespace_normaliser=lambda x:x):
1528
1529 expected_path = in_srcdir(expected_file)
1530 actual_path = in_testdir(actual_file)
1531
1532 if os.path.exists(expected_path):
1533 expected_str = normaliser(read_no_crs(expected_path))
1534 # Create the .normalised file in the testdir, not in the srcdir.
1535 expected_normalised_file = add_suffix(expected_file, 'normalised')
1536 expected_normalised_path = in_testdir(expected_normalised_file)
1537 else:
1538 expected_str = ''
1539 expected_normalised_path = '/dev/null'
1540
1541 actual_raw = read_no_crs(actual_path)
1542 actual_str = normaliser(actual_raw)
1543
1544 # See Note [Output comparison].
1545 if whitespace_normaliser(expected_str) == whitespace_normaliser(actual_str):
1546 return 1
1547 else:
1548 if config.verbose >= 1 and _expect_pass(way):
1549 print('Actual ' + kind + ' output differs from expected:')
1550
1551 if expected_normalised_path != '/dev/null':
1552 write_file(expected_normalised_path, expected_str)
1553
1554 actual_normalised_path = add_suffix(actual_path, 'normalised')
1555 write_file(actual_normalised_path, actual_str)
1556
1557 if config.verbose >= 1 and _expect_pass(way):
1558 # See Note [Output comparison].
1559 r = runCmd('diff -uw "{0}" "{1}"'.format(expected_normalised_path,
1560 actual_normalised_path),
1561 print_output = 1)
1562
1563 # If for some reason there were no non-whitespace differences,
1564 # then do a full diff
1565 if r == 0:
1566 r = runCmd('diff -u "{0}" "{1}"'.format(expected_normalised_path,
1567 actual_normalised_path),
1568 print_output = 1)
1569
1570 if config.accept and (getTestOpts().expect == 'fail' or
1571 way in getTestOpts().expect_fail_for):
1572 if_verbose(1, 'Test is expected to fail. Not accepting new output.')
1573 return 0
1574 elif config.accept and actual_raw:
1575 if_verbose(1, 'Accepting new output.')
1576 write_file(expected_path, actual_raw)
1577 return 1
1578 elif config.accept:
1579 if_verbose(1, 'No output. Deleting "{0}".'.format(expected_path))
1580 os.remove(expected_path)
1581 return 1
1582 else:
1583 return 0
1584
1585 # Note [Output comparison]
1586 #
1587 # We do two types of output comparison:
1588 #
1589 # 1. To decide whether a test has failed. We apply a `normaliser` and an
1590 # optional `whitespace_normaliser` to the expected and the actual
1591 # output, before comparing the two.
1592 #
1593 # 2. To show as a diff to the user when the test indeed failed. We apply
1594 # the same `normaliser` function to the outputs, to make the diff as
1595 # small as possible (only showing the actual problem). But we don't
1596 # apply the `whitespace_normaliser` here, because it might completely
1597 # squash all whitespace, making the diff unreadable. Instead we rely
1598 # on the `diff` program to ignore whitespace changes as much as
1599 # possible (#10152).
1600
1601 def normalise_whitespace( str ):
1602 # Merge contiguous whitespace characters into a single space.
1603 return ' '.join(w for w in str.split())
1604
1605 callSite_re = re.compile(r', called at (.+):[\d]+:[\d]+ in [\w\-\.]+:')
1606
1607 def normalise_callstacks(s):
1608 opts = getTestOpts()
1609 def repl(matches):
1610 location = matches.group(1)
1611 location = normalise_slashes_(location)
1612 return ', called at {0}:<line>:<column> in <package-id>:'.format(location)
1613 # Ignore line number differences in call stacks (#10834).
1614 s = re.sub(callSite_re, repl, s)
1615 # Ignore the change in how we identify implicit call-stacks
1616 s = s.replace('from ImplicitParams', 'from HasCallStack')
1617 if not opts.keep_prof_callstacks:
1618 # Don't output prof callstacks. Test output should be
1619 # independent from the WAY we run the test.
1620 s = re.sub(r'CallStack \(from -prof\):(\n .*)*\n?', '', s)
1621 return s
1622
1623 tyCon_re = re.compile(r'TyCon\s*\d+L?\#\#\s*\d+L?\#\#\s*', flags=re.MULTILINE)
1624
1625 def normalise_type_reps(str):
1626 """ Normalise out fingerprints from Typeable TyCon representations """
1627 return re.sub(tyCon_re, 'TyCon FINGERPRINT FINGERPRINT ', str)
1628
1629 def normalise_errmsg( str ):
1630 """Normalise error-messages emitted via stderr"""
1631 # IBM AIX's `ld` is a bit chatty
1632 if opsys('aix'):
1633 str = str.replace('ld: 0706-027 The -x flag is ignored.\n', '')
1634 # remove " error:" and lower-case " Warning:" to make patch for
1635 # trac issue #10021 smaller
1636 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1637 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1638 str = normalise_callstacks(str)
1639 str = normalise_type_reps(str)
1640
1641 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1642 # the colon is there because it appears in error messages; this
1643 # hacky solution is used in place of more sophisticated filename
1644 # mangling
1645 str = re.sub('([^\\s])\\.exe', '\\1', str)
1646
1647 # normalise slashes, minimise Windows/Unix filename differences
1648 str = re.sub('\\\\', '/', str)
1649
1650 # The inplace ghc's are called ghc-stage[123] to avoid filename
1651 # collisions, so we need to normalise that to just "ghc"
1652 str = re.sub('ghc-stage[123]', 'ghc', str)
1653
1654 # Error messages simetimes contain integer implementation package
1655 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1656
1657 # Also filter out bullet characters. This is because bullets are used to
1658 # separate error sections, and tests shouldn't be sensitive to how the
1659 # the division happens.
1660 bullet = '•'.encode('utf8') if isinstance(str, bytes) else '•'
1661 str = str.replace(bullet, '')
1662
1663 # Windows only, this is a bug in hsc2hs but it is preventing
1664 # stable output for the testsuite. See Trac #9775. For now we filter out this
1665 # warning message to get clean output.
1666 if config.msys:
1667 str = re.sub('Failed to remove file (.*); error= (.*)$', '', str)
1668 str = re.sub('DeleteFile "(.+)": permission denied \(Access is denied\.\)(.*)$', '', str)
1669
1670 return str
1671
1672 # normalise a .prof file, so that we can reasonably compare it against
1673 # a sample. This doesn't compare any of the actual profiling data,
1674 # only the shape of the profile and the number of entries.
1675 def normalise_prof (str):
1676 # strip everything up to the line beginning "COST CENTRE"
1677 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1678
1679 # strip results for CAFs, these tend to change unpredictably
1680 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1681
1682 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1683 # sometimes under MAIN.
1684 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1685
1686 # We have something like this:
1687 #
1688 # MAIN MAIN <built-in> 53 0 0.0 0.2 0.0 100.0
1689 # CAF Main <entire-module> 105 0 0.0 0.3 0.0 62.5
1690 # readPrec Main Main_1.hs:7:13-16 109 1 0.0 0.6 0.0 0.6
1691 # readPrec Main Main_1.hs:4:13-16 107 1 0.0 0.6 0.0 0.6
1692 # main Main Main_1.hs:(10,1)-(20,20) 106 1 0.0 20.2 0.0 61.0
1693 # == Main Main_1.hs:7:25-26 114 1 0.0 0.0 0.0 0.0
1694 # == Main Main_1.hs:4:25-26 113 1 0.0 0.0 0.0 0.0
1695 # showsPrec Main Main_1.hs:7:19-22 112 2 0.0 1.2 0.0 1.2
1696 # showsPrec Main Main_1.hs:4:19-22 111 2 0.0 0.9 0.0 0.9
1697 # readPrec Main Main_1.hs:7:13-16 110 0 0.0 18.8 0.0 18.8
1698 # readPrec Main Main_1.hs:4:13-16 108 0 0.0 19.9 0.0 19.9
1699 #
1700 # then we remove all the specific profiling data, leaving only the cost
1701 # centre name, module, src, and entries, to end up with this: (modulo
1702 # whitespace between columns)
1703 #
1704 # MAIN MAIN <built-in> 0
1705 # readPrec Main Main_1.hs:7:13-16 1
1706 # readPrec Main Main_1.hs:4:13-16 1
1707 # == Main Main_1.hs:7:25-26 1
1708 # == Main Main_1.hs:4:25-26 1
1709 # showsPrec Main Main_1.hs:7:19-22 2
1710 # showsPrec Main Main_1.hs:4:19-22 2
1711 # readPrec Main Main_1.hs:7:13-16 0
1712 # readPrec Main Main_1.hs:4:13-16 0
1713
1714 # Split 9 whitespace-separated groups, take columns 1 (cost-centre), 2
1715 # (module), 3 (src), and 5 (entries). SCC names can't have whitespace, so
1716 # this works fine.
1717 str = re.sub(r'\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*',
1718 '\\1 \\2 \\3 \\5\n', str)
1719 return str
1720
1721 def normalise_slashes_( str ):
1722 str = re.sub('\\\\', '/', str)
1723 return str
1724
1725 def normalise_exe_( str ):
1726 str = re.sub('\.exe', '', str)
1727 return str
1728
1729 def normalise_output( str ):
1730 # remove " error:" and lower-case " Warning:" to make patch for
1731 # trac issue #10021 smaller
1732 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1733 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1734 # Remove a .exe extension (for Windows)
1735 # This can occur in error messages generated by the program.
1736 str = re.sub('([^\\s])\\.exe', '\\1', str)
1737 str = normalise_callstacks(str)
1738 str = normalise_type_reps(str)
1739 return str
1740
1741 def normalise_asm( str ):
1742 lines = str.split('\n')
1743 # Only keep instructions and labels not starting with a dot.
1744 metadata = re.compile('^[ \t]*\\..*$')
1745 out = []
1746 for line in lines:
1747 # Drop metadata directives (e.g. ".type")
1748 if not metadata.match(line):
1749 line = re.sub('@plt', '', line)
1750 instr = line.lstrip().split()
1751 # Drop empty lines.
1752 if not instr:
1753 continue
1754 # Drop operands, except for call instructions.
1755 elif instr[0] == 'call':
1756 out.append(instr[0] + ' ' + instr[1])
1757 else:
1758 out.append(instr[0])
1759 out = '\n'.join(out)
1760 return out
1761
1762 def if_verbose( n, s ):
1763 if config.verbose >= n:
1764 print(s)
1765
1766 def if_verbose_dump( n, f ):
1767 if config.verbose >= n:
1768 try:
1769 with io.open(f) as file:
1770 print(file.read())
1771 except Exception:
1772 print('')
1773
1774 def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, print_output=0):
1775 timeout_prog = strip_quotes(config.timeout_prog)
1776 timeout = str(int(ceil(config.timeout * timeout_multiplier)))
1777
1778 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1779 cmd = cmd.format(**config.__dict__)
1780 if_verbose(3, cmd + ('< ' + os.path.basename(stdin) if stdin else ''))
1781
1782 # declare the buffers to a default
1783 stdin_buffer = None
1784
1785 # ***** IMPORTANT *****
1786 # We have to treat input and output as
1787 # just binary data here. Don't try to decode
1788 # it to a string, since we have tests that actually
1789 # feed malformed utf-8 to see how GHC handles it.
1790 if stdin:
1791 with io.open(stdin, 'rb') as f:
1792 stdin_buffer = f.read()
1793
1794 stdout_buffer = b''
1795 stderr_buffer = b''
1796
1797 hStdErr = subprocess.PIPE
1798 if stderr is subprocess.STDOUT:
1799 hStdErr = subprocess.STDOUT
1800
1801 try:
1802 # cmd is a complex command in Bourne-shell syntax
1803 # e.g (cd . && 'C:/users/simonpj/HEAD/inplace/bin/ghc-stage2' ...etc)
1804 # Hence it must ultimately be run by a Bourne shell. It's timeout's job
1805 # to invoke the Bourne shell
1806
1807 r = subprocess.Popen([timeout_prog, timeout, cmd],
1808 stdin=subprocess.PIPE,
1809 stdout=subprocess.PIPE,
1810 stderr=hStdErr)
1811
1812 stdout_buffer, stderr_buffer = r.communicate(stdin_buffer)
1813 finally:
1814 if config.verbose >= 1 and print_output >= 1:
1815 if stdout_buffer:
1816 sys.stdout.buffer.write(stdout_buffer)
1817 if stderr_buffer:
1818 sys.stderr.buffer.write(stderr_buffer)
1819
1820 if stdout:
1821 with io.open(stdout, 'wb') as f:
1822 f.write(stdout_buffer)
1823 if stderr:
1824 if stderr is not subprocess.STDOUT:
1825 with io.open(stderr, 'wb') as f:
1826 f.write(stderr_buffer)
1827
1828 if r.returncode == 98:
1829 # The python timeout program uses 98 to signal that ^C was pressed
1830 stopNow()
1831 if r.returncode == 99 and getTestOpts().exit_code != 99:
1832 # Only print a message when timeout killed the process unexpectedly.
1833 if_verbose(1, 'Timeout happened...killed process "{0}"...\n'.format(cmd))
1834 return r.returncode
1835
1836 # -----------------------------------------------------------------------------
1837 # checking if ghostscript is available for checking the output of hp2ps
1838
1839 def genGSCmd(psfile):
1840 return '{{gs}} -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE "{0}"'.format(psfile)
1841
1842 def gsNotWorking():
1843 global gs_working
1844 print("GhostScript not available for hp2ps tests")
1845
1846 global gs_working
1847 gs_working = 0
1848 if config.have_profiling:
1849 if config.gs != '':
1850 resultGood = runCmd(genGSCmd(config.confdir + '/good.ps'));
1851 if resultGood == 0:
1852 resultBad = runCmd(genGSCmd(config.confdir + '/bad.ps') +
1853 ' >/dev/null 2>&1')
1854 if resultBad != 0:
1855 print("GhostScript available for hp2ps tests")
1856 gs_working = 1;
1857 else:
1858 gsNotWorking();
1859 else:
1860 gsNotWorking();
1861 else:
1862 gsNotWorking();
1863
1864 def add_suffix( name, suffix ):
1865 if suffix == '':
1866 return name
1867 else:
1868 return name + '.' + suffix
1869
1870 def add_hs_lhs_suffix(name):
1871 if getTestOpts().c_src:
1872 return add_suffix(name, 'c')
1873 elif getTestOpts().cmm_src:
1874 return add_suffix(name, 'cmm')
1875 elif getTestOpts().objc_src:
1876 return add_suffix(name, 'm')
1877 elif getTestOpts().objcpp_src:
1878 return add_suffix(name, 'mm')
1879 elif getTestOpts().literate:
1880 return add_suffix(name, 'lhs')
1881 else:
1882 return add_suffix(name, 'hs')
1883
1884 def replace_suffix( name, suffix ):
1885 base, suf = os.path.splitext(name)
1886 return base + '.' + suffix
1887
1888 def in_testdir(name, suffix=''):
1889 return os.path.join(getTestOpts().testdir, add_suffix(name, suffix))
1890
1891 def in_srcdir(name, suffix=''):
1892 return os.path.join(getTestOpts().srcdir, add_suffix(name, suffix))
1893
1894 # Finding the sample output. The filename is of the form
1895 #
1896 # <test>.stdout[-ws-<wordsize>][-<platform>]
1897 #
1898 def find_expected_file(name, suff):
1899 basename = add_suffix(name, suff)
1900
1901 files = [basename + ws + plat
1902 for plat in ['-' + config.platform, '-' + config.os, '']
1903 for ws in ['-ws-' + config.wordsize, '']]
1904
1905 for f in files:
1906 if os.path.exists(in_srcdir(f)):
1907 return f
1908
1909 return basename
1910
1911 if config.msys:
1912 import stat
1913 import time
1914 def cleanup():
1915 testdir = getTestOpts().testdir
1916 max_attempts = 5
1917 retries = max_attempts
1918 def on_error(function, path, excinfo):
1919 # At least one test (T11489) removes the write bit from a file it
1920 # produces. Windows refuses to delete read-only files with a
1921 # permission error. Try setting the write bit and try again.
1922 os.chmod(path, stat.S_IWRITE)
1923 function(path)
1924
1925 # On Windows we have to retry the delete a couple of times.
1926 # The reason for this is that a FileDelete command just marks a
1927 # file for deletion. The file is really only removed when the last
1928 # handle to the file is closed. Unfortunately there are a lot of
1929 # system services that can have a file temporarily opened using a shared
1930 # readonly lock, such as the built in AV and search indexer.
1931 #
1932 # We can't really guarantee that these are all off, so what we can do is
1933 # whenever after a rmtree the folder still exists to try again and wait a bit.
1934 #
1935 # Based on what I've seen from the tests on CI server, is that this is relatively rare.
1936 # So overall we won't be retrying a lot. If after a reasonable amount of time the folder is
1937 # still locked then abort the current test by throwing an exception, this so it won't fail
1938 # with an even more cryptic error.
1939 #
1940 # See Trac #13162
1941 exception = None
1942 while retries > 0 and os.path.exists(testdir):
1943 time.sleep((max_attempts-retries)*6)
1944 try:
1945 shutil.rmtree(testdir, onerror=on_error, ignore_errors=False)
1946 except Exception as e:
1947 exception = e
1948 retries -= 1
1949
1950 if retries == 0 and os.path.exists(testdir):
1951 raise Exception("Unable to remove folder '%s': %s\nUnable to start current test."
1952 % (testdir, exception))
1953 else:
1954 def cleanup():
1955 testdir = getTestOpts().testdir
1956 if os.path.exists(testdir):
1957 shutil.rmtree(testdir, ignore_errors=False)
1958
1959
1960 # -----------------------------------------------------------------------------
1961 # Return a list of all the files ending in '.T' below directories roots.
1962
1963 def findTFiles(roots):
1964 for root in roots:
1965 for path, dirs, files in os.walk(root, topdown=True):
1966 # Never pick up .T files in uncleaned .run directories.
1967 dirs[:] = [dir for dir in sorted(dirs)
1968 if not dir.endswith(testdir_suffix)]
1969 for filename in files:
1970 if filename.endswith('.T'):
1971 yield os.path.join(path, filename)
1972
1973 # -----------------------------------------------------------------------------
1974 # Output a test summary to the specified file object
1975
1976 def summary(t, file, short=False):
1977
1978 file.write('\n')
1979 printUnexpectedTests(file,
1980 [t.unexpected_passes, t.unexpected_failures,
1981 t.unexpected_stat_failures, t.framework_failures])
1982
1983 if short:
1984 # Only print the list of unexpected tests above.
1985 return
1986
1987 file.write('SUMMARY for test run started at '
1988 + time.strftime("%c %Z", t.start_time) + '\n'
1989 + str(datetime.timedelta(seconds=
1990 round(time.time() - time.mktime(t.start_time)))).rjust(8)
1991 + ' spent to go through\n'
1992 + repr(t.total_tests).rjust(8)
1993 + ' total tests, which gave rise to\n'
1994 + repr(t.total_test_cases).rjust(8)
1995 + ' test cases, of which\n'
1996 + repr(t.n_tests_skipped).rjust(8)
1997 + ' were skipped\n'
1998 + '\n'
1999 + repr(len(t.missing_libs)).rjust(8)
2000 + ' had missing libraries\n'
2001 + repr(t.n_expected_passes).rjust(8)
2002 + ' expected passes\n'
2003 + repr(t.n_expected_failures).rjust(8)
2004 + ' expected failures\n'
2005 + '\n'
2006 + repr(len(t.framework_failures)).rjust(8)
2007 + ' caused framework failures\n'
2008 + repr(len(t.framework_warnings)).rjust(8)
2009 + ' caused framework warnings\n'
2010 + repr(len(t.unexpected_passes)).rjust(8)
2011 + ' unexpected passes\n'
2012 + repr(len(t.unexpected_failures)).rjust(8)
2013 + ' unexpected failures\n'
2014 + repr(len(t.unexpected_stat_failures)).rjust(8)
2015 + ' unexpected stat failures\n'
2016 + '\n')
2017
2018 if t.unexpected_passes:
2019 file.write('Unexpected passes:\n')
2020 printTestInfosSummary(file, t.unexpected_passes)
2021
2022 if t.unexpected_failures:
2023 file.write('Unexpected failures:\n')
2024 printTestInfosSummary(file, t.unexpected_failures)
2025
2026 if t.unexpected_stat_failures:
2027 file.write('Unexpected stat failures:\n')
2028 printTestInfosSummary(file, t.unexpected_stat_failures)
2029
2030 if t.framework_failures:
2031 file.write('Framework failures:\n')
2032 printTestInfosSummary(file, t.framework_failures)
2033
2034 if t.framework_warnings:
2035 file.write('Framework warnings:\n')
2036 printTestInfosSummary(file, t.framework_warnings)
2037
2038 if stopping():
2039 file.write('WARNING: Testsuite run was terminated early\n')
2040
2041 def printUnexpectedTests(file, testInfoss):
2042 unexpected = set(name for testInfos in testInfoss
2043 for (_, name, _, _) in testInfos
2044 if not name.endswith('.T'))
2045 if unexpected:
2046 file.write('Unexpected results from:\n')
2047 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2048 file.write('\n')
2049
2050 def printTestInfosSummary(file, testInfos):
2051 maxDirLen = max(len(directory) for (directory, _, _, _) in testInfos)
2052 for (directory, name, reason, way) in testInfos:
2053 directory = directory.ljust(maxDirLen)
2054 file.write(' {directory} {name} [{reason}] ({way})\n'.format(**locals()))
2055 file.write('\n')
2056
2057 def modify_lines(s, f):
2058 s = '\n'.join([f(l) for l in s.splitlines()])
2059 if s and s[-1] != '\n':
2060 # Prevent '\ No newline at end of file' warnings when diffing.
2061 s += '\n'
2062 return s