Enabled warn on tabs by default (fixes #9230)
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 from __future__ import print_function
6
7 import shutil
8 import sys
9 import os
10 import errno
11 import string
12 import re
13 import traceback
14 import time
15 import datetime
16 import copy
17 import glob
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print("Warning: subprocess not found, will fall back to spawnv")
26
27 from testglobals import *
28 from testutil import *
29
30 if config.use_threads:
31 import threading
32 try:
33 import thread
34 except ImportError: # Python 3
35 import _thread as thread
36
37 global wantToStop
38 wantToStop = False
39 def stopNow():
40 global wantToStop
41 wantToStop = True
42 def stopping():
43 return wantToStop
44
45 # Options valid for the current test only (these get reset to
46 # testdir_testopts after each test).
47
48 global testopts_local
49 if config.use_threads:
50 testopts_local = threading.local()
51 else:
52 class TestOpts_Local:
53 pass
54 testopts_local = TestOpts_Local()
55
56 def getTestOpts():
57 return testopts_local.x
58
59 def setLocalTestOpts(opts):
60 global testopts_local
61 testopts_local.x=opts
62
63 def isStatsTest():
64 opts = getTestOpts()
65 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
66
67
68 # This can be called at the top of a file of tests, to set default test options
69 # for the following tests.
70 def setTestOpts( f ):
71 global thisdir_settings
72 thisdir_settings = [thisdir_settings, f]
73
74 # -----------------------------------------------------------------------------
75 # Canned setup functions for common cases. eg. for a test you might say
76 #
77 # test('test001', normal, compile, [''])
78 #
79 # to run it without any options, but change it to
80 #
81 # test('test001', expect_fail, compile, [''])
82 #
83 # to expect failure for this test.
84
85 def normal( name, opts ):
86 return;
87
88 def skip( name, opts ):
89 opts.skip = 1
90
91 def expect_fail( name, opts ):
92 opts.expect = 'fail';
93
94 def reqlib( lib ):
95 return lambda name, opts, l=lib: _reqlib (name, opts, l )
96
97 # Cache the results of looking to see if we have a library or not.
98 # This makes quite a difference, especially on Windows.
99 have_lib = {}
100
101 def _reqlib( name, opts, lib ):
102 if lib in have_lib:
103 got_it = have_lib[lib]
104 else:
105 if have_subprocess:
106 # By preference we use subprocess, as the alternative uses
107 # /dev/null which mingw doesn't have.
108 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 stdout=subprocess.PIPE,
110 stderr=subprocess.PIPE)
111 # read from stdout and stderr to avoid blocking due to
112 # buffers filling
113 p.communicate()
114 r = p.wait()
115 else:
116 r = os.system(config.ghc_pkg + ' describe ' + lib
117 + ' > /dev/null 2> /dev/null')
118 got_it = r == 0
119 have_lib[lib] = got_it
120
121 if not got_it:
122 opts.expect = 'missing-lib'
123
124 def req_profiling( name, opts ):
125 if not config.have_profiling:
126 opts.expect = 'fail'
127
128 def req_shared_libs( name, opts ):
129 if not config.have_shared_libs:
130 opts.expect = 'fail'
131
132 def req_interp( name, opts ):
133 if not config.have_interp:
134 opts.expect = 'fail'
135
136 def req_smp( name, opts ):
137 if not config.have_smp:
138 opts.expect = 'fail'
139
140 def ignore_output( name, opts ):
141 opts.ignore_output = 1
142
143 def no_stdin( name, opts ):
144 opts.no_stdin = 1
145
146 def combined_output( name, opts ):
147 opts.combined_output = True
148
149 # -----
150
151 def expect_fail_for( ways ):
152 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153
154 def _expect_fail_for( name, opts, ways ):
155 opts.expect_fail_for = ways
156
157 def expect_broken( bug ):
158 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
159
160 def _expect_broken( name, opts, bug ):
161 record_broken(name, opts, bug)
162 opts.expect = 'fail';
163
164 def expect_broken_for( bug, ways ):
165 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
166
167 def _expect_broken_for( name, opts, bug, ways ):
168 record_broken(name, opts, bug)
169 opts.expect_fail_for = ways
170
171 def record_broken(name, opts, bug):
172 global brokens
173 me = (bug, opts.testdir, name)
174 if not me in brokens:
175 brokens.append(me)
176
177 # -----
178
179 def omit_ways( ways ):
180 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181
182 def _omit_ways( name, opts, ways ):
183 opts.omit_ways = ways
184
185 # -----
186
187 def only_ways( ways ):
188 return lambda name, opts, w=ways: _only_ways( name, opts, w )
189
190 def _only_ways( name, opts, ways ):
191 opts.only_ways = ways
192
193 # -----
194
195 def extra_ways( ways ):
196 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197
198 def _extra_ways( name, opts, ways ):
199 opts.extra_ways = ways
200
201 # -----
202
203 def omit_compiler_types( compiler_types ):
204 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
205
206 def _omit_compiler_types( name, opts, compiler_types ):
207 if config.compiler_type in compiler_types:
208 opts.skip = 1
209
210 # -----
211
212 def only_compiler_types( compiler_types ):
213 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
214
215 def _only_compiler_types( name, opts, compiler_types ):
216 if config.compiler_type not in compiler_types:
217 opts.skip = 1
218
219 # -----
220
221 def set_stdin( file ):
222 return lambda name, opts, f=file: _set_stdin(name, opts, f);
223
224 def _set_stdin( name, opts, f ):
225 opts.stdin = f
226
227 # -----
228
229 def exit_code( val ):
230 return lambda name, opts, v=val: _exit_code(name, opts, v);
231
232 def _exit_code( name, opts, v ):
233 opts.exit_code = v
234
235 def signal_exit_code( val ):
236 if opsys('solaris2'):
237 return exit_code( val );
238 else:
239 # When application running on Linux receives fatal error
240 # signal, then its exit code is encoded as 128 + signal
241 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
242 # I assume that Mac OS X behaves in the same way at least Mac
243 # OS X builder behavior suggests this.
244 return exit_code( val+128 );
245
246 # -----
247
248 def timeout_multiplier( val ):
249 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
250
251 def _timeout_multiplier( name, opts, v ):
252 opts.timeout_multiplier = v
253
254 # -----
255
256 def extra_run_opts( val ):
257 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
258
259 def _extra_run_opts( name, opts, v ):
260 opts.extra_run_opts = v
261
262 # -----
263
264 def extra_hc_opts( val ):
265 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
266
267 def _extra_hc_opts( name, opts, v ):
268 opts.extra_hc_opts = v
269
270 # -----
271
272 def extra_clean( files ):
273 return lambda name, opts, v=files: _extra_clean(name, opts, v);
274
275 def _extra_clean( name, opts, v ):
276 opts.clean_files = v
277
278 # -----
279
280 def stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
282
283 def _stats_num_field( name, opts, field, expecteds ):
284 if field in opts.stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 if type(expecteds) is list:
288 for (b, expected, dev) in expecteds:
289 if b:
290 opts.stats_range_fields[field] = (expected, dev)
291 return
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 else:
295 (expected, dev) = expecteds
296 opts.stats_range_fields[field] = (expected, dev)
297
298 def compiler_stats_num_field( field, expecteds ):
299 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
300
301 def _compiler_stats_num_field( name, opts, field, expecteds ):
302 if field in opts.compiler_stats_range_fields:
303 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
304
305 # Compiler performance numbers change when debugging is on, making the results
306 # useless and confusing. Therefore, skip if debugging is on.
307 if compiler_debugged():
308 skip(name, opts)
309
310 for (b, expected, dev) in expecteds:
311 if b:
312 opts.compiler_stats_range_fields[field] = (expected, dev)
313 return
314
315 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
316
317 # -----
318
319 def when(b, f):
320 # When list_brokens is on, we want to see all expect_broken calls,
321 # so we always do f
322 if b or config.list_broken:
323 return f
324 else:
325 return normal
326
327 def unless(b, f):
328 return when(not b, f)
329
330 def doing_ghci():
331 return 'ghci' in config.run_ways
332
333 def ghci_dynamic( ):
334 return config.ghc_dynamic
335
336 def fast():
337 return config.fast
338
339 def platform( plat ):
340 return config.platform == plat
341
342 def opsys( os ):
343 return config.os == os
344
345 def arch( arch ):
346 return config.arch == arch
347
348 def wordsize( ws ):
349 return config.wordsize == str(ws)
350
351 def msys( ):
352 return config.msys
353
354 def cygwin( ):
355 return config.cygwin
356
357 def have_vanilla( ):
358 return config.have_vanilla
359
360 def have_dynamic( ):
361 return config.have_dynamic
362
363 def have_profiling( ):
364 return config.have_profiling
365
366 def in_tree_compiler( ):
367 return config.in_tree_compiler
368
369 def compiler_type( compiler ):
370 return config.compiler_type == compiler
371
372 def compiler_lt( compiler, version ):
373 return config.compiler_type == compiler and \
374 version_lt(config.compiler_version, version)
375
376 def compiler_le( compiler, version ):
377 return config.compiler_type == compiler and \
378 version_le(config.compiler_version, version)
379
380 def compiler_gt( compiler, version ):
381 return config.compiler_type == compiler and \
382 version_gt(config.compiler_version, version)
383
384 def compiler_ge( compiler, version ):
385 return config.compiler_type == compiler and \
386 version_ge(config.compiler_version, version)
387
388 def unregisterised( ):
389 return config.unregisterised
390
391 def compiler_profiled( ):
392 return config.compiler_profiled
393
394 def compiler_debugged( ):
395 return config.compiler_debugged
396
397 def tag( t ):
398 return t in config.compiler_tags
399
400 # ---
401
402 def namebase( nb ):
403 return lambda opts, nb=nb: _namebase(opts, nb)
404
405 def _namebase( opts, nb ):
406 opts.with_namebase = nb
407
408 # ---
409
410 def high_memory_usage(name, opts):
411 opts.alone = True
412
413 # If a test is for a multi-CPU race, then running the test alone
414 # increases the chance that we'll actually see it.
415 def multi_cpu_race(name, opts):
416 opts.alone = True
417
418 # ---
419 def literate( name, opts ):
420 opts.literate = 1;
421
422 def c_src( name, opts ):
423 opts.c_src = 1;
424
425 def objc_src( name, opts ):
426 opts.objc_src = 1;
427
428 def objcpp_src( name, opts ):
429 opts.objcpp_src = 1;
430
431 def cmm_src( name, opts ):
432 opts.cmm_src = 1;
433
434 def outputdir( odir ):
435 return lambda name, opts, d=odir: _outputdir(name, opts, d)
436
437 def _outputdir( name, opts, odir ):
438 opts.outputdir = odir;
439
440 # ----
441
442 def pre_cmd( cmd ):
443 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
444
445 def _pre_cmd( name, opts, cmd ):
446 opts.pre_cmd = cmd
447
448 # ----
449
450 def clean_cmd( cmd ):
451 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
452
453 def _clean_cmd( name, opts, cmd ):
454 opts.clean_cmd = cmd
455
456 # ----
457
458 def cmd_prefix( prefix ):
459 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
460
461 def _cmd_prefix( name, opts, prefix ):
462 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
463
464 # ----
465
466 def cmd_wrapper( fun ):
467 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
468
469 def _cmd_wrapper( name, opts, fun ):
470 opts.cmd_wrapper = fun
471
472 # ----
473
474 def compile_cmd_prefix( prefix ):
475 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
476
477 def _compile_cmd_prefix( name, opts, prefix ):
478 opts.compile_cmd_prefix = prefix
479
480 # ----
481
482 def check_stdout( f ):
483 return lambda name, opts, f=f: _check_stdout(name, opts, f)
484
485 def _check_stdout( name, opts, f ):
486 opts.check_stdout = f
487
488 # ----
489
490 def normalise_slashes( name, opts ):
491 opts.extra_normaliser = normalise_slashes_
492
493 def normalise_exe( name, opts ):
494 opts.extra_normaliser = normalise_exe_
495
496 def normalise_fun( fun ):
497 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
498
499 def _normalise_fun( name, opts, f ):
500 opts.extra_normaliser = f
501
502 def normalise_errmsg_fun( fun ):
503 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
504
505 def _normalise_errmsg_fun( name, opts, f ):
506 opts.extra_errmsg_normaliser = f
507
508 def two_normalisers(f, g):
509 return lambda x, f=f, g=g: f(g(x))
510
511 # ----
512 # Function for composing two opt-fns together
513
514 def executeSetups(fs, name, opts):
515 if type(fs) is list:
516 # If we have a list of setups, then execute each one
517 for f in fs:
518 executeSetups(f, name, opts)
519 else:
520 # fs is a single function, so just apply it
521 fs(name, opts)
522
523 # -----------------------------------------------------------------------------
524 # The current directory of tests
525
526 def newTestDir( dir ):
527 global thisdir_settings
528 # reset the options for this test directory
529 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
530
531 def _newTestDir( name, opts, dir ):
532 opts.testdir = dir
533 opts.compiler_always_flags = config.compiler_always_flags
534
535 # -----------------------------------------------------------------------------
536 # Actually doing tests
537
538 parallelTests = []
539 aloneTests = []
540 allTestNames = set([])
541
542 def runTest (opts, name, func, args):
543 ok = 0
544
545 if config.use_threads:
546 t.thread_pool.acquire()
547 try:
548 while config.threads<(t.running_threads+1):
549 t.thread_pool.wait()
550 t.running_threads = t.running_threads+1
551 ok=1
552 t.thread_pool.release()
553 thread.start_new_thread(test_common_thread, (name, opts, func, args))
554 except:
555 if not ok:
556 t.thread_pool.release()
557 else:
558 test_common_work (name, opts, func, args)
559
560 # name :: String
561 # setup :: TestOpts -> IO ()
562 def test (name, setup, func, args):
563 global aloneTests
564 global parallelTests
565 global allTestNames
566 global thisdir_settings
567 if name in allTestNames:
568 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
569 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
570 framework_fail(name, 'bad_name', 'This test has an invalid name')
571
572 # Make a deep copy of the default_testopts, as we need our own copy
573 # of any dictionaries etc inside it. Otherwise, if one test modifies
574 # them, all tests will see the modified version!
575 myTestOpts = copy.deepcopy(default_testopts)
576
577 executeSetups([thisdir_settings, setup], name, myTestOpts)
578
579 thisTest = lambda : runTest(myTestOpts, name, func, args)
580 if myTestOpts.alone:
581 aloneTests.append(thisTest)
582 else:
583 parallelTests.append(thisTest)
584 allTestNames.add(name)
585
586 if config.use_threads:
587 def test_common_thread(name, opts, func, args):
588 t.lock.acquire()
589 try:
590 test_common_work(name,opts,func,args)
591 finally:
592 t.lock.release()
593 t.thread_pool.acquire()
594 t.running_threads = t.running_threads - 1
595 t.thread_pool.notify()
596 t.thread_pool.release()
597
598 def get_package_cache_timestamp():
599 if config.package_conf_cache_file == '':
600 return 0.0
601 else:
602 try:
603 return os.stat(config.package_conf_cache_file).st_mtime
604 except:
605 return 0.0
606
607
608 def test_common_work (name, opts, func, args):
609 try:
610 t.total_tests = t.total_tests+1
611 setLocalTestOpts(opts)
612
613 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
614
615 # All the ways we might run this test
616 if func == compile or func == multimod_compile:
617 all_ways = config.compile_ways
618 elif func == compile_and_run or func == multimod_compile_and_run:
619 all_ways = config.run_ways
620 elif func == ghci_script:
621 if 'ghci' in config.run_ways:
622 all_ways = ['ghci']
623 else:
624 all_ways = []
625 else:
626 all_ways = ['normal']
627
628 # A test itself can request extra ways by setting opts.extra_ways
629 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
630
631 t.total_test_cases = t.total_test_cases + len(all_ways)
632
633 ok_way = lambda way: \
634 not getTestOpts().skip \
635 and (config.only == [] or name in config.only) \
636 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
637 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
638 and (not (config.skip_perf_tests and isStatsTest())) \
639 and way not in getTestOpts().omit_ways
640
641 # Which ways we are asked to skip
642 do_ways = list(filter (ok_way,all_ways))
643
644 # In fast mode, we skip all but one way
645 if config.fast and len(do_ways) > 0:
646 do_ways = [do_ways[0]]
647
648 if not config.clean_only:
649 # Run the required tests...
650 for way in do_ways:
651 if stopping():
652 break
653 do_test (name, way, func, args)
654
655 for way in all_ways:
656 if way not in do_ways:
657 skiptest (name,way)
658
659 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
660 pretest_cleanup(name)
661 clean([name + suff for suff in [
662 '', '.exe', '.exe.manifest', '.genscript',
663 '.stderr.normalised', '.stdout.normalised',
664 '.run.stderr.normalised', '.run.stdout.normalised',
665 '.comp.stderr.normalised', '.comp.stdout.normalised',
666 '.interp.stderr.normalised', '.interp.stdout.normalised',
667 '.stats', '.comp.stats',
668 '.hi', '.o', '.prof', '.exe.prof', '.hc',
669 '_stub.h', '_stub.c', '_stub.o',
670 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
671
672 if func == multi_compile or func == multi_compile_fail:
673 extra_mods = args[1]
674 clean([replace_suffix(fx[0],'o') for fx in extra_mods])
675 clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])
676
677
678 clean(getTestOpts().clean_files)
679
680 if getTestOpts().outputdir != None:
681 odir = in_testdir(getTestOpts().outputdir)
682 try:
683 shutil.rmtree(odir)
684 except:
685 pass
686
687 try:
688 shutil.rmtree(in_testdir('.hpc.' + name))
689 except:
690 pass
691
692 try:
693 cleanCmd = getTestOpts().clean_cmd
694 if cleanCmd != None:
695 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
696 if result != 0:
697 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
698 except:
699 framework_fail(name, 'cleaning', 'clean-command exception')
700
701 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
702
703 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
704 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
705
706 try:
707 for f in files_written[name]:
708 if os.path.exists(f):
709 try:
710 if not f in files_written_not_removed[name]:
711 files_written_not_removed[name].append(f)
712 except:
713 files_written_not_removed[name] = [f]
714 except:
715 pass
716 except Exception as e:
717 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
718
719 def clean(strs):
720 for str in strs:
721 for name in glob.glob(in_testdir(str)):
722 clean_full_path(name)
723
724 def clean_full_path(name):
725 try:
726 # Remove files...
727 os.remove(name)
728 except OSError as e1:
729 try:
730 # ... and empty directories
731 os.rmdir(name)
732 except OSError as e2:
733 # We don't want to fail here, but we do want to know
734 # what went wrong, so print out the exceptions.
735 # ENOENT isn't a problem, though, as we clean files
736 # that don't necessarily exist.
737 if e1.errno != errno.ENOENT:
738 print(e1)
739 if e2.errno != errno.ENOENT:
740 print(e2)
741
742 def do_test(name, way, func, args):
743 full_name = name + '(' + way + ')'
744
745 try:
746 if_verbose(2, "=====> %s %d of %d %s " % \
747 (full_name, t.total_tests, len(allTestNames), \
748 [t.n_unexpected_passes, \
749 t.n_unexpected_failures, \
750 t.n_framework_failures]))
751
752 if config.use_threads:
753 t.lock.release()
754
755 try:
756 preCmd = getTestOpts().pre_cmd
757 if preCmd != None:
758 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
759 if result != 0:
760 framework_fail(name, way, 'pre-command failed: ' + str(result))
761 except:
762 framework_fail(name, way, 'pre-command exception')
763
764 try:
765 result = func(*[name,way] + args)
766 finally:
767 if config.use_threads:
768 t.lock.acquire()
769
770 if getTestOpts().expect != 'pass' and \
771 getTestOpts().expect != 'fail' and \
772 getTestOpts().expect != 'missing-lib':
773 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
774
775 try:
776 passFail = result['passFail']
777 except:
778 passFail = 'No passFail found'
779
780 if passFail == 'pass':
781 if getTestOpts().expect == 'pass' \
782 and way not in getTestOpts().expect_fail_for:
783 t.n_expected_passes = t.n_expected_passes + 1
784 if name in t.expected_passes:
785 t.expected_passes[name].append(way)
786 else:
787 t.expected_passes[name] = [way]
788 else:
789 if_verbose(1, '*** unexpected pass for %s' % full_name)
790 t.n_unexpected_passes = t.n_unexpected_passes + 1
791 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
792 elif passFail == 'fail':
793 if getTestOpts().expect == 'pass' \
794 and way not in getTestOpts().expect_fail_for:
795 if_verbose(1, '*** unexpected failure for %s' % full_name)
796 t.n_unexpected_failures = t.n_unexpected_failures + 1
797 reason = result['reason']
798 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
799 else:
800 if getTestOpts().expect == 'missing-lib':
801 t.n_missing_libs = t.n_missing_libs + 1
802 if name in t.missing_libs:
803 t.missing_libs[name].append(way)
804 else:
805 t.missing_libs[name] = [way]
806 else:
807 t.n_expected_failures = t.n_expected_failures + 1
808 if name in t.expected_failures:
809 t.expected_failures[name].append(way)
810 else:
811 t.expected_failures[name] = [way]
812 else:
813 framework_fail(name, way, 'bad result ' + passFail)
814 except KeyboardInterrupt:
815 stopNow()
816 except:
817 framework_fail(name, way, 'do_test exception')
818 traceback.print_exc()
819
820 def addPassingTestInfo (testInfos, directory, name, way):
821 directory = re.sub('^\\.[/\\\\]', '', directory)
822
823 if not directory in testInfos:
824 testInfos[directory] = {}
825
826 if not name in testInfos[directory]:
827 testInfos[directory][name] = []
828
829 testInfos[directory][name].append(way)
830
831 def addFailingTestInfo (testInfos, directory, name, reason, way):
832 directory = re.sub('^\\.[/\\\\]', '', directory)
833
834 if not directory in testInfos:
835 testInfos[directory] = {}
836
837 if not name in testInfos[directory]:
838 testInfos[directory][name] = {}
839
840 if not reason in testInfos[directory][name]:
841 testInfos[directory][name][reason] = []
842
843 testInfos[directory][name][reason].append(way)
844
845 def skiptest (name, way):
846 # print 'Skipping test \"', name, '\"'
847 t.n_tests_skipped = t.n_tests_skipped + 1
848 if name in t.tests_skipped:
849 t.tests_skipped[name].append(way)
850 else:
851 t.tests_skipped[name] = [way]
852
853 def framework_fail( name, way, reason ):
854 full_name = name + '(' + way + ')'
855 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
856 t.n_framework_failures = t.n_framework_failures + 1
857 if name in t.framework_failures:
858 t.framework_failures[name].append(way)
859 else:
860 t.framework_failures[name] = [way]
861
862 def badResult(result):
863 try:
864 if result['passFail'] == 'pass':
865 return False
866 return True
867 except:
868 return True
869
870 def passed():
871 return {'passFail': 'pass'}
872
873 def failBecause(reason):
874 return {'passFail': 'fail', 'reason': reason}
875
876 # -----------------------------------------------------------------------------
877 # Generic command tests
878
879 # A generic command test is expected to run and exit successfully.
880 #
881 # The expected exit code can be changed via exit_code() as normal, and
882 # the expected stdout/stderr are stored in <testname>.stdout and
883 # <testname>.stderr. The output of the command can be ignored
884 # altogether by using run_command_ignore_output instead of
885 # run_command.
886
887 def run_command( name, way, cmd ):
888 return simple_run( name, '', cmd, '' )
889
890 # -----------------------------------------------------------------------------
891 # GHCi tests
892
893 def ghci_script_without_flag(flag):
894 def apply(name, way, script):
895 overrides = filter(lambda f: f != flag, getTestOpts().compiler_always_flags)
896 return ghci_script_override_default_flags(overrides)(name, way, script)
897
898 return apply
899
900 def ghci_script_override_default_flags(overrides):
901 def apply(name, way, script):
902 return ghci_script(name, way, script, overrides)
903
904 return apply
905
906 def ghci_script( name, way, script, override_flags = None ):
907 # Use overriden default flags when given
908 if override_flags:
909 default_flags = override_flags
910 else:
911 default_flags = getTestOpts().compiler_always_flags
912
913 # filter out -fforce-recomp from compiler_always_flags, because we're
914 # actually testing the recompilation behaviour in the GHCi tests.
915 flags = [f for f in default_flags if f != '-fforce-recomp']
916 flags.append(getTestOpts().extra_hc_opts)
917 if getTestOpts().outputdir != None:
918 flags.extend(["-outputdir", getTestOpts().outputdir])
919
920 # We pass HC and HC_OPTS as environment variables, so that the
921 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
922 cmd = "HC='" + config.compiler + "' " + \
923 "HC_OPTS='" + ' '.join(flags) + "' " + \
924 "'" + config.compiler + "'" + \
925 ' --interactive -v0 -ignore-dot-ghci ' + \
926 ' '.join(flags)
927
928 getTestOpts().stdin = script
929 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
930
931 # -----------------------------------------------------------------------------
932 # Compile-only tests
933
934 def compile_override_default_flags(overrides):
935 def apply(name, way, extra_opts):
936 return do_compile(name, way, 0, '', [], extra_opts, overrides)
937
938 return apply
939
940 def compile_fail_override_default_flags(overrides):
941 def apply(name, way, extra_opts):
942 return do_compile(name, way, 1, '', [], extra_opts, overrides)
943
944 return apply
945
946 def compile_without_flag(flag):
947 def apply(name, way, extra_opts):
948 overrides = filter(lambda f: f != flag, getTestOpts().compiler_always_flags)
949 return compile_override_default_flags(overrides)(name, way, extra_opts)
950
951 return apply
952
953 def compile_fail_without_flag(flag):
954 def apply(name, way, extra_opts):
955 overrides = filter(lambda f: f != flag, getTestOpts().compiler_always_flags)
956 return compile_fail_override_default_flags(overrides)(name, way, extra_opts)
957
958 return apply
959
960 def compile( name, way, extra_hc_opts ):
961 return do_compile( name, way, 0, '', [], extra_hc_opts )
962
963 def compile_fail( name, way, extra_hc_opts ):
964 return do_compile( name, way, 1, '', [], extra_hc_opts )
965
966 def multimod_compile( name, way, top_mod, extra_hc_opts ):
967 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
968
969 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
970 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
971
972 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
973 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
974
975 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
976 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
977
978 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts, override_flags = None ):
979 # print 'Compile only, extra args = ', extra_hc_opts
980 pretest_cleanup(name)
981
982 result = extras_build( way, extra_mods, extra_hc_opts )
983 if badResult(result):
984 return result
985 extra_hc_opts = result['hc_opts']
986
987 force = 0
988 if extra_mods:
989 force = 1
990 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force, override_flags )
991
992 if badResult(result):
993 return result
994
995 # the actual stderr should always match the expected, regardless
996 # of whether we expected the compilation to fail or not (successful
997 # compilations may generate warnings).
998
999 if getTestOpts().with_namebase == None:
1000 namebase = name
1001 else:
1002 namebase = getTestOpts().with_namebase
1003
1004 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1005 actual_stderr_file = qualify(name, 'comp.stderr')
1006
1007 if not compare_outputs('stderr', \
1008 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1009 expected_stderr_file, actual_stderr_file):
1010 return failBecause('stderr mismatch')
1011
1012 # no problems found, this test passed
1013 return passed()
1014
1015 def compile_cmp_asm( name, way, extra_hc_opts ):
1016 print('Compile only, extra args = ', extra_hc_opts)
1017 pretest_cleanup(name)
1018 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1019
1020 if badResult(result):
1021 return result
1022
1023 # the actual stderr should always match the expected, regardless
1024 # of whether we expected the compilation to fail or not (successful
1025 # compilations may generate warnings).
1026
1027 if getTestOpts().with_namebase == None:
1028 namebase = name
1029 else:
1030 namebase = getTestOpts().with_namebase
1031
1032 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1033 actual_asm_file = qualify(name, 's')
1034
1035 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1036 expected_asm_file, actual_asm_file):
1037 return failBecause('asm mismatch')
1038
1039 # no problems found, this test passed
1040 return passed()
1041
1042 # -----------------------------------------------------------------------------
1043 # Compile-and-run tests
1044
1045 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1046 # print 'Compile and run, extra args = ', extra_hc_opts
1047 pretest_cleanup(name)
1048
1049 result = extras_build( way, extra_mods, extra_hc_opts )
1050 if badResult(result):
1051 return result
1052 extra_hc_opts = result['hc_opts']
1053
1054 if way == 'ghci': # interpreted...
1055 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1056 else: # compiled...
1057 force = 0
1058 if extra_mods:
1059 force = 1
1060
1061 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1062 if badResult(result):
1063 return result
1064
1065 cmd = './' + name;
1066
1067 # we don't check the compiler's stderr for a compile-and-run test
1068 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1069
1070 def compile_and_run( name, way, extra_hc_opts ):
1071 return compile_and_run__( name, way, '', [], extra_hc_opts)
1072
1073 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1074 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1075
1076 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1077 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1078
1079 def stats( name, way, stats_file ):
1080 opts = getTestOpts()
1081 return checkStats(name, way, stats_file, opts.stats_range_fields)
1082
1083 # -----------------------------------------------------------------------------
1084 # Check -t stats info
1085
1086 def checkStats(name, way, stats_file, range_fields):
1087 full_name = name + '(' + way + ')'
1088
1089 result = passed()
1090 if len(range_fields) > 0:
1091 f = open(in_testdir(stats_file))
1092 contents = f.read()
1093 f.close()
1094
1095 for (field, (expected, dev)) in range_fields.items():
1096 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1097 if m == None:
1098 print('Failed to find field: ', field)
1099 result = failBecause('no such stats field')
1100 val = int(m.group(1))
1101
1102 lowerBound = trunc( expected * ((100 - float(dev))/100))
1103 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1104
1105 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1106
1107 if val < lowerBound:
1108 print(field, 'value is too low:')
1109 print('(If this is because you have improved GHC, please')
1110 print('update the test so that GHC doesn\'t regress again)')
1111 result = failBecause('stat too good')
1112 if val > upperBound:
1113 print(field, 'value is too high:')
1114 result = failBecause('stat not good enough')
1115
1116 if val < lowerBound or val > upperBound or config.verbose >= 4:
1117 valStr = str(val)
1118 valLen = len(valStr)
1119 expectedStr = str(expected)
1120 expectedLen = len(expectedStr)
1121 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1122
1123 def display(descr, val, extra):
1124 print(descr, str(val).rjust(length), extra)
1125
1126 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1127 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1128 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1129 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1130 if val != expected:
1131 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1132
1133 return result
1134
1135 # -----------------------------------------------------------------------------
1136 # Build a single-module program
1137
1138 def extras_build( way, extra_mods, extra_hc_opts ):
1139 for modopts in extra_mods:
1140 mod, opts = modopts
1141 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1142 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1143 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1144 if badResult(result):
1145 return result
1146
1147 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1148
1149
1150 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce, override_flags = None ):
1151 opts = getTestOpts()
1152 errname = add_suffix(name, 'comp.stderr')
1153 rm_no_fail( qualify(errname, '') )
1154
1155 if top_mod != '':
1156 srcname = top_mod
1157 rm_no_fail( qualify(name, '') )
1158 base, suf = os.path.splitext(top_mod)
1159 rm_no_fail( qualify(base, '') )
1160 rm_no_fail( qualify(base, 'exe') )
1161 elif addsuf:
1162 srcname = add_hs_lhs_suffix(name)
1163 rm_no_fail( qualify(name, '') )
1164 else:
1165 srcname = name
1166 rm_no_fail( qualify(name, 'o') )
1167
1168 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1169
1170 to_do = ''
1171 if top_mod != '':
1172 to_do = '--make '
1173 if link:
1174 to_do = to_do + '-o ' + name
1175 elif link:
1176 to_do = '-o ' + name
1177 elif opts.compile_to_hc:
1178 to_do = '-C'
1179 else:
1180 to_do = '-c' # just compile
1181
1182 stats_file = name + '.comp.stats'
1183 if len(opts.compiler_stats_range_fields) > 0:
1184 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1185
1186 # Required by GHC 7.3+, harmless for earlier versions:
1187 if (getTestOpts().c_src or
1188 getTestOpts().objc_src or
1189 getTestOpts().objcpp_src or
1190 getTestOpts().cmm_src):
1191 extra_hc_opts += ' -no-hs-main '
1192
1193 if getTestOpts().compile_cmd_prefix == '':
1194 cmd_prefix = ''
1195 else:
1196 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1197
1198 if override_flags:
1199 comp_flags = copy.copy(override_flags)
1200 else:
1201 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1202
1203 if noforce:
1204 comp_flags = [f for f in comp_flags if f != '-fforce-recomp']
1205 if getTestOpts().outputdir != None:
1206 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1207
1208 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1209 + config.compiler + "' " \
1210 + ' '.join(comp_flags) + ' ' \
1211 + to_do + ' ' + srcname + ' ' \
1212 + ' '.join(config.way_flags(name)[way]) + ' ' \
1213 + extra_hc_opts + ' ' \
1214 + opts.extra_hc_opts + ' ' \
1215 + '>' + errname + ' 2>&1'
1216
1217 result = runCmdFor(name, cmd)
1218
1219 if result != 0 and not should_fail:
1220 actual_stderr = qualify(name, 'comp.stderr')
1221 if_verbose(1,'Compile failed (status ' + repr(result) + ') errors were:')
1222 if_verbose_dump(1,actual_stderr)
1223
1224 # ToDo: if the sub-shell was killed by ^C, then exit
1225
1226 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1227
1228 if badResult(statsResult):
1229 return statsResult
1230
1231 if should_fail:
1232 if result == 0:
1233 return failBecause('exit code 0')
1234 else:
1235 if result != 0:
1236 return failBecause('exit code non-0')
1237
1238 return passed()
1239
1240 # -----------------------------------------------------------------------------
1241 # Run a program and check its output
1242 #
1243 # If testname.stdin exists, route input from that, else
1244 # from /dev/null. Route output to testname.run.stdout and
1245 # testname.run.stderr. Returns the exit code of the run.
1246
1247 def simple_run( name, way, prog, args ):
1248 opts = getTestOpts()
1249
1250 # figure out what to use for stdin
1251 if opts.stdin != '':
1252 use_stdin = opts.stdin
1253 else:
1254 stdin_file = add_suffix(name, 'stdin')
1255 if os.path.exists(in_testdir(stdin_file)):
1256 use_stdin = stdin_file
1257 else:
1258 use_stdin = '/dev/null'
1259
1260 run_stdout = add_suffix(name,'run.stdout')
1261 run_stderr = add_suffix(name,'run.stderr')
1262
1263 rm_no_fail(qualify(name,'run.stdout'))
1264 rm_no_fail(qualify(name,'run.stderr'))
1265 rm_no_fail(qualify(name, 'hp'))
1266 rm_no_fail(qualify(name,'ps'))
1267 rm_no_fail(qualify(name, 'prof'))
1268
1269 my_rts_flags = rts_flags(way)
1270
1271 stats_file = name + '.stats'
1272 if len(opts.stats_range_fields) > 0:
1273 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1274
1275 if opts.no_stdin:
1276 stdin_comes_from = ''
1277 else:
1278 stdin_comes_from = ' <' + use_stdin
1279
1280 if opts.combined_output:
1281 redirection = ' >' + run_stdout \
1282 + ' 2>&1'
1283 else:
1284 redirection = ' >' + run_stdout \
1285 + ' 2>' + run_stderr
1286
1287 cmd = prog + ' ' + args + ' ' \
1288 + my_rts_flags + ' ' \
1289 + stdin_comes_from \
1290 + redirection
1291
1292 if opts.cmd_wrapper != None:
1293 cmd = opts.cmd_wrapper(cmd);
1294
1295 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1296
1297 # run the command
1298 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1299
1300 exit_code = result >> 8
1301 signal = result & 0xff
1302
1303 # check the exit code
1304 if exit_code != opts.exit_code:
1305 print('Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')')
1306 dump_stdout(name)
1307 dump_stderr(name)
1308 return failBecause('bad exit code')
1309
1310 check_hp = my_rts_flags.find("-h") != -1
1311 check_prof = my_rts_flags.find("-p") != -1
1312
1313 if not opts.ignore_output:
1314 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1315 bad_stdout = not check_stdout_ok(name)
1316 if bad_stderr:
1317 return failBecause('bad stderr')
1318 if bad_stdout:
1319 return failBecause('bad stdout')
1320 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1321 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1322 return failBecause('bad heap profile')
1323 if check_prof and not check_prof_ok(name):
1324 return failBecause('bad profile')
1325
1326 return checkStats(name, way, stats_file, opts.stats_range_fields)
1327
1328 def rts_flags(way):
1329 if (way == ''):
1330 return ''
1331 else:
1332 args = config.way_rts_flags[way]
1333
1334 if args == []:
1335 return ''
1336 else:
1337 return '+RTS ' + ' '.join(args) + ' -RTS'
1338
1339 # -----------------------------------------------------------------------------
1340 # Run a program in the interpreter and check its output
1341
1342 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1343 outname = add_suffix(name, 'interp.stdout')
1344 errname = add_suffix(name, 'interp.stderr')
1345 rm_no_fail(outname)
1346 rm_no_fail(errname)
1347 rm_no_fail(name)
1348
1349 if (top_mod == ''):
1350 srcname = add_hs_lhs_suffix(name)
1351 else:
1352 srcname = top_mod
1353
1354 scriptname = add_suffix(name, 'genscript')
1355 qscriptname = in_testdir(scriptname)
1356 rm_no_fail(qscriptname)
1357
1358 delimiter = '===== program output begins here\n'
1359
1360 script = open(qscriptname, 'w')
1361 if not compile_only:
1362 # set the prog name and command-line args to match the compiled
1363 # environment.
1364 script.write(':set prog ' + name + '\n')
1365 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1366 # Add marker lines to the stdout and stderr output files, so we
1367 # can separate GHCi's output from the program's.
1368 script.write(':! echo ' + delimiter)
1369 script.write(':! echo 1>&2 ' + delimiter)
1370 # Set stdout to be line-buffered to match the compiled environment.
1371 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1372 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1373 # in the event of an exception as for the compiled program.
1374 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1375 script.close()
1376
1377 # figure out what to use for stdin
1378 if getTestOpts().stdin != '':
1379 stdin_file = in_testdir(getTestOpts().stdin)
1380 else:
1381 stdin_file = qualify(name, 'stdin')
1382
1383 if os.path.exists(stdin_file):
1384 stdin = open(stdin_file, 'r')
1385 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1386
1387 script.close()
1388
1389 flags = copy.copy(getTestOpts().compiler_always_flags)
1390 if getTestOpts().outputdir != None:
1391 flags.extend(["-outputdir", getTestOpts().outputdir])
1392
1393 cmd = "'" + config.compiler + "' " \
1394 + ' '.join(flags) + ' ' \
1395 + srcname + ' ' \
1396 + ' '.join(config.way_flags(name)[way]) + ' ' \
1397 + extra_hc_opts + ' ' \
1398 + getTestOpts().extra_hc_opts + ' ' \
1399 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1400
1401 if getTestOpts().cmd_wrapper != None:
1402 cmd = getTestOpts().cmd_wrapper(cmd);
1403
1404 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1405
1406 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1407
1408 exit_code = result >> 8
1409 signal = result & 0xff
1410
1411 # split the stdout into compilation/program output
1412 split_file(in_testdir(outname), delimiter,
1413 qualify(name, 'comp.stdout'),
1414 qualify(name, 'run.stdout'))
1415 split_file(in_testdir(errname), delimiter,
1416 qualify(name, 'comp.stderr'),
1417 qualify(name, 'run.stderr'))
1418
1419 # check the exit code
1420 if exit_code != getTestOpts().exit_code:
1421 print('Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1422 dump_stdout(name)
1423 dump_stderr(name)
1424 return failBecause('bad exit code')
1425
1426 # ToDo: if the sub-shell was killed by ^C, then exit
1427
1428 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1429 check_stdout_ok(name)):
1430 return passed()
1431 else:
1432 return failBecause('bad stdout or stderr')
1433
1434
1435 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1436 infile = open(in_fn)
1437 out1 = open(out1_fn, 'w')
1438 out2 = open(out2_fn, 'w')
1439
1440 line = infile.readline()
1441 line = re.sub('\r', '', line) # ignore Windows EOL
1442 while (re.sub('^\s*','',line) != delimiter and line != ''):
1443 out1.write(line)
1444 line = infile.readline()
1445 line = re.sub('\r', '', line)
1446 out1.close()
1447
1448 line = infile.readline()
1449 while (line != ''):
1450 out2.write(line)
1451 line = infile.readline()
1452 out2.close()
1453
1454 # -----------------------------------------------------------------------------
1455 # Utils
1456
1457 def check_stdout_ok( name ):
1458 if getTestOpts().with_namebase == None:
1459 namebase = name
1460 else:
1461 namebase = getTestOpts().with_namebase
1462
1463 actual_stdout_file = qualify(name, 'run.stdout')
1464 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1465
1466 def norm(str):
1467 if platform_specific:
1468 return str
1469 else:
1470 return normalise_output(str)
1471
1472 two_norm = two_normalisers(norm, getTestOpts().extra_normaliser)
1473
1474 check_stdout = getTestOpts().check_stdout
1475 if check_stdout:
1476 return check_stdout(actual_stdout_file, two_norm)
1477
1478 return compare_outputs('stdout', \
1479 two_norm, \
1480 expected_stdout_file, actual_stdout_file)
1481
1482 def dump_stdout( name ):
1483 print('Stdout:')
1484 print(read_no_crs(qualify(name, 'run.stdout')))
1485
1486 def check_stderr_ok( name ):
1487 if getTestOpts().with_namebase == None:
1488 namebase = name
1489 else:
1490 namebase = getTestOpts().with_namebase
1491
1492 actual_stderr_file = qualify(name, 'run.stderr')
1493 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1494
1495 def norm(str):
1496 if platform_specific:
1497 return str
1498 else:
1499 return normalise_errmsg(str)
1500
1501 return compare_outputs('stderr', \
1502 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1503 expected_stderr_file, actual_stderr_file)
1504
1505 def dump_stderr( name ):
1506 print("Stderr:")
1507 print(read_no_crs(qualify(name, 'run.stderr')))
1508
1509 def read_no_crs(file):
1510 str = ''
1511 try:
1512 h = open(file)
1513 str = h.read()
1514 h.close
1515 except:
1516 # On Windows, if the program fails very early, it seems the
1517 # files stdout/stderr are redirected to may not get created
1518 pass
1519 return re.sub('\r', '', str)
1520
1521 def write_file(file, str):
1522 h = open(file, 'w')
1523 h.write(str)
1524 h.close
1525
1526 def check_hp_ok(name):
1527
1528 # do not qualify for hp2ps because we should be in the right directory
1529 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1530
1531 hp2psResult = runCmdExitCode(hp2psCmd)
1532
1533 actual_ps_file = qualify(name, 'ps')
1534
1535 if(hp2psResult == 0):
1536 if (os.path.exists(actual_ps_file)):
1537 if gs_working:
1538 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1539 if (gsResult == 0):
1540 return (True)
1541 else:
1542 print("hp2ps output for " + name + "is not valid PostScript")
1543 else: return (True) # assume postscript is valid without ghostscript
1544 else:
1545 print("hp2ps did not generate PostScript for " + name)
1546 return (False)
1547 else:
1548 print("hp2ps error when processing heap profile for " + name)
1549 return(False)
1550
1551 def check_prof_ok(name):
1552
1553 prof_file = qualify(name,'prof')
1554
1555 if not os.path.exists(prof_file):
1556 print(prof_file + " does not exist")
1557 return(False)
1558
1559 if os.path.getsize(qualify(name,'prof')) == 0:
1560 print(prof_file + " is empty")
1561 return(False)
1562
1563 if getTestOpts().with_namebase == None:
1564 namebase = name
1565 else:
1566 namebase = getTestOpts().with_namebase
1567
1568 (platform_specific, expected_prof_file) = \
1569 platform_wordsize_qualify(namebase, 'prof.sample')
1570
1571 # sample prof file is not required
1572 if not os.path.exists(expected_prof_file):
1573 return True
1574 else:
1575 return compare_outputs('prof', \
1576 two_normalisers(normalise_whitespace,normalise_prof), \
1577 expected_prof_file, prof_file)
1578
1579 # Compare expected output to actual output, and optionally accept the
1580 # new output. Returns true if output matched or was accepted, false
1581 # otherwise.
1582 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1583 if os.path.exists(expected_file):
1584 expected_raw = read_no_crs(expected_file)
1585 # print "norm:", normaliser(expected_raw)
1586 expected_str = normaliser(expected_raw)
1587 expected_file_for_diff = expected_file
1588 else:
1589 expected_str = ''
1590 expected_file_for_diff = '/dev/null'
1591
1592 actual_raw = read_no_crs(actual_file)
1593 actual_str = normaliser(actual_raw)
1594
1595 if expected_str == actual_str:
1596 return 1
1597 else:
1598 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1599
1600 if expected_file_for_diff == '/dev/null':
1601 expected_normalised_file = '/dev/null'
1602 else:
1603 expected_normalised_file = expected_file + ".normalised"
1604 write_file(expected_normalised_file, expected_str)
1605
1606 actual_normalised_file = actual_file + ".normalised"
1607 write_file(actual_normalised_file, actual_str)
1608
1609 # Ignore whitespace when diffing. We should only get to this
1610 # point if there are non-whitespace differences
1611 #
1612 # Note we are diffing the *actual* output, not the normalised
1613 # output. The normalised output may have whitespace squashed
1614 # (including newlines) so the diff would be hard to read.
1615 # This does mean that the diff might contain changes that
1616 # would be normalised away.
1617 if (config.verbose >= 1):
1618 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1619 ' ' + actual_file )
1620
1621 # If for some reason there were no non-whitespace differences,
1622 # then do a full diff
1623 if r == 0:
1624 r = os.system( 'diff -u ' + expected_file_for_diff + \
1625 ' ' + actual_file )
1626
1627 if config.accept:
1628 if_verbose(1, 'Accepting new output.')
1629 write_file(expected_file, actual_raw)
1630 return 1
1631 else:
1632 return 0
1633
1634
1635 def normalise_whitespace( str ):
1636 # Merge contiguous whitespace characters into a single space.
1637 str = re.sub('[ \t\n]+', ' ', str)
1638 return str
1639
1640 def normalise_errmsg( str ):
1641 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1642 # the colon is there because it appears in error messages; this
1643 # hacky solution is used in place of more sophisticated filename
1644 # mangling
1645 str = re.sub('([^\\s])\\.exe', '\\1', str)
1646 # normalise slashes, minimise Windows/Unix filename differences
1647 str = re.sub('\\\\', '/', str)
1648 # The inplace ghc's are called ghc-stage[123] to avoid filename
1649 # collisions, so we need to normalise that to just "ghc"
1650 str = re.sub('ghc-stage[123]', 'ghc', str)
1651 # Error messages simetimes contain integer implementation package
1652 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1653 return str
1654
1655 # normalise a .prof file, so that we can reasonably compare it against
1656 # a sample. This doesn't compare any of the actual profiling data,
1657 # only the shape of the profile and the number of entries.
1658 def normalise_prof (str):
1659 # strip everything up to the line beginning "COST CENTRE"
1660 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1661
1662 # strip results for CAFs, these tend to change unpredictably
1663 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1664
1665 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1666 # sometimes under MAIN.
1667 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1668
1669 # We have somthing like this:
1670
1671 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1672 # k Main 204 1 0.0 0.0 0.0 0.0
1673 # foo Main 205 1 0.0 0.0 0.0 0.0
1674 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1675
1676 # then we remove all the specific profiling data, leaving only the
1677 # cost centre name, module, and entries, to end up with this:
1678
1679 # MAIN MAIN 0
1680 # k Main 1
1681 # foo Main 1
1682 # foo.bar Main 1
1683
1684 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1685 return str
1686
1687 def normalise_slashes_( str ):
1688 str = re.sub('\\\\', '/', str)
1689 return str
1690
1691 def normalise_exe_( str ):
1692 str = re.sub('\.exe', '', str)
1693 return str
1694
1695 def normalise_output( str ):
1696 # Remove a .exe extension (for Windows)
1697 # This can occur in error messages generated by the program.
1698 str = re.sub('([^\\s])\\.exe', '\\1', str)
1699 return str
1700
1701 def normalise_asm( str ):
1702 lines = str.split('\n')
1703 # Only keep instructions and labels not starting with a dot.
1704 metadata = re.compile('^[ \t]*\\..*$')
1705 out = []
1706 for line in lines:
1707 # Drop metadata directives (e.g. ".type")
1708 if not metadata.match(line):
1709 line = re.sub('@plt', '', line)
1710 instr = line.lstrip().split()
1711 # Drop empty lines.
1712 if not instr:
1713 continue
1714 # Drop operands, except for call instructions.
1715 elif instr[0] == 'call':
1716 out.append(instr[0] + ' ' + instr[1])
1717 else:
1718 out.append(instr[0])
1719 out = '\n'.join(out)
1720 return out
1721
1722 def if_verbose( n, s ):
1723 if config.verbose >= n:
1724 print(s)
1725
1726 def if_verbose_dump( n, f ):
1727 if config.verbose >= n:
1728 try:
1729 print(open(f).read())
1730 except:
1731 print('')
1732
1733 def rawSystem(cmd_and_args):
1734 # We prefer subprocess.call to os.spawnv as the latter
1735 # seems to send its arguments through a shell or something
1736 # with the Windows (non-cygwin) python. An argument "a b c"
1737 # turns into three arguments ["a", "b", "c"].
1738
1739 # However, subprocess is new in python 2.4, so fall back to
1740 # using spawnv if we don't have it
1741
1742 if have_subprocess:
1743 return subprocess.call(cmd_and_args)
1744 else:
1745 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1746
1747 # Note that this doesn't handle the timeout itself; it is just used for
1748 # commands that have timeout handling built-in.
1749 def rawSystemWithTimeout(cmd_and_args):
1750 r = rawSystem(cmd_and_args)
1751 if r == 98:
1752 # The python timeout program uses 98 to signal that ^C was pressed
1753 stopNow()
1754 return r
1755
1756 # cmd is a complex command in Bourne-shell syntax
1757 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1758 # Hence it must ultimately be run by a Bourne shell
1759 #
1760 # Mostly it invokes the command wrapped in 'timeout' thus
1761 # timeout 300 'cd . && ...blah blah'
1762 # so it's timeout's job to invoke the Bourne shell
1763 #
1764 # But watch out for the case when there is no timeout program!
1765 # Then, when using the native Python, os.system will invoke the cmd shell
1766
1767 def runCmd( cmd ):
1768 if_verbose( 3, cmd )
1769 r = 0
1770 if config.os == 'mingw32':
1771 # On MinGW, we will always have timeout
1772 assert config.timeout_prog!=''
1773
1774 if config.timeout_prog != '':
1775 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1776 else:
1777 r = os.system(cmd)
1778 return r << 8
1779
1780 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1781 if_verbose( 3, cmd )
1782 r = 0
1783 if config.os == 'mingw32':
1784 # On MinGW, we will always have timeout
1785 assert config.timeout_prog!=''
1786 timeout = int(ceil(config.timeout * timeout_multiplier))
1787
1788 if config.timeout_prog != '':
1789 if config.check_files_written:
1790 fn = name + ".strace"
1791 r = rawSystemWithTimeout(
1792 ["strace", "-o", fn, "-fF",
1793 "-e", "creat,open,chdir,clone,vfork",
1794 config.timeout_prog, str(timeout), cmd])
1795 addTestFilesWritten(name, fn)
1796 rm_no_fail(fn)
1797 else:
1798 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1799 else:
1800 r = os.system(cmd)
1801 return r << 8
1802
1803 def runCmdExitCode( cmd ):
1804 return (runCmd(cmd) >> 8);
1805
1806
1807 # -----------------------------------------------------------------------------
1808 # checking for files being written to by multiple tests
1809
1810 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1811 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1812 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1813 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1814 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1815 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1816 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1817 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1818 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1819 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1820 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1821 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1822 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1823 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1824 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1825 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1826 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1827
1828 # Files that are read or written but shouldn't be:
1829 # * ghci_history shouldn't be read or written by tests
1830 # * things under package.conf.d shouldn't be written by tests
1831 bad_file_usages = {}
1832
1833 # Mapping from tests to the list of files that they write
1834 files_written = {}
1835
1836 # Mapping from tests to the list of files that they write but don't clean
1837 files_written_not_removed = {}
1838
1839 def add_bad_file_usage(name, file):
1840 try:
1841 if not file in bad_file_usages[name]:
1842 bad_file_usages[name].append(file)
1843 except:
1844 bad_file_usages[name] = [file]
1845
1846 def mkPath(curdir, path):
1847 # Given the current full directory is 'curdir', what is the full
1848 # path to 'path'?
1849 return os.path.realpath(os.path.join(curdir, path))
1850
1851 def addTestFilesWritten(name, fn):
1852 if config.use_threads:
1853 with t.lockFilesWritten:
1854 addTestFilesWrittenHelper(name, fn)
1855 else:
1856 addTestFilesWrittenHelper(name, fn)
1857
1858 def addTestFilesWrittenHelper(name, fn):
1859 started = False
1860 working_directories = {}
1861
1862 with open(fn, 'r') as f:
1863 for line in f:
1864 m_pid = re_strace_pid.match(line)
1865 if m_pid:
1866 pid = m_pid.group(1)
1867 content = m_pid.group(2)
1868 elif re_strace_unavailable.match(line):
1869 next
1870 else:
1871 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1872
1873 m_open = re_strace_open.match(content)
1874 m_chdir = re_strace_chdir.match(content)
1875 m_clone = re_strace_clone.match(content)
1876 m_vfork = re_strace_vfork.match(content)
1877
1878 if not started:
1879 working_directories[pid] = os.getcwd()
1880 started = True
1881
1882 if m_open:
1883 file = m_open.group(1)
1884 file = mkPath(working_directories[pid], file)
1885 if file.endswith("ghci_history"):
1886 add_bad_file_usage(name, file)
1887 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1888 flags = m_open.group(2).split('|')
1889 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1890 if re.match('package\.conf\.d', file):
1891 add_bad_file_usage(name, file)
1892 else:
1893 try:
1894 if not file in files_written[name]:
1895 files_written[name].append(file)
1896 except:
1897 files_written[name] = [file]
1898 elif 'O_RDONLY' in flags:
1899 pass
1900 else:
1901 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1902 elif m_chdir:
1903 # We optimistically assume that unfinished chdir's are going to succeed
1904 dir = m_chdir.group(1)
1905 working_directories[pid] = mkPath(working_directories[pid], dir)
1906 elif m_clone:
1907 working_directories[m_clone.group(2)] = working_directories[pid]
1908 elif m_vfork:
1909 working_directories[m_vfork.group(2)] = working_directories[pid]
1910 elif re_strace_open_resumed.match(content):
1911 pass
1912 elif re_strace_chdir_resumed.match(content):
1913 pass
1914 elif re_strace_vfork_unfinished.match(content):
1915 pass
1916 elif re_strace_clone_unfinished.match(content):
1917 pass
1918 elif re_strace_ignore_sigchild.match(content):
1919 pass
1920 elif re_strace_ignore_sigvtalarm.match(content):
1921 pass
1922 elif re_strace_ignore_sigint.match(content):
1923 pass
1924 elif re_strace_ignore_sigfpe.match(content):
1925 pass
1926 elif re_strace_ignore_sigsegv.match(content):
1927 pass
1928 elif re_strace_ignore_sigpipe.match(content):
1929 pass
1930 else:
1931 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1932
1933 def checkForFilesWrittenProblems(file):
1934 foundProblem = False
1935
1936 files_written_inverted = {}
1937 for t in files_written.keys():
1938 for f in files_written[t]:
1939 try:
1940 files_written_inverted[f].append(t)
1941 except:
1942 files_written_inverted[f] = [t]
1943
1944 for f in files_written_inverted.keys():
1945 if len(files_written_inverted[f]) > 1:
1946 if not foundProblem:
1947 foundProblem = True
1948 file.write("\n")
1949 file.write("\nSome files are written by multiple tests:\n")
1950 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1951 if foundProblem:
1952 file.write("\n")
1953
1954 # -----
1955
1956 if len(files_written_not_removed) > 0:
1957 file.write("\n")
1958 file.write("\nSome files written but not removed:\n")
1959 tests = list(files_written_not_removed.keys())
1960 tests.sort()
1961 for t in tests:
1962 for f in files_written_not_removed[t]:
1963 file.write(" " + t + ": " + f + "\n")
1964 file.write("\n")
1965
1966 # -----
1967
1968 if len(bad_file_usages) > 0:
1969 file.write("\n")
1970 file.write("\nSome bad file usages:\n")
1971 tests = list(bad_file_usages.keys())
1972 tests.sort()
1973 for t in tests:
1974 for f in bad_file_usages[t]:
1975 file.write(" " + t + ": " + f + "\n")
1976 file.write("\n")
1977
1978 # -----------------------------------------------------------------------------
1979 # checking if ghostscript is available for checking the output of hp2ps
1980
1981 def genGSCmd(psfile):
1982 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1983
1984 def gsNotWorking():
1985 global gs_working
1986 print("GhostScript not available for hp2ps tests")
1987
1988 global gs_working
1989 gs_working = 0
1990 if config.have_profiling:
1991 if config.gs != '':
1992 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
1993 if resultGood == 0:
1994 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
1995 if resultBad != 0:
1996 print("GhostScript available for hp2ps tests")
1997 gs_working = 1;
1998 else:
1999 gsNotWorking();
2000 else:
2001 gsNotWorking();
2002 else:
2003 gsNotWorking();
2004
2005 def rm_no_fail( file ):
2006 try:
2007 os.remove( file )
2008 finally:
2009 return
2010
2011 def add_suffix( name, suffix ):
2012 if suffix == '':
2013 return name
2014 else:
2015 return name + '.' + suffix
2016
2017 def add_hs_lhs_suffix(name):
2018 if getTestOpts().c_src:
2019 return add_suffix(name, 'c')
2020 elif getTestOpts().cmm_src:
2021 return add_suffix(name, 'cmm')
2022 elif getTestOpts().objc_src:
2023 return add_suffix(name, 'm')
2024 elif getTestOpts().objcpp_src:
2025 return add_suffix(name, 'mm')
2026 elif getTestOpts().literate:
2027 return add_suffix(name, 'lhs')
2028 else:
2029 return add_suffix(name, 'hs')
2030
2031 def replace_suffix( name, suffix ):
2032 base, suf = os.path.splitext(name)
2033 return base + '.' + suffix
2034
2035 def in_testdir( name ):
2036 return (getTestOpts().testdir + '/' + name)
2037
2038 def qualify( name, suff ):
2039 return in_testdir(add_suffix(name, suff))
2040
2041
2042 # Finding the sample output. The filename is of the form
2043 #
2044 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2045 #
2046 # and we pick the most specific version available. The <version> is
2047 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2048 # more fine-grained control use if_compiler_lt().
2049 #
2050 def platform_wordsize_qualify( name, suff ):
2051
2052 basepath = qualify(name, suff)
2053
2054 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2055 for (platformSpecific, plat) in [(1, '-' + config.platform),
2056 (1, '-' + config.os),
2057 (0, '')]
2058 for ws in ['-ws-' + config.wordsize, '']
2059 for comp in ['-' + config.compiler_type, '']
2060 for vers in ['-' + config.compiler_maj_version, '']]
2061
2062 dir = glob.glob(basepath + '*')
2063 dir = [normalise_slashes_(d) for d in dir]
2064
2065 for (platformSpecific, f) in paths:
2066 if f in dir:
2067 return (platformSpecific,f)
2068
2069 return (0, basepath)
2070
2071 # Clean up prior to the test, so that we can't spuriously conclude
2072 # that it passed on the basis of old run outputs.
2073 def pretest_cleanup(name):
2074 if getTestOpts().outputdir != None:
2075 odir = in_testdir(getTestOpts().outputdir)
2076 try:
2077 shutil.rmtree(odir)
2078 except:
2079 pass
2080 os.mkdir(odir)
2081
2082 rm_no_fail(qualify(name,'interp.stderr'))
2083 rm_no_fail(qualify(name,'interp.stdout'))
2084 rm_no_fail(qualify(name,'comp.stderr'))
2085 rm_no_fail(qualify(name,'comp.stdout'))
2086 rm_no_fail(qualify(name,'run.stderr'))
2087 rm_no_fail(qualify(name,'run.stdout'))
2088 rm_no_fail(qualify(name,'tix'))
2089 rm_no_fail(qualify(name,'exe.tix'))
2090 # simple_build zaps the following:
2091 # rm_nofail(qualify("o"))
2092 # rm_nofail(qualify(""))
2093 # not interested in the return code
2094
2095 # -----------------------------------------------------------------------------
2096 # Return a list of all the files ending in '.T' below directories roots.
2097
2098 def findTFiles(roots):
2099 # It would be better to use os.walk, but that
2100 # gives backslashes on Windows, which trip the
2101 # testsuite later :-(
2102 return [filename for root in roots for filename in findTFiles_(root)]
2103
2104 def findTFiles_(path):
2105 if os.path.isdir(path):
2106 paths = [path + '/' + x for x in os.listdir(path)]
2107 return findTFiles(paths)
2108 elif path[-2:] == '.T':
2109 return [path]
2110 else:
2111 return []
2112
2113 # -----------------------------------------------------------------------------
2114 # Output a test summary to the specified file object
2115
2116 def summary(t, file):
2117
2118 file.write('\n')
2119 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2120 file.write('OVERALL SUMMARY for test run started at '
2121 + time.strftime("%c %Z", t.start_time) + '\n'
2122 + str(datetime.timedelta(seconds=
2123 round(time.time() - time.mktime(t.start_time)))).rjust(8)
2124 + ' spent to go through\n'
2125 + repr(t.total_tests).rjust(8)
2126 + ' total tests, which gave rise to\n'
2127 + repr(t.total_test_cases).rjust(8)
2128 + ' test cases, of which\n'
2129 + repr(t.n_tests_skipped).rjust(8)
2130 + ' were skipped\n'
2131 + '\n'
2132 + repr(t.n_missing_libs).rjust(8)
2133 + ' had missing libraries\n'
2134 + repr(t.n_expected_passes).rjust(8)
2135 + ' expected passes\n'
2136 + repr(t.n_expected_failures).rjust(8)
2137 + ' expected failures\n'
2138 + '\n'
2139 + repr(t.n_framework_failures).rjust(8)
2140 + ' caused framework failures\n'
2141 + repr(t.n_unexpected_passes).rjust(8)
2142 + ' unexpected passes\n'
2143 + repr(t.n_unexpected_failures).rjust(8)
2144 + ' unexpected failures\n'
2145 + '\n')
2146
2147 if t.n_unexpected_passes > 0:
2148 file.write('Unexpected passes:\n')
2149 printPassingTestInfosSummary(file, t.unexpected_passes)
2150
2151 if t.n_unexpected_failures > 0:
2152 file.write('Unexpected failures:\n')
2153 printFailingTestInfosSummary(file, t.unexpected_failures)
2154
2155 if config.check_files_written:
2156 checkForFilesWrittenProblems(file)
2157
2158 if stopping():
2159 file.write('WARNING: Testsuite run was terminated early\n')
2160
2161 def printUnexpectedTests(file, testInfoss):
2162 unexpected = []
2163 for testInfos in testInfoss:
2164 directories = testInfos.keys()
2165 for directory in directories:
2166 tests = list(testInfos[directory].keys())
2167 unexpected += tests
2168 if unexpected != []:
2169 file.write('Unexpected results from:\n')
2170 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2171 file.write('\n')
2172
2173 def printPassingTestInfosSummary(file, testInfos):
2174 directories = list(testInfos.keys())
2175 directories.sort()
2176 maxDirLen = max(len(x) for x in directories)
2177 for directory in directories:
2178 tests = list(testInfos[directory].keys())
2179 tests.sort()
2180 for test in tests:
2181 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2182 ' (' + ','.join(testInfos[directory][test]) + ')\n')
2183 file.write('\n')
2184
2185 def printFailingTestInfosSummary(file, testInfos):
2186 directories = list(testInfos.keys())
2187 directories.sort()
2188 maxDirLen = max(len(d) for d in directories)
2189 for directory in directories:
2190 tests = list(testInfos[directory].keys())
2191 tests.sort()
2192 for test in tests:
2193 reasons = testInfos[directory][test].keys()
2194 for reason in reasons:
2195 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2196 ' [' + reason + ']' + \
2197 ' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
2198 file.write('\n')
2199
2200 def getStdout(cmd):
2201 if have_subprocess:
2202 p = subprocess.Popen(cmd,
2203 stdout=subprocess.PIPE,
2204 stderr=subprocess.PIPE)
2205 (stdout, stderr) = p.communicate()
2206 r = p.wait()
2207 if r != 0:
2208 raise Exception("Command failed: " + str(cmd))
2209 if stderr != '':
2210 raise Exception("stderr from command: " + str(cmd))
2211 return stdout
2212 else:
2213 raise Exception("Need subprocess to get stdout, but don't have it")