Make failing perf test output easier to read
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for all the tests in the current "directory". After
44 # each test, we reset the options to these. To change the options for
45 # multiple tests, the function setTestOpts() below can be used to alter
46 # these options.
47 global thisdir_testopts
48 thisdir_testopts = TestOptions()
49
50 def getThisDirTestOpts():
51 return thisdir_testopts
52
53 # Options valid for the current test only (these get reset to
54 # testdir_testopts after each test).
55
56 global testopts_local
57 if config.use_threads:
58 testopts_local = threading.local()
59 else:
60 class TestOpts_Local:
61 pass
62 testopts_local = TestOpts_Local()
63
64 def getTestOpts():
65 return testopts_local.x
66
67 def setLocalTestOpts(opts):
68 global testopts_local
69 testopts_local.x=opts
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 f( thisdir_testopts );
75
76 # -----------------------------------------------------------------------------
77 # Canned setup functions for common cases. eg. for a test you might say
78 #
79 # test('test001', normal, compile, [''])
80 #
81 # to run it without any options, but change it to
82 #
83 # test('test001', expect_fail, compile, [''])
84 #
85 # to expect failure for this test.
86
87 def normal( opts ):
88 return;
89
90 def skip( opts ):
91 opts.skip = 1
92
93 def expect_fail( opts ):
94 opts.expect = 'fail';
95
96 def reqlib( lib ):
97 return lambda opts, l=lib: _reqlib (opts, l )
98
99 # Cache the results of looking to see if we have a library or not.
100 # This makes quite a difference, especially on Windows.
101 have_lib = {}
102
103 def _reqlib( opts, lib ):
104 if have_lib.has_key(lib):
105 got_it = have_lib[lib]
106 else:
107 if have_subprocess:
108 # By preference we use subprocess, as the alternative uses
109 # /dev/null which mingw doesn't have.
110 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' describe ' + lib
119 + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def expect_broken( bug ):
143 return lambda opts, b=bug: _expect_broken (opts, b )
144
145 def _expect_broken( opts, bug ):
146 opts.expect = 'fail';
147
148 def ignore_output( opts ):
149 opts.ignore_output = 1
150
151 def no_stdin( opts ):
152 opts.no_stdin = 1
153
154 def combined_output( opts ):
155 opts.combined_output = True
156
157 # -----
158
159 def expect_fail_for( ways ):
160 return lambda opts, w=ways: _expect_fail_for( opts, w )
161
162 def _expect_fail_for( opts, ways ):
163 opts.expect_fail_for = ways
164
165 def expect_broken_for( bug, ways ):
166 return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )
167
168 def _expect_broken_for( opts, bug, ways ):
169 opts.expect_fail_for = ways
170
171 # -----
172
173 def omit_ways( ways ):
174 return lambda opts, w=ways: _omit_ways( opts, w )
175
176 def _omit_ways( opts, ways ):
177 opts.omit_ways = ways
178
179 # -----
180
181 def only_ways( ways ):
182 return lambda opts, w=ways: _only_ways( opts, w )
183
184 def _only_ways( opts, ways ):
185 opts.only_ways = ways
186
187 # -----
188
189 def extra_ways( ways ):
190 return lambda opts, w=ways: _extra_ways( opts, w )
191
192 def _extra_ways( opts, ways ):
193 opts.extra_ways = ways
194
195 # -----
196
197 def omit_compiler_types( compiler_types ):
198 return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)
199
200 def _omit_compiler_types( opts, compiler_types ):
201 if config.compiler_type in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def only_compiler_types( compiler_types ):
207 return lambda opts, c=compiler_types: _only_compiler_types(opts, c)
208
209 def _only_compiler_types( opts, compiler_types ):
210 if config.compiler_type not in compiler_types:
211 opts.skip = 1
212
213 # -----
214
215 def set_stdin( file ):
216 return lambda opts, f=file: _set_stdin(opts, f);
217
218 def _set_stdin( opts, f ):
219 opts.stdin = f
220
221 # -----
222
223 def exit_code( val ):
224 return lambda opts, v=val: _exit_code(opts, v);
225
226 def _exit_code( opts, v ):
227 opts.exit_code = v
228
229 # -----
230
231 def timeout_multiplier( val ):
232 return lambda opts, v=val: _timeout_multiplier(opts, v)
233
234 def _timeout_multiplier( opts, v ):
235 opts.timeout_multiplier = v
236
237 # -----
238
239 def extra_run_opts( val ):
240 return lambda opts, v=val: _extra_run_opts(opts, v);
241
242 def _extra_run_opts( opts, v ):
243 opts.extra_run_opts = v
244
245 # -----
246
247 def extra_hc_opts( val ):
248 return lambda opts, v=val: _extra_hc_opts(opts, v);
249
250 def _extra_hc_opts( opts, v ):
251 opts.extra_hc_opts = v
252
253 # -----
254
255 def extra_clean( files ):
256 return lambda opts, v=files: _extra_clean(opts, v);
257
258 def _extra_clean( opts, v ):
259 opts.clean_files = v
260
261 # -----
262
263 def stats_range_field( field, min, max ):
264 return lambda opts, f=field, x=min, y=max: _stats_range_field(opts, f, x, y);
265
266 def _stats_range_field( opts, f, x, y ):
267 # copy the dictionary, as the config gets shared between all tests
268 opts.stats_range_fields = opts.stats_range_fields.copy()
269 opts.stats_range_fields[f] = (x, y)
270
271 def compiler_stats_range_field( field, min, max ):
272 return lambda opts, f=field, x=min, y=max: _compiler_stats_range_field(opts, f, x, y);
273
274 def _compiler_stats_range_field( opts, f, x, y ):
275 # copy the dictionary, as the config gets shared between all tests
276 opts.compiler_stats_range_fields = opts.compiler_stats_range_fields.copy()
277 opts.compiler_stats_range_fields[f] = (x, y)
278
279 # -----
280
281 def skip_if_no_ghci(opts):
282 if not ('ghci' in config.run_ways):
283 opts.skip = 1
284
285 # ----
286
287 def skip_if_fast(opts):
288 if config.fast:
289 opts.skip = 1
290
291 # -----
292
293 def if_platform( plat, f ):
294 if config.platform == plat:
295 return f
296 else:
297 return normal
298
299 def unless_platform( plat, f ):
300 if config.platform != plat:
301 return f
302 else:
303 return normal
304
305 def if_os( os, f ):
306 if config.os == os:
307 return f
308 else:
309 return normal
310
311 def unless_os( os, f ):
312 if config.os == os:
313 return normal
314 else:
315 return f
316
317 def if_arch( arch, f ):
318 if config.arch == arch:
319 return f
320 else:
321 return normal
322
323 def unless_arch( arch, f ):
324 if config.arch == arch:
325 return normal
326 else:
327 return f
328
329 def if_wordsize( ws, f ):
330 if config.wordsize == str(ws):
331 return f
332 else:
333 return normal
334
335 def unless_wordsize( ws, f ):
336 if config.wordsize == str(ws):
337 return normal
338 else:
339 return f
340
341 def if_unregisterised( f ):
342 if config.unregisterised:
343 return f
344 else:
345 return normal
346
347 def unless_unregisterised( f ):
348 if config.unregisterised:
349 return normal
350 else:
351 return f
352
353 def if_msys( f ):
354 if config.msys:
355 return f
356 else:
357 return normal
358
359 def if_cygwin( f ):
360 if config.cygwin:
361 return f
362 else:
363 return normal
364
365 def when_have_vanilla( f ):
366 if config.have_vanilla:
367 return f
368 else:
369 return normal
370
371 def unless_have_vanilla( f ):
372 if config.have_vanilla:
373 return normal
374 else:
375 return f
376
377 def when_have_dynamic( f ):
378 if config.have_dynamic:
379 return f
380 else:
381 return normal
382
383 def unless_have_dynamic( f ):
384 if config.have_dynamic:
385 return normal
386 else:
387 return f
388
389 def when_have_profiling( f ):
390 if config.have_profiling:
391 return f
392 else:
393 return normal
394
395 def unless_have_profiling( f ):
396 if config.have_profiling:
397 return normal
398 else:
399 return f
400
401 # ---
402
403 def if_ghci_dynamic( f ):
404 if config.ghc_dynamic_by_default:
405 return f
406 else:
407 return normal
408
409 def if_in_tree_compiler( f ):
410 if config.in_tree_compiler:
411 return f
412 else:
413 return normal
414
415 def unless_in_tree_compiler( f ):
416 if config.in_tree_compiler:
417 return normal
418 else:
419 return f
420
421 def if_compiler_type( compiler, f ):
422 if config.compiler_type == compiler:
423 return f
424 else:
425 return normal
426
427 def if_compiler_profiled( f ):
428 if config.compiler_profiled:
429 return f
430 else:
431 return normal
432
433 def unless_compiler_profiled( f ):
434 if config.compiler_profiled:
435 return normal
436 else:
437 return f
438
439 def if_compiler_lt( compiler, version, f ):
440 if config.compiler_type == compiler and \
441 version_lt(config.compiler_version, version):
442 return f
443 else:
444 return normal
445
446 def if_compiler_le( compiler, version, f ):
447 if config.compiler_type == compiler and \
448 version_le(config.compiler_version, version):
449 return f
450 else:
451 return normal
452
453 def if_compiler_gt( compiler, version, f ):
454 if config.compiler_type == compiler and \
455 version_gt(config.compiler_version, version):
456 return f
457 else:
458 return normal
459
460 def if_compiler_ge( compiler, version, f ):
461 if config.compiler_type == compiler and \
462 version_ge(config.compiler_version, version):
463 return f
464 else:
465 return normal
466
467 def if_compiler_debugged( f ):
468 if config.compiler_debugged:
469 return f
470 else:
471 return normal
472
473 def namebase( nb ):
474 return lambda opts, nb=nb: _namebase(opts, nb)
475
476 def _namebase( opts, nb ):
477 opts.with_namebase = nb
478
479 # ---
480
481 def if_tag( tag, f ):
482 if tag in config.compiler_tags:
483 return f
484 else:
485 return normal
486
487 def unless_tag( tag, f ):
488 if not (tag in config.compiler_tags):
489 return f
490 else:
491 return normal
492
493 # ---
494 def high_memory_usage(opts):
495 opts.alone = True
496
497 # ---
498 def literate( opts ):
499 opts.literate = 1;
500
501 def c_src( opts ):
502 opts.c_src = 1;
503
504 def objc_src( opts ):
505 opts.objc_src = 1;
506
507 def objcpp_src( opts ):
508 opts.objcpp_src = 1;
509
510 def cmm_src( opts ):
511 opts.cmm_src = 1;
512
513 def outputdir( odir ):
514 return lambda opts, d=odir: _outputdir(opts, d)
515
516 def _outputdir( opts, odir ):
517 opts.outputdir = odir;
518
519 # ----
520
521 def pre_cmd( cmd ):
522 return lambda opts, c=cmd: _pre_cmd(opts, cmd)
523
524 def _pre_cmd( opts, cmd ):
525 opts.pre_cmd = cmd
526
527 # ----
528
529 def clean_cmd( cmd ):
530 return lambda opts, c=cmd: _clean_cmd(opts, cmd)
531
532 def _clean_cmd( opts, cmd ):
533 opts.clean_cmd = cmd
534
535 # ----
536
537 def cmd_prefix( prefix ):
538 return lambda opts, p=prefix: _cmd_prefix(opts, prefix)
539
540 def _cmd_prefix( opts, prefix ):
541 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
542
543 # ----
544
545 def cmd_wrapper( fun ):
546 return lambda opts, f=fun: _cmd_wrapper(opts, fun)
547
548 def _cmd_wrapper( opts, fun ):
549 opts.cmd_wrapper = fun
550
551 # ----
552
553 def compile_cmd_prefix( prefix ):
554 return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)
555
556 def _compile_cmd_prefix( opts, prefix ):
557 opts.compile_cmd_prefix = prefix
558
559 # ----
560
561 def normalise_slashes( opts ):
562 opts.extra_normaliser = normalise_slashes_
563
564 def normalise_exe( opts ):
565 opts.extra_normaliser = normalise_exe_
566
567 def normalise_fun( fun ):
568 return lambda opts, f=fun: _normalise_fun(opts, f)
569
570 def _normalise_fun( opts, f ):
571 opts.extra_normaliser = f
572
573 def normalise_errmsg_fun( fun ):
574 return lambda opts, f=fun: _normalise_errmsg_fun(opts, f)
575
576 def _normalise_errmsg_fun( opts, f ):
577 opts.extra_errmsg_normaliser = f
578
579 def two_normalisers(f, g):
580 return lambda x, f=f, g=g: f(g(x))
581
582 # ----
583 # Function for composing two opt-fns together
584
585 def composes( fs ):
586 return reduce(lambda f, g: compose(f, g), fs)
587
588 def compose( f, g ):
589 return lambda opts, f=f, g=g: _compose(opts,f,g)
590
591 def _compose( opts, f, g ):
592 f(opts)
593 g(opts)
594
595 # -----------------------------------------------------------------------------
596 # The current directory of tests
597
598 def newTestDir( dir ):
599 global thisdir_testopts
600 # reset the options for this test directory
601 thisdir_testopts = copy.copy(default_testopts)
602 thisdir_testopts.testdir = dir
603 thisdir_testopts.compiler_always_flags = config.compiler_always_flags
604
605 # -----------------------------------------------------------------------------
606 # Actually doing tests
607
608 parallelTests = []
609 aloneTests = []
610 allTestNames = set([])
611
612 def runTest (opts, name, func, args):
613 ok = 0
614
615 if config.use_threads:
616 t.thread_pool.acquire()
617 try:
618 while config.threads<(t.running_threads+1):
619 t.thread_pool.wait()
620 t.running_threads = t.running_threads+1
621 ok=1
622 t.thread_pool.release()
623 thread.start_new_thread(test_common_thread, (name, opts, func, args))
624 except:
625 if not ok:
626 t.thread_pool.release()
627 else:
628 test_common_work (name, opts, func, args)
629
630 # name :: String
631 # setup :: TestOpts -> IO ()
632 def test (name, setup, func, args):
633 global aloneTests
634 global parallelTests
635 global allTestNames
636 if name in allTestNames:
637 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
638 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
639 framework_fail(name, 'bad_name', 'This test has an invalid name')
640 myTestOpts = copy.copy(thisdir_testopts)
641
642 if type(setup) is types.ListType:
643 setup = composes(setup)
644
645 setup(myTestOpts)
646
647 thisTest = lambda : runTest(myTestOpts, name, func, args)
648 if myTestOpts.alone:
649 aloneTests.append(thisTest)
650 else:
651 parallelTests.append(thisTest)
652 allTestNames.add(name)
653
654 if config.use_threads:
655 def test_common_thread(name, opts, func, args):
656 t.lock.acquire()
657 try:
658 test_common_work(name,opts,func,args)
659 finally:
660 t.lock.release()
661 t.thread_pool.acquire()
662 t.running_threads = t.running_threads - 1
663 t.thread_pool.notify()
664 t.thread_pool.release()
665
666 def get_package_cache_timestamp():
667 if config.package_conf_cache_file == '':
668 return 0.0
669 else:
670 try:
671 return os.stat(config.package_conf_cache_file).st_mtime
672 except:
673 return 0.0
674
675
676 def test_common_work (name, opts, func, args):
677 try:
678 t.total_tests = t.total_tests+1
679 setLocalTestOpts(opts)
680
681 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
682
683 # All the ways we might run this test
684 if func == compile or func == multimod_compile:
685 all_ways = config.compile_ways
686 elif func == compile_and_run or func == multimod_compile_and_run:
687 all_ways = config.run_ways
688 elif func == ghci_script:
689 if 'ghci' in config.run_ways:
690 all_ways = ['ghci']
691 else:
692 all_ways = []
693 else:
694 all_ways = ['normal']
695
696 # A test itself can request extra ways by setting opts.extra_ways
697 all_ways = all_ways + filter(lambda way: way not in all_ways,
698 opts.extra_ways)
699
700 t.total_test_cases = t.total_test_cases + len(all_ways)
701
702 ok_way = lambda way: \
703 not getTestOpts().skip \
704 and (config.only == [] or name in config.only) \
705 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
706 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
707 and way not in getTestOpts().omit_ways
708
709 # Which ways we are asked to skip
710 do_ways = filter (ok_way,all_ways)
711
712 # In fast mode, we skip all but one way
713 if config.fast and len(do_ways) > 0:
714 do_ways = [do_ways[0]]
715
716 if not config.clean_only:
717 # Run the required tests...
718 for way in do_ways:
719 if stopping():
720 break
721 do_test (name, way, func, args)
722
723 for way in all_ways:
724 if way not in do_ways:
725 skiptest (name,way)
726
727 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
728 clean(map (lambda suff: name + suff,
729 ['', '.exe', '.exe.manifest', '.genscript',
730 '.stderr.normalised', '.stdout.normalised',
731 '.run.stderr', '.run.stdout',
732 '.run.stderr.normalised', '.run.stdout.normalised',
733 '.comp.stderr', '.comp.stdout',
734 '.comp.stderr.normalised', '.comp.stdout.normalised',
735 '.interp.stderr', '.interp.stdout',
736 '.interp.stderr.normalised', '.interp.stdout.normalised',
737 '.stats', '.comp.stats',
738 '.hi', '.o', '.prof', '.exe.prof', '.hc',
739 '_stub.h', '_stub.c', '_stub.o',
740 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
741
742 if func == multi_compile or func == multi_compile_fail:
743 extra_mods = args[1]
744 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
745 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
746
747 clean(getTestOpts().clean_files)
748
749 if getTestOpts().outputdir != None:
750 odir = in_testdir(getTestOpts().outputdir)
751 try:
752 shutil.rmtree(odir)
753 except:
754 pass
755
756 try:
757 shutil.rmtree(in_testdir('.hpc.' + name))
758 except:
759 pass
760
761 try:
762 cleanCmd = getTestOpts().clean_cmd
763 if cleanCmd != None:
764 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
765 if result != 0:
766 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
767 except e:
768 framework_fail(name, 'cleaning', 'clean-command exception')
769
770 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
771
772 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
773 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
774
775 try:
776 for f in files_written[name]:
777 if os.path.exists(f):
778 try:
779 if not f in files_written_not_removed[name]:
780 files_written_not_removed[name].append(f)
781 except:
782 files_written_not_removed[name] = [f]
783 except:
784 pass
785 except Exception, e:
786 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
787
788 def clean(strs):
789 for str in strs:
790 for name in glob.glob(in_testdir(str)):
791 clean_full_path(name)
792
793 def clean_full_path(name):
794 try:
795 # Remove files...
796 os.remove(name)
797 except OSError, e1:
798 try:
799 # ... and empty directories
800 os.rmdir(name)
801 except OSError, e2:
802 # We don't want to fail here, but we do want to know
803 # what went wrong, so print out the exceptions.
804 # ENOENT isn't a problem, though, as we clean files
805 # that don't necessarily exist.
806 if e1.errno != errno.ENOENT:
807 print e1
808 if e2.errno != errno.ENOENT:
809 print e2
810
811 def do_test(name, way, func, args):
812 full_name = name + '(' + way + ')'
813
814 try:
815 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
816 str([t.n_unexpected_passes, \
817 t.n_unexpected_failures, \
818 t.n_framework_failures])
819
820 if config.use_threads:
821 t.lock.release()
822
823 try:
824 preCmd = getTestOpts().pre_cmd
825 if preCmd != None:
826 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
827 if result != 0:
828 framework_fail(name, way, 'pre-command failed: ' + str(result))
829 except e:
830 framework_fail(name, way, 'pre-command exception')
831
832 try:
833 result = apply(func, [name,way] + args)
834 finally:
835 if config.use_threads:
836 t.lock.acquire()
837
838 if getTestOpts().expect != 'pass' and \
839 getTestOpts().expect != 'fail' and \
840 getTestOpts().expect != 'missing-lib':
841 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
842
843 try:
844 passFail = result['passFail']
845 except:
846 passFail = 'No passFail found'
847
848 if passFail == 'pass':
849 if getTestOpts().expect == 'pass' \
850 and way not in getTestOpts().expect_fail_for:
851 t.n_expected_passes = t.n_expected_passes + 1
852 if name in t.expected_passes:
853 t.expected_passes[name].append(way)
854 else:
855 t.expected_passes[name] = [way]
856 else:
857 print '*** unexpected pass for', full_name
858 t.n_unexpected_passes = t.n_unexpected_passes + 1
859 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
860 elif passFail == 'fail':
861 if getTestOpts().expect == 'pass' \
862 and way not in getTestOpts().expect_fail_for:
863 print '*** unexpected failure for', full_name
864 t.n_unexpected_failures = t.n_unexpected_failures + 1
865 reason = result['reason']
866 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
867 else:
868 if getTestOpts().expect == 'missing-lib':
869 t.n_missing_libs = t.n_missing_libs + 1
870 if name in t.missing_libs:
871 t.missing_libs[name].append(way)
872 else:
873 t.missing_libs[name] = [way]
874 else:
875 t.n_expected_failures = t.n_expected_failures + 1
876 if name in t.expected_failures:
877 t.expected_failures[name].append(way)
878 else:
879 t.expected_failures[name] = [way]
880 else:
881 framework_fail(name, way, 'bad result ' + passFail)
882 except KeyboardInterrupt:
883 stopNow()
884 except:
885 framework_fail(name, way, 'do_test exception')
886 traceback.print_exc()
887
888 def addPassingTestInfo (testInfos, directory, name, way):
889 directory = re.sub('^\\.[/\\\\]', '', directory)
890
891 if not directory in testInfos:
892 testInfos[directory] = {}
893
894 if not name in testInfos[directory]:
895 testInfos[directory][name] = []
896
897 testInfos[directory][name].append(way)
898
899 def addFailingTestInfo (testInfos, directory, name, reason, way):
900 directory = re.sub('^\\.[/\\\\]', '', directory)
901
902 if not directory in testInfos:
903 testInfos[directory] = {}
904
905 if not name in testInfos[directory]:
906 testInfos[directory][name] = {}
907
908 if not reason in testInfos[directory][name]:
909 testInfos[directory][name][reason] = []
910
911 testInfos[directory][name][reason].append(way)
912
913 def skiptest (name, way):
914 # print 'Skipping test \"', name, '\"'
915 t.n_tests_skipped = t.n_tests_skipped + 1
916 if name in t.tests_skipped:
917 t.tests_skipped[name].append(way)
918 else:
919 t.tests_skipped[name] = [way]
920
921 def framework_fail( name, way, reason ):
922 full_name = name + '(' + way + ')'
923 print '*** framework failure for', full_name, reason
924 t.n_framework_failures = t.n_framework_failures + 1
925 if name in t.framework_failures:
926 t.framework_failures[name].append(way)
927 else:
928 t.framework_failures[name] = [way]
929
930 def badResult(result):
931 try:
932 if result['passFail'] == 'pass':
933 return False
934 return True
935 except:
936 return True
937
938 def passed():
939 return {'passFail': 'pass'}
940
941 def failBecause(reason):
942 return {'passFail': 'fail', 'reason': reason}
943
944 # -----------------------------------------------------------------------------
945 # Generic command tests
946
947 # A generic command test is expected to run and exit successfully.
948 #
949 # The expected exit code can be changed via exit_code() as normal, and
950 # the expected stdout/stderr are stored in <testname>.stdout and
951 # <testname>.stderr. The output of the command can be ignored
952 # altogether by using run_command_ignore_output instead of
953 # run_command.
954
955 def run_command( name, way, cmd ):
956 return simple_run( name, '', cmd, '' )
957
958 # -----------------------------------------------------------------------------
959 # GHCi tests
960
961 def ghci_script( name, way, script ):
962 # filter out -fforce-recomp from compiler_always_flags, because we're
963 # actually testing the recompilation behaviour in the GHCi tests.
964 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
965 flags.append(getTestOpts().extra_hc_opts)
966 if getTestOpts().outputdir != None:
967 flags.extend(["-outputdir", getTestOpts().outputdir])
968
969 # We pass HC and HC_OPTS as environment variables, so that the
970 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
971 cmd = "HC='" + config.compiler + "' " + \
972 "HC_OPTS='" + join(flags,' ') + "' " + \
973 "'" + config.compiler + "'" + \
974 ' --interactive -v0 -ignore-dot-ghci ' + \
975 join(flags,' ')
976
977 getTestOpts().stdin = script
978 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
979
980 # -----------------------------------------------------------------------------
981 # Compile-only tests
982
983 def compile( name, way, extra_hc_opts ):
984 return do_compile( name, way, 0, '', [], extra_hc_opts )
985
986 def compile_fail( name, way, extra_hc_opts ):
987 return do_compile( name, way, 1, '', [], extra_hc_opts )
988
989 def multimod_compile( name, way, top_mod, extra_hc_opts ):
990 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
991
992 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
993 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
994
995 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
996 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
997
998 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
999 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1000
1001 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1002 # print 'Compile only, extra args = ', extra_hc_opts
1003 pretest_cleanup(name)
1004
1005 result = extras_build( way, extra_mods, extra_hc_opts )
1006 if badResult(result):
1007 return result
1008 extra_hc_opts = result['hc_opts']
1009
1010 force = 0
1011 if extra_mods:
1012 force = 1
1013 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1014
1015 if badResult(result):
1016 return result
1017
1018 # the actual stderr should always match the expected, regardless
1019 # of whether we expected the compilation to fail or not (successful
1020 # compilations may generate warnings).
1021
1022 if getTestOpts().with_namebase == None:
1023 namebase = name
1024 else:
1025 namebase = getTestOpts().with_namebase
1026
1027 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1028 actual_stderr_file = qualify(name, 'comp.stderr')
1029
1030 if not compare_outputs('stderr', \
1031 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1032 expected_stderr_file, actual_stderr_file):
1033 return failBecause('stderr mismatch')
1034
1035 # no problems found, this test passed
1036 return passed()
1037
1038 def compile_cmp_asm( name, way, extra_hc_opts ):
1039 print 'Compile only, extra args = ', extra_hc_opts
1040 pretest_cleanup(name)
1041 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1042
1043 if badResult(result):
1044 return result
1045
1046 # the actual stderr should always match the expected, regardless
1047 # of whether we expected the compilation to fail or not (successful
1048 # compilations may generate warnings).
1049
1050 if getTestOpts().with_namebase == None:
1051 namebase = name
1052 else:
1053 namebase = getTestOpts().with_namebase
1054
1055 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1056 actual_asm_file = qualify(name, 's')
1057
1058 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1059 expected_asm_file, actual_asm_file):
1060 return failBecause('asm mismatch')
1061
1062 # no problems found, this test passed
1063 return passed()
1064
1065 # -----------------------------------------------------------------------------
1066 # Compile-and-run tests
1067
1068 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1069 # print 'Compile and run, extra args = ', extra_hc_opts
1070 pretest_cleanup(name)
1071
1072 result = extras_build( way, extra_mods, extra_hc_opts )
1073 if badResult(result):
1074 return result
1075 extra_hc_opts = result['hc_opts']
1076
1077 if way == 'ghci': # interpreted...
1078 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1079 elif way == 'extcore' or way == 'optextcore' :
1080 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1081 else: # compiled...
1082 force = 0
1083 if extra_mods:
1084 force = 1
1085
1086 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1087 if badResult(result):
1088 return result
1089
1090 cmd = './' + name;
1091
1092 # we don't check the compiler's stderr for a compile-and-run test
1093 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1094
1095 def compile_and_run( name, way, extra_hc_opts ):
1096 return compile_and_run__( name, way, '', [], extra_hc_opts)
1097
1098 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1099 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1100
1101 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1102 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1103
1104 def stats( name, way, stats_file ):
1105 opts = getTestOpts()
1106 return checkStats(stats_file, opts.stats_range_fields)
1107
1108 # -----------------------------------------------------------------------------
1109 # Check -t stats info
1110
1111 def checkStats(stats_file, range_fields):
1112 result = passed()
1113 if len(range_fields) > 0:
1114 f = open(in_testdir(stats_file))
1115 contents = f.read()
1116 f.close()
1117
1118 for (field, (expected, dev)) in range_fields.items():
1119 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1120 if m == None:
1121 print 'Failed to find field: ', field
1122 result = failBecause('no such stats field')
1123 val = int(m.group(1))
1124
1125 lowerBound = trunc( expected * ((100 - float(dev))/100));
1126 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1127
1128 if val < lowerBound:
1129 print field, 'value is too low:'
1130 print '(If this is because you have improved GHC, please'
1131 print 'update the test so that GHC doesn\'t regress again)'
1132 result = failBecause('stat too good')
1133 if val > upperBound:
1134 print field, 'value is too high:'
1135 result = failBecause('stat not good enough')
1136
1137 if val < lowerBound or val > upperBound:
1138 valStr = str(val)
1139 valLen = len(valStr)
1140 expectedStr = str(expected)
1141 expectedLen = len(expectedStr)
1142 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1143 def display(descr, val, extra):
1144 valStr = str(val)
1145 print descr, (' ' * (length - len(valStr))) + valStr, extra
1146 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1147 display(' Lower bound ' + field + ':', lowerBound, '')
1148 display(' Upper bound ' + field + ':', upperBound, '')
1149 display(' Actual ' + field + ':', val, '')
1150
1151 return result
1152
1153 # -----------------------------------------------------------------------------
1154 # Build a single-module program
1155
1156 def extras_build( way, extra_mods, extra_hc_opts ):
1157 for modopts in extra_mods:
1158 mod, opts = modopts
1159 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1160 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1161 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1162 if badResult(result):
1163 return result
1164
1165 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1166
1167
1168 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1169 opts = getTestOpts()
1170 errname = add_suffix(name, 'comp.stderr')
1171 rm_no_fail( qualify(errname, '') )
1172
1173 if top_mod != '':
1174 srcname = top_mod
1175 rm_no_fail( qualify(name, '') )
1176 base, suf = os.path.splitext(top_mod)
1177 rm_no_fail( qualify(base, '') )
1178 rm_no_fail( qualify(base, 'exe') )
1179 elif addsuf:
1180 srcname = add_hs_lhs_suffix(name)
1181 rm_no_fail( qualify(name, '') )
1182 else:
1183 srcname = name
1184 rm_no_fail( qualify(name, 'o') )
1185
1186 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1187
1188 to_do = ''
1189 if top_mod != '':
1190 to_do = '--make '
1191 if link:
1192 to_do = to_do + '-o ' + name
1193 elif link:
1194 to_do = '-o ' + name
1195 elif opts.compile_to_hc:
1196 to_do = '-C'
1197 else:
1198 to_do = '-c' # just compile
1199
1200 stats_file = name + '.comp.stats'
1201 if len(opts.compiler_stats_range_fields) > 0:
1202 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1203
1204 # Required by GHC 7.3+, harmless for earlier versions:
1205 if (getTestOpts().c_src or
1206 getTestOpts().objc_src or
1207 getTestOpts().objcpp_src or
1208 getTestOpts().cmm_src):
1209 extra_hc_opts += ' -no-hs-main '
1210
1211 if getTestOpts().compile_cmd_prefix == '':
1212 cmd_prefix = ''
1213 else:
1214 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1215
1216 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1217 if noforce:
1218 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1219 if getTestOpts().outputdir != None:
1220 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1221
1222 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1223 + config.compiler + "' " \
1224 + join(comp_flags,' ') + ' ' \
1225 + to_do + ' ' + srcname + ' ' \
1226 + join(config.way_flags(name)[way],' ') + ' ' \
1227 + extra_hc_opts + ' ' \
1228 + opts.extra_hc_opts + ' ' \
1229 + '>' + errname + ' 2>&1'
1230
1231 result = runCmdFor(name, cmd)
1232
1233 if result != 0 and not should_fail:
1234 actual_stderr = qualify(name, 'comp.stderr')
1235 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1236 if_verbose_dump(1,actual_stderr)
1237
1238 # ToDo: if the sub-shell was killed by ^C, then exit
1239
1240 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1241
1242 if badResult(statsResult):
1243 return statsResult
1244
1245 if should_fail:
1246 if result == 0:
1247 return failBecause('exit code 0')
1248 else:
1249 if result != 0:
1250 return failBecause('exit code non-0')
1251
1252 return passed()
1253
1254 # -----------------------------------------------------------------------------
1255 # Run a program and check its output
1256 #
1257 # If testname.stdin exists, route input from that, else
1258 # from /dev/null. Route output to testname.run.stdout and
1259 # testname.run.stderr. Returns the exit code of the run.
1260
1261 def simple_run( name, way, prog, args ):
1262 opts = getTestOpts()
1263
1264 # figure out what to use for stdin
1265 if opts.stdin != '':
1266 use_stdin = opts.stdin
1267 else:
1268 stdin_file = add_suffix(name, 'stdin')
1269 if os.path.exists(in_testdir(stdin_file)):
1270 use_stdin = stdin_file
1271 else:
1272 use_stdin = '/dev/null'
1273
1274 run_stdout = add_suffix(name,'run.stdout')
1275 run_stderr = add_suffix(name,'run.stderr')
1276
1277 rm_no_fail(qualify(name,'run.stdout'))
1278 rm_no_fail(qualify(name,'run.stderr'))
1279 rm_no_fail(qualify(name, 'hp'))
1280 rm_no_fail(qualify(name,'ps'))
1281 rm_no_fail(qualify(name, 'prof'))
1282
1283 my_rts_flags = rts_flags(way)
1284
1285 stats_file = name + '.stats'
1286 if len(opts.stats_range_fields) > 0:
1287 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1288
1289 if opts.no_stdin:
1290 stdin_comes_from = ''
1291 else:
1292 stdin_comes_from = ' <' + use_stdin
1293
1294 if opts.combined_output:
1295 redirection = ' >' + run_stdout \
1296 + ' 2>&1'
1297 else:
1298 redirection = ' >' + run_stdout \
1299 + ' 2>' + run_stderr
1300
1301 cmd = prog + ' ' + args + ' ' \
1302 + my_rts_flags + ' ' \
1303 + stdin_comes_from \
1304 + redirection
1305
1306 if opts.cmd_wrapper != None:
1307 cmd = opts.cmd_wrapper(cmd);
1308
1309 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1310
1311 # run the command
1312 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1313
1314 exit_code = result >> 8
1315 signal = result & 0xff
1316
1317 # check the exit code
1318 if exit_code != opts.exit_code:
1319 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1320 dump_stdout(name)
1321 dump_stderr(name)
1322 return failBecause('bad exit code')
1323
1324 check_hp = my_rts_flags.find("-h") != -1
1325 check_prof = my_rts_flags.find("-p") != -1
1326
1327 if not opts.ignore_output:
1328 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1329 bad_stdout = not check_stdout_ok(name)
1330 if bad_stderr:
1331 return failBecause('bad stderr')
1332 if bad_stdout:
1333 return failBecause('bad stdout')
1334 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1335 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1336 return failBecause('bad heap profile')
1337 if check_prof and not check_prof_ok(name):
1338 return failBecause('bad profile')
1339
1340 return checkStats(stats_file, opts.stats_range_fields)
1341
1342 def rts_flags(way):
1343 if (way == ''):
1344 return ''
1345 else:
1346 args = config.way_rts_flags[way]
1347
1348 if args == []:
1349 return ''
1350 else:
1351 return '+RTS ' + join(args,' ') + ' -RTS'
1352
1353 # -----------------------------------------------------------------------------
1354 # Run a program in the interpreter and check its output
1355
1356 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1357 outname = add_suffix(name, 'interp.stdout')
1358 errname = add_suffix(name, 'interp.stderr')
1359 rm_no_fail(outname)
1360 rm_no_fail(errname)
1361 rm_no_fail(name)
1362
1363 if (top_mod == ''):
1364 srcname = add_hs_lhs_suffix(name)
1365 else:
1366 srcname = top_mod
1367
1368 scriptname = add_suffix(name, 'genscript')
1369 qscriptname = in_testdir(scriptname)
1370 rm_no_fail(qscriptname)
1371
1372 delimiter = '===== program output begins here\n'
1373
1374 script = open(qscriptname, 'w')
1375 if not compile_only:
1376 # set the prog name and command-line args to match the compiled
1377 # environment.
1378 script.write(':set prog ' + name + '\n')
1379 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1380 # Add marker lines to the stdout and stderr output files, so we
1381 # can separate GHCi's output from the program's.
1382 script.write(':! echo ' + delimiter)
1383 script.write(':! echo 1>&2 ' + delimiter)
1384 # Set stdout to be line-buffered to match the compiled environment.
1385 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1386 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1387 # in the event of an exception as for the compiled program.
1388 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1389 script.close()
1390
1391 # figure out what to use for stdin
1392 if getTestOpts().stdin != '':
1393 stdin_file = in_testdir(getTestOpts().stdin)
1394 else:
1395 stdin_file = qualify(name, 'stdin')
1396
1397 if os.path.exists(stdin_file):
1398 stdin = open(stdin_file, 'r')
1399 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1400
1401 script.close()
1402
1403 flags = copy.copy(getTestOpts().compiler_always_flags)
1404 if getTestOpts().outputdir != None:
1405 flags.extend(["-outputdir", getTestOpts().outputdir])
1406
1407 cmd = "'" + config.compiler + "' " \
1408 + join(flags,' ') + ' ' \
1409 + srcname + ' ' \
1410 + join(config.way_flags(name)[way],' ') + ' ' \
1411 + extra_hc_opts + ' ' \
1412 + getTestOpts().extra_hc_opts + ' ' \
1413 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1414
1415 if getTestOpts().cmd_wrapper != None:
1416 cmd = getTestOpts().cmd_wrapper(cmd);
1417
1418 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1419
1420 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1421
1422 exit_code = result >> 8
1423 signal = result & 0xff
1424
1425 # split the stdout into compilation/program output
1426 split_file(in_testdir(outname), delimiter,
1427 qualify(name, 'comp.stdout'),
1428 qualify(name, 'run.stdout'))
1429 split_file(in_testdir(errname), delimiter,
1430 qualify(name, 'comp.stderr'),
1431 qualify(name, 'run.stderr'))
1432
1433 # check the exit code
1434 if exit_code != getTestOpts().exit_code:
1435 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1436 dump_stdout(name)
1437 dump_stderr(name)
1438 return failBecause('bad exit code')
1439
1440 # ToDo: if the sub-shell was killed by ^C, then exit
1441
1442 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1443 check_stdout_ok(name)):
1444 return passed()
1445 else:
1446 return failBecause('bad stdout or stderr')
1447
1448
1449 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1450 infile = open(in_fn)
1451 out1 = open(out1_fn, 'w')
1452 out2 = open(out2_fn, 'w')
1453
1454 line = infile.readline()
1455 line = re.sub('\r', '', line) # ignore Windows EOL
1456 while (re.sub('^\s*','',line) != delimiter and line != ''):
1457 out1.write(line)
1458 line = infile.readline()
1459 line = re.sub('\r', '', line)
1460 out1.close()
1461
1462 line = infile.readline()
1463 while (line != ''):
1464 out2.write(line)
1465 line = infile.readline()
1466 out2.close()
1467
1468 # -----------------------------------------------------------------------------
1469 # Generate External Core for the given program, then compile the resulting Core
1470 # and compare its output to the expected output
1471
1472 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1473
1474 depsfilename = qualify(name, 'deps')
1475 errname = add_suffix(name, 'comp.stderr')
1476 qerrname = qualify(errname,'')
1477
1478 hcname = qualify(name, 'hc')
1479 oname = qualify(name, 'o')
1480
1481 rm_no_fail( qerrname )
1482 rm_no_fail( qualify(name, '') )
1483
1484 if (top_mod == ''):
1485 srcname = add_hs_lhs_suffix(name)
1486 else:
1487 srcname = top_mod
1488
1489 qcorefilename = qualify(name, 'hcr')
1490 corefilename = add_suffix(name, 'hcr')
1491 rm_no_fail(qcorefilename)
1492
1493 # Generate External Core
1494
1495 if (top_mod == ''):
1496 to_do = ' ' + srcname + ' '
1497 else:
1498 to_do = ' --make ' + top_mod + ' '
1499
1500 flags = copy.copy(getTestOpts().compiler_always_flags)
1501 if getTestOpts().outputdir != None:
1502 flags.extend(["-outputdir", getTestOpts().outputdir])
1503 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1504 + config.compiler + "' " \
1505 + join(flags,' ') + ' ' \
1506 + join(config.way_flags(name)[way],' ') + ' ' \
1507 + extra_hc_opts + ' ' \
1508 + getTestOpts().extra_hc_opts \
1509 + to_do \
1510 + '>' + errname + ' 2>&1'
1511 result = runCmdFor(name, cmd)
1512
1513 exit_code = result >> 8
1514
1515 if exit_code != 0:
1516 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1517 if_verbose_dump(1,qerrname)
1518 return failBecause('ext core exit code non-0')
1519
1520 # Compile the resulting files -- if there's more than one module, we need to read the output
1521 # of the previous compilation in order to find the dependencies
1522 if (top_mod == ''):
1523 to_compile = corefilename
1524 else:
1525 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1526 deps = open(depsfilename).read()
1527 deplist = string.replace(deps, '\n',' ');
1528 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1529 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1530
1531 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1532 if getTestOpts().outputdir != None:
1533 flags.extend(["-outputdir", getTestOpts().outputdir])
1534
1535 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1536 + config.compiler + "' " \
1537 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1538 + to_compile + ' ' \
1539 + extra_hc_opts + ' ' \
1540 + getTestOpts().extra_hc_opts + ' ' \
1541 + flags \
1542 + ' -fglasgow-exts -o ' + name \
1543 + '>' + errname + ' 2>&1'
1544
1545 result = runCmdFor(name, cmd)
1546 exit_code = result >> 8
1547
1548 if exit_code != 0:
1549 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1550 if_verbose_dump(1,qerrname)
1551 return failBecause('ext core exit code non-0')
1552
1553 # Clean up
1554 rm_no_fail ( oname )
1555 rm_no_fail ( hcname )
1556 rm_no_fail ( qcorefilename )
1557 rm_no_fail ( depsfilename )
1558
1559 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1560
1561 # -----------------------------------------------------------------------------
1562 # Utils
1563
1564 def check_stdout_ok( name ):
1565 if getTestOpts().with_namebase == None:
1566 namebase = name
1567 else:
1568 namebase = getTestOpts().with_namebase
1569
1570 actual_stdout_file = qualify(name, 'run.stdout')
1571 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1572
1573 def norm(str):
1574 if platform_specific:
1575 return str
1576 else:
1577 return normalise_output(str)
1578
1579 return compare_outputs('stdout', \
1580 two_normalisers(norm, getTestOpts().extra_normaliser), \
1581 expected_stdout_file, actual_stdout_file)
1582
1583 def dump_stdout( name ):
1584 print 'Stdout:'
1585 print read_no_crs(qualify(name, 'run.stdout'))
1586
1587 def check_stderr_ok( name ):
1588 if getTestOpts().with_namebase == None:
1589 namebase = name
1590 else:
1591 namebase = getTestOpts().with_namebase
1592
1593 actual_stderr_file = qualify(name, 'run.stderr')
1594 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1595
1596 def norm(str):
1597 if platform_specific:
1598 return str
1599 else:
1600 return normalise_errmsg(str)
1601
1602 return compare_outputs('stderr', \
1603 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1604 expected_stderr_file, actual_stderr_file)
1605
1606 def dump_stderr( name ):
1607 print "Stderr:"
1608 print read_no_crs(qualify(name, 'run.stderr'))
1609
1610 def read_no_crs(file):
1611 str = ''
1612 try:
1613 h = open(file)
1614 str = h.read()
1615 h.close
1616 except:
1617 # On Windows, if the program fails very early, it seems the
1618 # files stdout/stderr are redirected to may not get created
1619 pass
1620 return re.sub('\r', '', str)
1621
1622 def write_file(file, str):
1623 h = open(file, 'w')
1624 h.write(str)
1625 h.close
1626
1627 def check_hp_ok(name):
1628
1629 # do not qualify for hp2ps because we should be in the right directory
1630 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1631
1632 hp2psResult = runCmdExitCode(hp2psCmd)
1633
1634 actual_ps_file = qualify(name, 'ps')
1635
1636 if(hp2psResult == 0):
1637 if (os.path.exists(actual_ps_file)):
1638 if gs_working:
1639 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1640 if (gsResult == 0):
1641 return (True)
1642 else:
1643 print "hp2ps output for " + name + "is not valid PostScript"
1644 else: return (True) # assume postscript is valid without ghostscript
1645 else:
1646 print "hp2ps did not generate PostScript for " + name
1647 return (False)
1648 else:
1649 print "hp2ps error when processing heap profile for " + name
1650 return(False)
1651
1652 def check_prof_ok(name):
1653
1654 prof_file = qualify(name,'prof')
1655
1656 if not os.path.exists(prof_file):
1657 print prof_file + " does not exist"
1658 return(False)
1659
1660 if os.path.getsize(qualify(name,'prof')) == 0:
1661 print prof_file + " is empty"
1662 return(False)
1663
1664 if getTestOpts().with_namebase == None:
1665 namebase = name
1666 else:
1667 namebase = getTestOpts().with_namebase
1668
1669 (platform_specific, expected_prof_file) = \
1670 platform_wordsize_qualify(namebase, 'prof.sample')
1671
1672 # sample prof file is not required
1673 if not os.path.exists(expected_prof_file):
1674 return True
1675 else:
1676 return compare_outputs('prof', \
1677 two_normalisers(normalise_whitespace,normalise_prof), \
1678 expected_prof_file, prof_file)
1679
1680 # Compare expected output to actual output, and optionally accept the
1681 # new output. Returns true if output matched or was accepted, false
1682 # otherwise.
1683 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1684 if os.path.exists(expected_file):
1685 expected_raw = read_no_crs(expected_file)
1686 # print "norm:", normaliser(expected_raw)
1687 expected_str = normaliser(expected_raw)
1688 expected_file_for_diff = expected_file
1689 else:
1690 expected_str = ''
1691 expected_file_for_diff = '/dev/null'
1692
1693 actual_raw = read_no_crs(actual_file)
1694 actual_str = normaliser(actual_raw)
1695
1696 if expected_str == actual_str:
1697 return 1
1698 else:
1699 print 'Actual ' + kind + ' output differs from expected:'
1700
1701 if expected_file_for_diff == '/dev/null':
1702 expected_normalised_file = '/dev/null'
1703 else:
1704 expected_normalised_file = expected_file + ".normalised"
1705 write_file(expected_normalised_file, expected_str)
1706
1707 actual_normalised_file = actual_file + ".normalised"
1708 write_file(actual_normalised_file, actual_str)
1709
1710 # Ignore whitespace when diffing. We should only get to this
1711 # point if there are non-whitespace differences
1712 #
1713 # Note we are diffing the *actual* output, not the normalised
1714 # output. The normalised output may have whitespace squashed
1715 # (including newlines) so the diff would be hard to read.
1716 # This does mean that the diff might contain changes that
1717 # would be normalised away.
1718 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1719 ' ' + actual_file )
1720
1721 # If for some reason there were no non-whitespace differences,
1722 # then do a full diff
1723 if r == 0:
1724 r = os.system( 'diff -u ' + expected_file_for_diff + \
1725 ' ' + actual_file )
1726
1727 if config.accept:
1728 print 'Accepting new output.'
1729 write_file(expected_file, actual_raw)
1730 return 1
1731 else:
1732 return 0
1733
1734
1735 def normalise_whitespace( str ):
1736 # Merge contiguous whitespace characters into a single space.
1737 str = re.sub('[ \t\n]+', ' ', str)
1738 return str
1739
1740 def normalise_errmsg( str ):
1741 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1742 # the colon is there because it appears in error messages; this
1743 # hacky solution is used in place of more sophisticated filename
1744 # mangling
1745 str = re.sub('([^\\s])\\.exe', '\\1', str)
1746 # normalise slashes, minimise Windows/Unix filename differences
1747 str = re.sub('\\\\', '/', str)
1748 # The inplace ghc's are called ghc-stage[123] to avoid filename
1749 # collisions, so we need to normalise that to just "ghc"
1750 str = re.sub('ghc-stage[123]', 'ghc', str)
1751 # We sometimes see the name of the integer-gmp package on stderr,
1752 # but this can change (either the implementation name or the
1753 # version number), so we canonicalise it here
1754 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1755 return str
1756
1757 # normalise a .prof file, so that we can reasonably compare it against
1758 # a sample. This doesn't compare any of the actual profiling data,
1759 # only the shape of the profile and the number of entries.
1760 def normalise_prof (str):
1761 # strip everything up to the line beginning "COST CENTRE"
1762 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1763
1764 # strip results for CAFs, these tend to change unpredictably
1765 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1766
1767 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1768 # sometimes under MAIN.
1769 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1770
1771 # We have somthing like this:
1772
1773 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1774 # k Main 204 1 0.0 0.0 0.0 0.0
1775 # foo Main 205 1 0.0 0.0 0.0 0.0
1776 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1777
1778 # then we remove all the specific profiling data, leaving only the
1779 # cost centre name, module, and entries, to end up with this:
1780
1781 # MAIN MAIN 0
1782 # k Main 1
1783 # foo Main 1
1784 # foo.bar Main 1
1785
1786 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1787 return str
1788
1789 def normalise_slashes_( str ):
1790 str = re.sub('\\\\', '/', str)
1791 return str
1792
1793 def normalise_exe_( str ):
1794 str = re.sub('\.exe', '', str)
1795 return str
1796
1797 def normalise_output( str ):
1798 # Remove a .exe extension (for Windows)
1799 # This can occur in error messages generated by the program.
1800 str = re.sub('([^\\s])\\.exe', '\\1', str)
1801 return str
1802
1803 def normalise_asm( str ):
1804 lines = str.split('\n')
1805 # Only keep instructions and labels not starting with a dot.
1806 metadata = re.compile('^[ \t]*\\..*$')
1807 out = []
1808 for line in lines:
1809 # Drop metadata directives (e.g. ".type")
1810 if not metadata.match(line):
1811 line = re.sub('@plt', '', line)
1812 instr = line.lstrip().split()
1813 # Drop empty lines.
1814 if not instr:
1815 continue
1816 # Drop operands, except for call instructions.
1817 elif instr[0] == 'call':
1818 out.append(instr[0] + ' ' + instr[1])
1819 else:
1820 out.append(instr[0])
1821 out = '\n'.join(out)
1822 return out
1823
1824 def if_verbose( n, str ):
1825 if config.verbose >= n:
1826 print str
1827
1828 def if_verbose_dump( n, f ):
1829 if config.verbose >= n:
1830 try:
1831 print open(f).read()
1832 except:
1833 print ''
1834
1835 def rawSystem(cmd_and_args):
1836 # We prefer subprocess.call to os.spawnv as the latter
1837 # seems to send its arguments through a shell or something
1838 # with the Windows (non-cygwin) python. An argument "a b c"
1839 # turns into three arguments ["a", "b", "c"].
1840
1841 # However, subprocess is new in python 2.4, so fall back to
1842 # using spawnv if we don't have it
1843
1844 if have_subprocess:
1845 return subprocess.call(cmd_and_args)
1846 else:
1847 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1848
1849 # Note that this doesn't handle the timeout itself; it is just used for
1850 # commands that have timeout handling built-in.
1851 def rawSystemWithTimeout(cmd_and_args):
1852 r = rawSystem(cmd_and_args)
1853 if r == 98:
1854 # The python timeout program uses 98 to signal that ^C was pressed
1855 stopNow()
1856 return r
1857
1858 # cmd is a complex command in Bourne-shell syntax
1859 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1860 # Hence it must ultimately be run by a Bourne shell
1861 #
1862 # Mostly it invokes the command wrapped in 'timeout' thus
1863 # timeout 300 'cd . && ...blah blah'
1864 # so it's timeout's job to invoke the Bourne shell
1865 #
1866 # But watch out for the case when there is no timeout program!
1867 # Then, when using the native Python, os.system will invoke the cmd shell
1868
1869 def runCmd( cmd ):
1870 if_verbose( 1, cmd )
1871 r = 0
1872 if config.os == 'mingw32':
1873 # On MinGW, we will always have timeout
1874 assert config.timeout_prog!=''
1875
1876 if config.timeout_prog != '':
1877 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1878 else:
1879 r = os.system(cmd)
1880 return r << 8
1881
1882 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1883 if_verbose( 1, cmd )
1884 r = 0
1885 if config.os == 'mingw32':
1886 # On MinGW, we will always have timeout
1887 assert config.timeout_prog!=''
1888 timeout = int(ceil(config.timeout * timeout_multiplier))
1889
1890 if config.timeout_prog != '':
1891 if config.check_files_written:
1892 fn = name + ".strace"
1893 r = rawSystemWithTimeout(
1894 ["strace", "-o", fn, "-fF",
1895 "-e", "creat,open,chdir,clone,vfork",
1896 config.timeout_prog, str(timeout), cmd])
1897 addTestFilesWritten(name, fn)
1898 rm_no_fail(fn)
1899 else:
1900 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1901 else:
1902 r = os.system(cmd)
1903 return r << 8
1904
1905 def runCmdExitCode( cmd ):
1906 return (runCmd(cmd) >> 8);
1907
1908
1909 # -----------------------------------------------------------------------------
1910 # checking for files being written to by multiple tests
1911
1912 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1913 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1914 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1915 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1916 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1917 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1918 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1919 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1920 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1921 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1922 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1923 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1924 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1925 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1926 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1927 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1928 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1929
1930 # Files that are read or written but shouldn't be:
1931 # * ghci_history shouldn't be read or written by tests
1932 # * things under package.conf.d shouldn't be written by tests
1933 bad_file_usages = {}
1934
1935 # Mapping from tests to the list of files that they write
1936 files_written = {}
1937
1938 # Mapping from tests to the list of files that they write but don't clean
1939 files_written_not_removed = {}
1940
1941 def add_bad_file_usage(name, file):
1942 try:
1943 if not file in bad_file_usages[name]:
1944 bad_file_usages[name].append(file)
1945 except:
1946 bad_file_usages[name] = [file]
1947
1948 def mkPath(curdir, path):
1949 # Given the current full directory is 'curdir', what is the full
1950 # path to 'path'?
1951 return os.path.realpath(os.path.join(curdir, path))
1952
1953 def addTestFilesWritten(name, fn):
1954 if config.use_threads:
1955 with t.lockFilesWritten:
1956 addTestFilesWrittenHelper(name, fn)
1957 else:
1958 addTestFilesWrittenHelper(name, fn)
1959
1960 def addTestFilesWrittenHelper(name, fn):
1961 started = False
1962 working_directories = {}
1963
1964 with open(fn, 'r') as f:
1965 for line in f:
1966 m_pid = re_strace_pid.match(line)
1967 if m_pid:
1968 pid = m_pid.group(1)
1969 content = m_pid.group(2)
1970 elif re_strace_unavailable.match(line):
1971 next
1972 else:
1973 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1974
1975 m_open = re_strace_open.match(content)
1976 m_chdir = re_strace_chdir.match(content)
1977 m_clone = re_strace_clone.match(content)
1978 m_vfork = re_strace_vfork.match(content)
1979
1980 if not started:
1981 working_directories[pid] = os.getcwd()
1982 started = True
1983
1984 if m_open:
1985 file = m_open.group(1)
1986 file = mkPath(working_directories[pid], file)
1987 if file.endswith("ghci_history"):
1988 add_bad_file_usage(name, file)
1989 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1990 flags = m_open.group(2).split('|')
1991 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1992 if re.match('package\.conf\.d', file):
1993 add_bad_file_usage(name, file)
1994 else:
1995 try:
1996 if not file in files_written[name]:
1997 files_written[name].append(file)
1998 except:
1999 files_written[name] = [file]
2000 elif 'O_RDONLY' in flags:
2001 pass
2002 else:
2003 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2004 elif m_chdir:
2005 # We optimistically assume that unfinished chdir's are going to succeed
2006 dir = m_chdir.group(1)
2007 working_directories[pid] = mkPath(working_directories[pid], dir)
2008 elif m_clone:
2009 working_directories[m_clone.group(2)] = working_directories[pid]
2010 elif m_vfork:
2011 working_directories[m_vfork.group(2)] = working_directories[pid]
2012 elif re_strace_open_resumed.match(content):
2013 pass
2014 elif re_strace_chdir_resumed.match(content):
2015 pass
2016 elif re_strace_vfork_unfinished.match(content):
2017 pass
2018 elif re_strace_clone_unfinished.match(content):
2019 pass
2020 elif re_strace_ignore_sigchild.match(content):
2021 pass
2022 elif re_strace_ignore_sigvtalarm.match(content):
2023 pass
2024 elif re_strace_ignore_sigint.match(content):
2025 pass
2026 elif re_strace_ignore_sigfpe.match(content):
2027 pass
2028 elif re_strace_ignore_sigsegv.match(content):
2029 pass
2030 elif re_strace_ignore_sigpipe.match(content):
2031 pass
2032 else:
2033 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2034
2035 def checkForFilesWrittenProblems(file):
2036 foundProblem = False
2037
2038 files_written_inverted = {}
2039 for t in files_written.keys():
2040 for f in files_written[t]:
2041 try:
2042 files_written_inverted[f].append(t)
2043 except:
2044 files_written_inverted[f] = [t]
2045
2046 for f in files_written_inverted.keys():
2047 if len(files_written_inverted[f]) > 1:
2048 if not foundProblem:
2049 foundProblem = True
2050 file.write("\n")
2051 file.write("\nSome files are written by multiple tests:\n")
2052 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2053 if foundProblem:
2054 file.write("\n")
2055
2056 # -----
2057
2058 if len(files_written_not_removed) > 0:
2059 file.write("\n")
2060 file.write("\nSome files written but not removed:\n")
2061 tests = files_written_not_removed.keys()
2062 tests.sort()
2063 for t in tests:
2064 for f in files_written_not_removed[t]:
2065 file.write(" " + t + ": " + f + "\n")
2066 file.write("\n")
2067
2068 # -----
2069
2070 if len(bad_file_usages) > 0:
2071 file.write("\n")
2072 file.write("\nSome bad file usages:\n")
2073 tests = bad_file_usages.keys()
2074 tests.sort()
2075 for t in tests:
2076 for f in bad_file_usages[t]:
2077 file.write(" " + t + ": " + f + "\n")
2078 file.write("\n")
2079
2080 # -----------------------------------------------------------------------------
2081 # checking if ghostscript is available for checking the output of hp2ps
2082
2083 def genGSCmd(psfile):
2084 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2085
2086 def gsNotWorking():
2087 global gs_working
2088 print "GhostScript not available for hp2ps tests"
2089
2090 global gs_working
2091 gs_working = 0
2092 if config.have_profiling:
2093 if config.gs != '':
2094 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2095 if resultGood == 0:
2096 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2097 if resultBad != 0:
2098 print "GhostScript available for hp2ps tests"
2099 gs_working = 1;
2100 else:
2101 gsNotWorking();
2102 else:
2103 gsNotWorking();
2104 else:
2105 gsNotWorking();
2106
2107 def rm_no_fail( file ):
2108 try:
2109 os.remove( file )
2110 finally:
2111 return
2112
2113 def add_suffix( name, suffix ):
2114 if suffix == '':
2115 return name
2116 else:
2117 return name + '.' + suffix
2118
2119 def add_hs_lhs_suffix(name):
2120 if getTestOpts().c_src:
2121 return add_suffix(name, 'c')
2122 elif getTestOpts().cmm_src:
2123 return add_suffix(name, 'cmm')
2124 elif getTestOpts().objc_src:
2125 return add_suffix(name, 'm')
2126 elif getTestOpts().objcpp_src:
2127 return add_suffix(name, 'mm')
2128 elif getTestOpts().literate:
2129 return add_suffix(name, 'lhs')
2130 else:
2131 return add_suffix(name, 'hs')
2132
2133 def replace_suffix( name, suffix ):
2134 base, suf = os.path.splitext(name)
2135 return base + '.' + suffix
2136
2137 def in_testdir( name ):
2138 return (getTestOpts().testdir + '/' + name)
2139
2140 def qualify( name, suff ):
2141 return in_testdir(add_suffix(name, suff))
2142
2143
2144 # Finding the sample output. The filename is of the form
2145 #
2146 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2147 #
2148 # and we pick the most specific version available. The <version> is
2149 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2150 # more fine-grained control use if_compiler_lt().
2151 #
2152 def platform_wordsize_qualify( name, suff ):
2153
2154 basepath = qualify(name, suff)
2155
2156 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2157 for (platformSpecific, plat) in [(1, '-' + config.platform),
2158 (1, '-' + config.os),
2159 (0, '')]
2160 for ws in ['-ws-' + config.wordsize, '']
2161 for comp in ['-' + config.compiler_type, '']
2162 for vers in ['-' + config.compiler_maj_version, '']]
2163
2164 dir = glob.glob(basepath + '*')
2165 dir = map (lambda d: normalise_slashes_(d), dir)
2166
2167 for (platformSpecific, f) in paths:
2168 if f in dir:
2169 return (platformSpecific,f)
2170
2171 return (0, basepath)
2172
2173 # Clean up prior to the test, so that we can't spuriously conclude
2174 # that it passed on the basis of old run outputs.
2175 def pretest_cleanup(name):
2176 if getTestOpts().outputdir != None:
2177 odir = in_testdir(getTestOpts().outputdir)
2178 try:
2179 shutil.rmtree(odir)
2180 except:
2181 pass
2182 os.mkdir(odir)
2183
2184 rm_no_fail(qualify(name,'comp.stderr'))
2185 rm_no_fail(qualify(name,'run.stderr'))
2186 rm_no_fail(qualify(name,'run.stdout'))
2187 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2188 # simple_build zaps the following:
2189 # rm_nofail(qualify("o"))
2190 # rm_nofail(qualify(""))
2191 # not interested in the return code
2192
2193 # -----------------------------------------------------------------------------
2194 # Return a list of all the files ending in '.T' below the directory dir.
2195
2196 def findTFiles(roots):
2197 return concat(map(findTFiles_,roots))
2198
2199 def findTFiles_(path):
2200 if os.path.isdir(path):
2201 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2202 return findTFiles(paths)
2203 elif path[-2:] == '.T':
2204 return [path]
2205 else:
2206 return []
2207
2208 # -----------------------------------------------------------------------------
2209 # Output a test summary to the specified file object
2210
2211 def summary(t, file):
2212
2213 file.write('\n')
2214 file.write('OVERALL SUMMARY for test run started at ' \
2215 + t.start_time + '\n'\
2216 + string.rjust(`t.total_tests`, 8) \
2217 + ' total tests, which gave rise to\n' \
2218 + string.rjust(`t.total_test_cases`, 8) \
2219 + ' test cases, of which\n' \
2220 + string.rjust(`t.n_framework_failures`, 8) \
2221 + ' caused framework failures\n' \
2222 + string.rjust(`t.n_tests_skipped`, 8)
2223 + ' were skipped\n\n' \
2224 + string.rjust(`t.n_expected_passes`, 8)
2225 + ' expected passes\n' \
2226 + string.rjust(`t.n_missing_libs`, 8)
2227 + ' had missing libraries\n' \
2228 + string.rjust(`t.n_expected_failures`, 8) \
2229 + ' expected failures\n' \
2230 + string.rjust(`t.n_unexpected_passes`, 8) \
2231 + ' unexpected passes\n'
2232 + string.rjust(`t.n_unexpected_failures`, 8) \
2233 + ' unexpected failures\n'
2234 + '\n')
2235
2236 if t.n_unexpected_passes > 0:
2237 file.write('Unexpected passes:\n')
2238 printPassingTestInfosSummary(file, t.unexpected_passes)
2239
2240 if t.n_unexpected_failures > 0:
2241 file.write('Unexpected failures:\n')
2242 printFailingTestInfosSummary(file, t.unexpected_failures)
2243
2244 if config.check_files_written:
2245 checkForFilesWrittenProblems(file)
2246
2247 if stopping():
2248 file.write('WARNING: Testsuite run was terminated early\n')
2249
2250 def printPassingTestInfosSummary(file, testInfos):
2251 directories = testInfos.keys()
2252 directories.sort()
2253 maxDirLen = max(map ((lambda x : len(x)), directories))
2254 for directory in directories:
2255 tests = testInfos[directory].keys()
2256 tests.sort()
2257 for test in tests:
2258 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2259 ' (' + join(testInfos[directory][test],',') + ')\n')
2260 file.write('\n')
2261
2262 def printFailingTestInfosSummary(file, testInfos):
2263 directories = testInfos.keys()
2264 directories.sort()
2265 maxDirLen = max(map ((lambda x : len(x)), directories))
2266 for directory in directories:
2267 tests = testInfos[directory].keys()
2268 tests.sort()
2269 for test in tests:
2270 reasons = testInfos[directory][test].keys()
2271 for reason in reasons:
2272 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2273 ' [' + reason + ']' + \
2274 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2275 file.write('\n')
2276
2277 def getStdout(cmd):
2278 if have_subprocess:
2279 p = subprocess.Popen(cmd,
2280 stdout=subprocess.PIPE,
2281 stderr=subprocess.PIPE)
2282 (stdout, stderr) = p.communicate()
2283 r = p.wait()
2284 if r != 0:
2285 raise Exception("Command failed: " + str(cmd))
2286 if stderr != '':
2287 raise Exception("stderr from command: " + str(cmd))
2288 return stdout
2289 else:
2290 raise Exception("Need subprocess to get stdout, but don't have it")