Define 'when' and 'unless' helpers
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for all the tests in the current "directory". After
44 # each test, we reset the options to these. To change the options for
45 # multiple tests, the function setTestOpts() below can be used to alter
46 # these options.
47 global thisdir_testopts
48 thisdir_testopts = TestOptions()
49
50 def getThisDirTestOpts():
51 return thisdir_testopts
52
53 # Options valid for the current test only (these get reset to
54 # testdir_testopts after each test).
55
56 global testopts_local
57 if config.use_threads:
58 testopts_local = threading.local()
59 else:
60 class TestOpts_Local:
61 pass
62 testopts_local = TestOpts_Local()
63
64 def getTestOpts():
65 return testopts_local.x
66
67 def setLocalTestOpts(opts):
68 global testopts_local
69 testopts_local.x=opts
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 f( thisdir_testopts );
75
76 # -----------------------------------------------------------------------------
77 # Canned setup functions for common cases. eg. for a test you might say
78 #
79 # test('test001', normal, compile, [''])
80 #
81 # to run it without any options, but change it to
82 #
83 # test('test001', expect_fail, compile, [''])
84 #
85 # to expect failure for this test.
86
87 def normal( opts ):
88 return;
89
90 def skip( opts ):
91 opts.skip = 1
92
93 def expect_fail( opts ):
94 opts.expect = 'fail';
95
96 def reqlib( lib ):
97 return lambda opts, l=lib: _reqlib (opts, l )
98
99 # Cache the results of looking to see if we have a library or not.
100 # This makes quite a difference, especially on Windows.
101 have_lib = {}
102
103 def _reqlib( opts, lib ):
104 if have_lib.has_key(lib):
105 got_it = have_lib[lib]
106 else:
107 if have_subprocess:
108 # By preference we use subprocess, as the alternative uses
109 # /dev/null which mingw doesn't have.
110 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' describe ' + lib
119 + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def expect_broken( bug ):
143 return lambda opts, b=bug: _expect_broken (opts, b )
144
145 def _expect_broken( opts, bug ):
146 opts.expect = 'fail';
147
148 def ignore_output( opts ):
149 opts.ignore_output = 1
150
151 def no_stdin( opts ):
152 opts.no_stdin = 1
153
154 def combined_output( opts ):
155 opts.combined_output = True
156
157 # -----
158
159 def expect_fail_for( ways ):
160 return lambda opts, w=ways: _expect_fail_for( opts, w )
161
162 def _expect_fail_for( opts, ways ):
163 opts.expect_fail_for = ways
164
165 def expect_broken_for( bug, ways ):
166 return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )
167
168 def _expect_broken_for( opts, bug, ways ):
169 opts.expect_fail_for = ways
170
171 # -----
172
173 def omit_ways( ways ):
174 return lambda opts, w=ways: _omit_ways( opts, w )
175
176 def _omit_ways( opts, ways ):
177 opts.omit_ways = ways
178
179 # -----
180
181 def only_ways( ways ):
182 return lambda opts, w=ways: _only_ways( opts, w )
183
184 def _only_ways( opts, ways ):
185 opts.only_ways = ways
186
187 # -----
188
189 def extra_ways( ways ):
190 return lambda opts, w=ways: _extra_ways( opts, w )
191
192 def _extra_ways( opts, ways ):
193 opts.extra_ways = ways
194
195 # -----
196
197 def omit_compiler_types( compiler_types ):
198 return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)
199
200 def _omit_compiler_types( opts, compiler_types ):
201 if config.compiler_type in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def only_compiler_types( compiler_types ):
207 return lambda opts, c=compiler_types: _only_compiler_types(opts, c)
208
209 def _only_compiler_types( opts, compiler_types ):
210 if config.compiler_type not in compiler_types:
211 opts.skip = 1
212
213 # -----
214
215 def set_stdin( file ):
216 return lambda opts, f=file: _set_stdin(opts, f);
217
218 def _set_stdin( opts, f ):
219 opts.stdin = f
220
221 # -----
222
223 def exit_code( val ):
224 return lambda opts, v=val: _exit_code(opts, v);
225
226 def _exit_code( opts, v ):
227 opts.exit_code = v
228
229 # -----
230
231 def timeout_multiplier( val ):
232 return lambda opts, v=val: _timeout_multiplier(opts, v)
233
234 def _timeout_multiplier( opts, v ):
235 opts.timeout_multiplier = v
236
237 # -----
238
239 def extra_run_opts( val ):
240 return lambda opts, v=val: _extra_run_opts(opts, v);
241
242 def _extra_run_opts( opts, v ):
243 opts.extra_run_opts = v
244
245 # -----
246
247 def extra_hc_opts( val ):
248 return lambda opts, v=val: _extra_hc_opts(opts, v);
249
250 def _extra_hc_opts( opts, v ):
251 opts.extra_hc_opts = v
252
253 # -----
254
255 def extra_clean( files ):
256 return lambda opts, v=files: _extra_clean(opts, v);
257
258 def _extra_clean( opts, v ):
259 opts.clean_files = v
260
261 # -----
262
263 def stats_range_field( field, min, max ):
264 return lambda opts, f=field, x=min, y=max: _stats_range_field(opts, f, x, y);
265
266 def _stats_range_field( opts, f, x, y ):
267 # copy the dictionary, as the config gets shared between all tests
268 opts.stats_range_fields = opts.stats_range_fields.copy()
269 opts.stats_range_fields[f] = (x, y)
270
271 def compiler_stats_range_field( field, min, max ):
272 return lambda opts, f=field, x=min, y=max: _compiler_stats_range_field(opts, f, x, y);
273
274 def _compiler_stats_range_field( opts, f, x, y ):
275 # copy the dictionary, as the config gets shared between all tests
276 opts.compiler_stats_range_fields = opts.compiler_stats_range_fields.copy()
277 opts.compiler_stats_range_fields[f] = (x, y)
278
279 # -----
280
281 def skip_if_no_ghci(opts):
282 if not ('ghci' in config.run_ways):
283 opts.skip = 1
284
285 # ----
286
287 def skip_if_fast(opts):
288 if config.fast:
289 opts.skip = 1
290
291 # -----
292
293 def when(b, f):
294 if b:
295 return f
296 else:
297 return normal
298
299 def unless(b, f):
300 return when(not b, f)
301
302 def if_platform( plat, f ):
303 if config.platform == plat:
304 return f
305 else:
306 return normal
307
308 def unless_platform( plat, f ):
309 if config.platform != plat:
310 return f
311 else:
312 return normal
313
314 def if_os( os, f ):
315 if config.os == os:
316 return f
317 else:
318 return normal
319
320 def unless_os( os, f ):
321 if config.os == os:
322 return normal
323 else:
324 return f
325
326 def if_arch( arch, f ):
327 if config.arch == arch:
328 return f
329 else:
330 return normal
331
332 def unless_arch( arch, f ):
333 if config.arch == arch:
334 return normal
335 else:
336 return f
337
338 def wordsize( ws ):
339 return config.wordsize == str(ws)
340
341 def if_unregisterised( f ):
342 if config.unregisterised:
343 return f
344 else:
345 return normal
346
347 def unless_unregisterised( f ):
348 if config.unregisterised:
349 return normal
350 else:
351 return f
352
353 def if_msys( f ):
354 if config.msys:
355 return f
356 else:
357 return normal
358
359 def if_cygwin( f ):
360 if config.cygwin:
361 return f
362 else:
363 return normal
364
365 def when_have_vanilla( f ):
366 if config.have_vanilla:
367 return f
368 else:
369 return normal
370
371 def unless_have_vanilla( f ):
372 if config.have_vanilla:
373 return normal
374 else:
375 return f
376
377 def when_have_dynamic( f ):
378 if config.have_dynamic:
379 return f
380 else:
381 return normal
382
383 def unless_have_dynamic( f ):
384 if config.have_dynamic:
385 return normal
386 else:
387 return f
388
389 def when_have_profiling( f ):
390 if config.have_profiling:
391 return f
392 else:
393 return normal
394
395 def unless_have_profiling( f ):
396 if config.have_profiling:
397 return normal
398 else:
399 return f
400
401 # ---
402
403 def if_ghci_dynamic( f ):
404 if config.ghc_dynamic_by_default:
405 return f
406 else:
407 return normal
408
409 def if_in_tree_compiler( f ):
410 if config.in_tree_compiler:
411 return f
412 else:
413 return normal
414
415 def unless_in_tree_compiler( f ):
416 if config.in_tree_compiler:
417 return normal
418 else:
419 return f
420
421 def if_compiler_type( compiler, f ):
422 if config.compiler_type == compiler:
423 return f
424 else:
425 return normal
426
427 def if_compiler_profiled( f ):
428 if config.compiler_profiled:
429 return f
430 else:
431 return normal
432
433 def unless_compiler_profiled( f ):
434 if config.compiler_profiled:
435 return normal
436 else:
437 return f
438
439 def if_compiler_lt( compiler, version, f ):
440 if config.compiler_type == compiler and \
441 version_lt(config.compiler_version, version):
442 return f
443 else:
444 return normal
445
446 def if_compiler_le( compiler, version, f ):
447 if config.compiler_type == compiler and \
448 version_le(config.compiler_version, version):
449 return f
450 else:
451 return normal
452
453 def if_compiler_gt( compiler, version, f ):
454 if config.compiler_type == compiler and \
455 version_gt(config.compiler_version, version):
456 return f
457 else:
458 return normal
459
460 def if_compiler_ge( compiler, version, f ):
461 if config.compiler_type == compiler and \
462 version_ge(config.compiler_version, version):
463 return f
464 else:
465 return normal
466
467 def if_compiler_debugged( f ):
468 if config.compiler_debugged:
469 return f
470 else:
471 return normal
472
473 def namebase( nb ):
474 return lambda opts, nb=nb: _namebase(opts, nb)
475
476 def _namebase( opts, nb ):
477 opts.with_namebase = nb
478
479 # ---
480
481 def if_tag( tag, f ):
482 if tag in config.compiler_tags:
483 return f
484 else:
485 return normal
486
487 def unless_tag( tag, f ):
488 if not (tag in config.compiler_tags):
489 return f
490 else:
491 return normal
492
493 # ---
494 def high_memory_usage(opts):
495 opts.alone = True
496
497 # ---
498 def literate( opts ):
499 opts.literate = 1;
500
501 def c_src( opts ):
502 opts.c_src = 1;
503
504 def objc_src( opts ):
505 opts.objc_src = 1;
506
507 def objcpp_src( opts ):
508 opts.objcpp_src = 1;
509
510 def cmm_src( opts ):
511 opts.cmm_src = 1;
512
513 def outputdir( odir ):
514 return lambda opts, d=odir: _outputdir(opts, d)
515
516 def _outputdir( opts, odir ):
517 opts.outputdir = odir;
518
519 # ----
520
521 def pre_cmd( cmd ):
522 return lambda opts, c=cmd: _pre_cmd(opts, cmd)
523
524 def _pre_cmd( opts, cmd ):
525 opts.pre_cmd = cmd
526
527 # ----
528
529 def clean_cmd( cmd ):
530 return lambda opts, c=cmd: _clean_cmd(opts, cmd)
531
532 def _clean_cmd( opts, cmd ):
533 opts.clean_cmd = cmd
534
535 # ----
536
537 def cmd_prefix( prefix ):
538 return lambda opts, p=prefix: _cmd_prefix(opts, prefix)
539
540 def _cmd_prefix( opts, prefix ):
541 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
542
543 # ----
544
545 def cmd_wrapper( fun ):
546 return lambda opts, f=fun: _cmd_wrapper(opts, fun)
547
548 def _cmd_wrapper( opts, fun ):
549 opts.cmd_wrapper = fun
550
551 # ----
552
553 def compile_cmd_prefix( prefix ):
554 return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)
555
556 def _compile_cmd_prefix( opts, prefix ):
557 opts.compile_cmd_prefix = prefix
558
559 # ----
560
561 def normalise_slashes( opts ):
562 opts.extra_normaliser = normalise_slashes_
563
564 def normalise_exe( opts ):
565 opts.extra_normaliser = normalise_exe_
566
567 def normalise_fun( fun ):
568 return lambda opts, f=fun: _normalise_fun(opts, f)
569
570 def _normalise_fun( opts, f ):
571 opts.extra_normaliser = f
572
573 def normalise_errmsg_fun( fun ):
574 return lambda opts, f=fun: _normalise_errmsg_fun(opts, f)
575
576 def _normalise_errmsg_fun( opts, f ):
577 opts.extra_errmsg_normaliser = f
578
579 def two_normalisers(f, g):
580 return lambda x, f=f, g=g: f(g(x))
581
582 # ----
583 # Function for composing two opt-fns together
584
585 def composes( fs ):
586 return reduce(lambda f, g: compose(f, g), fs)
587
588 def compose( f, g ):
589 return lambda opts, f=f, g=g: _compose(opts,f,g)
590
591 def _compose( opts, f, g ):
592 f(opts)
593 g(opts)
594
595 # -----------------------------------------------------------------------------
596 # The current directory of tests
597
598 def newTestDir( dir ):
599 global thisdir_testopts
600 # reset the options for this test directory
601 thisdir_testopts = copy.copy(default_testopts)
602 thisdir_testopts.testdir = dir
603 thisdir_testopts.compiler_always_flags = config.compiler_always_flags
604
605 # -----------------------------------------------------------------------------
606 # Actually doing tests
607
608 parallelTests = []
609 aloneTests = []
610 allTestNames = set([])
611
612 def runTest (opts, name, func, args):
613 ok = 0
614
615 if config.use_threads:
616 t.thread_pool.acquire()
617 try:
618 while config.threads<(t.running_threads+1):
619 t.thread_pool.wait()
620 t.running_threads = t.running_threads+1
621 ok=1
622 t.thread_pool.release()
623 thread.start_new_thread(test_common_thread, (name, opts, func, args))
624 except:
625 if not ok:
626 t.thread_pool.release()
627 else:
628 test_common_work (name, opts, func, args)
629
630 # name :: String
631 # setup :: TestOpts -> IO ()
632 def test (name, setup, func, args):
633 global aloneTests
634 global parallelTests
635 global allTestNames
636 if name in allTestNames:
637 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
638 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
639 framework_fail(name, 'bad_name', 'This test has an invalid name')
640 myTestOpts = copy.copy(thisdir_testopts)
641
642 if type(setup) is types.ListType:
643 setup = composes(setup)
644
645 setup(myTestOpts)
646
647 thisTest = lambda : runTest(myTestOpts, name, func, args)
648 if myTestOpts.alone:
649 aloneTests.append(thisTest)
650 else:
651 parallelTests.append(thisTest)
652 allTestNames.add(name)
653
654 if config.use_threads:
655 def test_common_thread(name, opts, func, args):
656 t.lock.acquire()
657 try:
658 test_common_work(name,opts,func,args)
659 finally:
660 t.lock.release()
661 t.thread_pool.acquire()
662 t.running_threads = t.running_threads - 1
663 t.thread_pool.notify()
664 t.thread_pool.release()
665
666 def get_package_cache_timestamp():
667 if config.package_conf_cache_file == '':
668 return 0.0
669 else:
670 try:
671 return os.stat(config.package_conf_cache_file).st_mtime
672 except:
673 return 0.0
674
675
676 def test_common_work (name, opts, func, args):
677 try:
678 t.total_tests = t.total_tests+1
679 setLocalTestOpts(opts)
680
681 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
682
683 # All the ways we might run this test
684 if func == compile or func == multimod_compile:
685 all_ways = config.compile_ways
686 elif func == compile_and_run or func == multimod_compile_and_run:
687 all_ways = config.run_ways
688 elif func == ghci_script:
689 if 'ghci' in config.run_ways:
690 all_ways = ['ghci']
691 else:
692 all_ways = []
693 else:
694 all_ways = ['normal']
695
696 # A test itself can request extra ways by setting opts.extra_ways
697 all_ways = all_ways + filter(lambda way: way not in all_ways,
698 opts.extra_ways)
699
700 t.total_test_cases = t.total_test_cases + len(all_ways)
701
702 ok_way = lambda way: \
703 not getTestOpts().skip \
704 and (config.only == [] or name in config.only) \
705 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
706 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
707 and way not in getTestOpts().omit_ways
708
709 # Which ways we are asked to skip
710 do_ways = filter (ok_way,all_ways)
711
712 # In fast mode, we skip all but one way
713 if config.fast and len(do_ways) > 0:
714 do_ways = [do_ways[0]]
715
716 if not config.clean_only:
717 # Run the required tests...
718 for way in do_ways:
719 if stopping():
720 break
721 do_test (name, way, func, args)
722
723 for way in all_ways:
724 if way not in do_ways:
725 skiptest (name,way)
726
727 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
728 clean(map (lambda suff: name + suff,
729 ['', '.exe', '.exe.manifest', '.genscript',
730 '.stderr.normalised', '.stdout.normalised',
731 '.run.stderr', '.run.stdout',
732 '.run.stderr.normalised', '.run.stdout.normalised',
733 '.comp.stderr', '.comp.stdout',
734 '.comp.stderr.normalised', '.comp.stdout.normalised',
735 '.interp.stderr', '.interp.stdout',
736 '.interp.stderr.normalised', '.interp.stdout.normalised',
737 '.stats', '.comp.stats',
738 '.hi', '.o', '.prof', '.exe.prof', '.hc',
739 '_stub.h', '_stub.c', '_stub.o',
740 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
741
742 if func == multi_compile or func == multi_compile_fail:
743 extra_mods = args[1]
744 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
745 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
746
747 clean(getTestOpts().clean_files)
748
749 if getTestOpts().outputdir != None:
750 odir = in_testdir(getTestOpts().outputdir)
751 try:
752 shutil.rmtree(odir)
753 except:
754 pass
755
756 try:
757 shutil.rmtree(in_testdir('.hpc.' + name))
758 except:
759 pass
760
761 try:
762 cleanCmd = getTestOpts().clean_cmd
763 if cleanCmd != None:
764 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
765 if result != 0:
766 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
767 except e:
768 framework_fail(name, 'cleaning', 'clean-command exception')
769
770 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
771
772 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
773 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
774
775 try:
776 for f in files_written[name]:
777 if os.path.exists(f):
778 try:
779 if not f in files_written_not_removed[name]:
780 files_written_not_removed[name].append(f)
781 except:
782 files_written_not_removed[name] = [f]
783 except:
784 pass
785 except Exception, e:
786 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
787
788 def clean(strs):
789 for str in strs:
790 for name in glob.glob(in_testdir(str)):
791 clean_full_path(name)
792
793 def clean_full_path(name):
794 try:
795 # Remove files...
796 os.remove(name)
797 except OSError, e1:
798 try:
799 # ... and empty directories
800 os.rmdir(name)
801 except OSError, e2:
802 # We don't want to fail here, but we do want to know
803 # what went wrong, so print out the exceptions.
804 # ENOENT isn't a problem, though, as we clean files
805 # that don't necessarily exist.
806 if e1.errno != errno.ENOENT:
807 print e1
808 if e2.errno != errno.ENOENT:
809 print e2
810
811 def do_test(name, way, func, args):
812 full_name = name + '(' + way + ')'
813
814 try:
815 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
816 str([t.n_unexpected_passes, \
817 t.n_unexpected_failures, \
818 t.n_framework_failures])
819
820 if config.use_threads:
821 t.lock.release()
822
823 try:
824 preCmd = getTestOpts().pre_cmd
825 if preCmd != None:
826 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
827 if result != 0:
828 framework_fail(name, way, 'pre-command failed: ' + str(result))
829 except e:
830 framework_fail(name, way, 'pre-command exception')
831
832 try:
833 result = apply(func, [name,way] + args)
834 finally:
835 if config.use_threads:
836 t.lock.acquire()
837
838 if getTestOpts().expect != 'pass' and \
839 getTestOpts().expect != 'fail' and \
840 getTestOpts().expect != 'missing-lib':
841 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
842
843 try:
844 passFail = result['passFail']
845 except:
846 passFail = 'No passFail found'
847
848 if passFail == 'pass':
849 if getTestOpts().expect == 'pass' \
850 and way not in getTestOpts().expect_fail_for:
851 t.n_expected_passes = t.n_expected_passes + 1
852 if name in t.expected_passes:
853 t.expected_passes[name].append(way)
854 else:
855 t.expected_passes[name] = [way]
856 else:
857 print '*** unexpected pass for', full_name
858 t.n_unexpected_passes = t.n_unexpected_passes + 1
859 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
860 elif passFail == 'fail':
861 if getTestOpts().expect == 'pass' \
862 and way not in getTestOpts().expect_fail_for:
863 print '*** unexpected failure for', full_name
864 t.n_unexpected_failures = t.n_unexpected_failures + 1
865 reason = result['reason']
866 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
867 else:
868 if getTestOpts().expect == 'missing-lib':
869 t.n_missing_libs = t.n_missing_libs + 1
870 if name in t.missing_libs:
871 t.missing_libs[name].append(way)
872 else:
873 t.missing_libs[name] = [way]
874 else:
875 t.n_expected_failures = t.n_expected_failures + 1
876 if name in t.expected_failures:
877 t.expected_failures[name].append(way)
878 else:
879 t.expected_failures[name] = [way]
880 else:
881 framework_fail(name, way, 'bad result ' + passFail)
882 except KeyboardInterrupt:
883 stopNow()
884 except:
885 framework_fail(name, way, 'do_test exception')
886 traceback.print_exc()
887
888 def addPassingTestInfo (testInfos, directory, name, way):
889 directory = re.sub('^\\.[/\\\\]', '', directory)
890
891 if not directory in testInfos:
892 testInfos[directory] = {}
893
894 if not name in testInfos[directory]:
895 testInfos[directory][name] = []
896
897 testInfos[directory][name].append(way)
898
899 def addFailingTestInfo (testInfos, directory, name, reason, way):
900 directory = re.sub('^\\.[/\\\\]', '', directory)
901
902 if not directory in testInfos:
903 testInfos[directory] = {}
904
905 if not name in testInfos[directory]:
906 testInfos[directory][name] = {}
907
908 if not reason in testInfos[directory][name]:
909 testInfos[directory][name][reason] = []
910
911 testInfos[directory][name][reason].append(way)
912
913 def skiptest (name, way):
914 # print 'Skipping test \"', name, '\"'
915 t.n_tests_skipped = t.n_tests_skipped + 1
916 if name in t.tests_skipped:
917 t.tests_skipped[name].append(way)
918 else:
919 t.tests_skipped[name] = [way]
920
921 def framework_fail( name, way, reason ):
922 full_name = name + '(' + way + ')'
923 print '*** framework failure for', full_name, reason
924 t.n_framework_failures = t.n_framework_failures + 1
925 if name in t.framework_failures:
926 t.framework_failures[name].append(way)
927 else:
928 t.framework_failures[name] = [way]
929
930 def badResult(result):
931 try:
932 if result['passFail'] == 'pass':
933 return False
934 return True
935 except:
936 return True
937
938 def passed():
939 return {'passFail': 'pass'}
940
941 def failBecause(reason):
942 return {'passFail': 'fail', 'reason': reason}
943
944 # -----------------------------------------------------------------------------
945 # Generic command tests
946
947 # A generic command test is expected to run and exit successfully.
948 #
949 # The expected exit code can be changed via exit_code() as normal, and
950 # the expected stdout/stderr are stored in <testname>.stdout and
951 # <testname>.stderr. The output of the command can be ignored
952 # altogether by using run_command_ignore_output instead of
953 # run_command.
954
955 def run_command( name, way, cmd ):
956 return simple_run( name, '', cmd, '' )
957
958 # -----------------------------------------------------------------------------
959 # GHCi tests
960
961 def ghci_script( name, way, script ):
962 # filter out -fforce-recomp from compiler_always_flags, because we're
963 # actually testing the recompilation behaviour in the GHCi tests.
964 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
965 flags.append(getTestOpts().extra_hc_opts)
966 if getTestOpts().outputdir != None:
967 flags.extend(["-outputdir", getTestOpts().outputdir])
968
969 # We pass HC and HC_OPTS as environment variables, so that the
970 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
971 cmd = "HC='" + config.compiler + "' " + \
972 "HC_OPTS='" + join(flags,' ') + "' " + \
973 "'" + config.compiler + "'" + \
974 ' --interactive -v0 -ignore-dot-ghci ' + \
975 join(flags,' ')
976
977 getTestOpts().stdin = script
978 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
979
980 # -----------------------------------------------------------------------------
981 # Compile-only tests
982
983 def compile( name, way, extra_hc_opts ):
984 return do_compile( name, way, 0, '', [], extra_hc_opts )
985
986 def compile_fail( name, way, extra_hc_opts ):
987 return do_compile( name, way, 1, '', [], extra_hc_opts )
988
989 def multimod_compile( name, way, top_mod, extra_hc_opts ):
990 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
991
992 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
993 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
994
995 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
996 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
997
998 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
999 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1000
1001 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1002 # print 'Compile only, extra args = ', extra_hc_opts
1003 pretest_cleanup(name)
1004
1005 result = extras_build( way, extra_mods, extra_hc_opts )
1006 if badResult(result):
1007 return result
1008 extra_hc_opts = result['hc_opts']
1009
1010 force = 0
1011 if extra_mods:
1012 force = 1
1013 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1014
1015 if badResult(result):
1016 return result
1017
1018 # the actual stderr should always match the expected, regardless
1019 # of whether we expected the compilation to fail or not (successful
1020 # compilations may generate warnings).
1021
1022 if getTestOpts().with_namebase == None:
1023 namebase = name
1024 else:
1025 namebase = getTestOpts().with_namebase
1026
1027 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1028 actual_stderr_file = qualify(name, 'comp.stderr')
1029
1030 if not compare_outputs('stderr', \
1031 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1032 expected_stderr_file, actual_stderr_file):
1033 return failBecause('stderr mismatch')
1034
1035 # no problems found, this test passed
1036 return passed()
1037
1038 def compile_cmp_asm( name, way, extra_hc_opts ):
1039 print 'Compile only, extra args = ', extra_hc_opts
1040 pretest_cleanup(name)
1041 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1042
1043 if badResult(result):
1044 return result
1045
1046 # the actual stderr should always match the expected, regardless
1047 # of whether we expected the compilation to fail or not (successful
1048 # compilations may generate warnings).
1049
1050 if getTestOpts().with_namebase == None:
1051 namebase = name
1052 else:
1053 namebase = getTestOpts().with_namebase
1054
1055 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1056 actual_asm_file = qualify(name, 's')
1057
1058 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1059 expected_asm_file, actual_asm_file):
1060 return failBecause('asm mismatch')
1061
1062 # no problems found, this test passed
1063 return passed()
1064
1065 # -----------------------------------------------------------------------------
1066 # Compile-and-run tests
1067
1068 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1069 # print 'Compile and run, extra args = ', extra_hc_opts
1070 pretest_cleanup(name)
1071
1072 result = extras_build( way, extra_mods, extra_hc_opts )
1073 if badResult(result):
1074 return result
1075 extra_hc_opts = result['hc_opts']
1076
1077 if way == 'ghci': # interpreted...
1078 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1079 elif way == 'extcore' or way == 'optextcore' :
1080 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1081 else: # compiled...
1082 force = 0
1083 if extra_mods:
1084 force = 1
1085
1086 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1087 if badResult(result):
1088 return result
1089
1090 cmd = './' + name;
1091
1092 # we don't check the compiler's stderr for a compile-and-run test
1093 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1094
1095 def compile_and_run( name, way, extra_hc_opts ):
1096 return compile_and_run__( name, way, '', [], extra_hc_opts)
1097
1098 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1099 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1100
1101 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1102 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1103
1104 def stats( name, way, stats_file ):
1105 opts = getTestOpts()
1106 return checkStats(stats_file, opts.stats_range_fields)
1107
1108 # -----------------------------------------------------------------------------
1109 # Check -t stats info
1110
1111 def checkStats(stats_file, range_fields):
1112 result = passed()
1113 if len(range_fields) > 0:
1114 f = open(in_testdir(stats_file))
1115 contents = f.read()
1116 f.close()
1117
1118 for (field, (expected, dev)) in range_fields.items():
1119 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1120 if m == None:
1121 print 'Failed to find field: ', field
1122 result = failBecause('no such stats field')
1123 val = int(m.group(1))
1124
1125 lowerBound = trunc( expected * ((100 - float(dev))/100));
1126 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1127
1128 if val < lowerBound:
1129 print field, 'value is too low:'
1130 print '(If this is because you have improved GHC, please'
1131 print 'update the test so that GHC doesn\'t regress again)'
1132 result = failBecause('stat too good')
1133 if val > upperBound:
1134 print field, 'value is too high:'
1135 result = failBecause('stat not good enough')
1136
1137 if val < lowerBound or val > upperBound:
1138 valStr = str(val)
1139 valLen = len(valStr)
1140 expectedStr = str(expected)
1141 expectedLen = len(expectedStr)
1142 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1143 def display(descr, val, extra):
1144 print descr, string.rjust(str(val), length), extra
1145 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1146 display(' Lower bound ' + field + ':', lowerBound, '')
1147 display(' Upper bound ' + field + ':', upperBound, '')
1148 display(' Actual ' + field + ':', val, '')
1149
1150 return result
1151
1152 # -----------------------------------------------------------------------------
1153 # Build a single-module program
1154
1155 def extras_build( way, extra_mods, extra_hc_opts ):
1156 for modopts in extra_mods:
1157 mod, opts = modopts
1158 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1159 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1160 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1161 if badResult(result):
1162 return result
1163
1164 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1165
1166
1167 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1168 opts = getTestOpts()
1169 errname = add_suffix(name, 'comp.stderr')
1170 rm_no_fail( qualify(errname, '') )
1171
1172 if top_mod != '':
1173 srcname = top_mod
1174 rm_no_fail( qualify(name, '') )
1175 base, suf = os.path.splitext(top_mod)
1176 rm_no_fail( qualify(base, '') )
1177 rm_no_fail( qualify(base, 'exe') )
1178 elif addsuf:
1179 srcname = add_hs_lhs_suffix(name)
1180 rm_no_fail( qualify(name, '') )
1181 else:
1182 srcname = name
1183 rm_no_fail( qualify(name, 'o') )
1184
1185 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1186
1187 to_do = ''
1188 if top_mod != '':
1189 to_do = '--make '
1190 if link:
1191 to_do = to_do + '-o ' + name
1192 elif link:
1193 to_do = '-o ' + name
1194 elif opts.compile_to_hc:
1195 to_do = '-C'
1196 else:
1197 to_do = '-c' # just compile
1198
1199 stats_file = name + '.comp.stats'
1200 if len(opts.compiler_stats_range_fields) > 0:
1201 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1202
1203 # Required by GHC 7.3+, harmless for earlier versions:
1204 if (getTestOpts().c_src or
1205 getTestOpts().objc_src or
1206 getTestOpts().objcpp_src or
1207 getTestOpts().cmm_src):
1208 extra_hc_opts += ' -no-hs-main '
1209
1210 if getTestOpts().compile_cmd_prefix == '':
1211 cmd_prefix = ''
1212 else:
1213 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1214
1215 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1216 if noforce:
1217 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1218 if getTestOpts().outputdir != None:
1219 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1220
1221 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1222 + config.compiler + "' " \
1223 + join(comp_flags,' ') + ' ' \
1224 + to_do + ' ' + srcname + ' ' \
1225 + join(config.way_flags(name)[way],' ') + ' ' \
1226 + extra_hc_opts + ' ' \
1227 + opts.extra_hc_opts + ' ' \
1228 + '>' + errname + ' 2>&1'
1229
1230 result = runCmdFor(name, cmd)
1231
1232 if result != 0 and not should_fail:
1233 actual_stderr = qualify(name, 'comp.stderr')
1234 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1235 if_verbose_dump(1,actual_stderr)
1236
1237 # ToDo: if the sub-shell was killed by ^C, then exit
1238
1239 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1240
1241 if badResult(statsResult):
1242 return statsResult
1243
1244 if should_fail:
1245 if result == 0:
1246 return failBecause('exit code 0')
1247 else:
1248 if result != 0:
1249 return failBecause('exit code non-0')
1250
1251 return passed()
1252
1253 # -----------------------------------------------------------------------------
1254 # Run a program and check its output
1255 #
1256 # If testname.stdin exists, route input from that, else
1257 # from /dev/null. Route output to testname.run.stdout and
1258 # testname.run.stderr. Returns the exit code of the run.
1259
1260 def simple_run( name, way, prog, args ):
1261 opts = getTestOpts()
1262
1263 # figure out what to use for stdin
1264 if opts.stdin != '':
1265 use_stdin = opts.stdin
1266 else:
1267 stdin_file = add_suffix(name, 'stdin')
1268 if os.path.exists(in_testdir(stdin_file)):
1269 use_stdin = stdin_file
1270 else:
1271 use_stdin = '/dev/null'
1272
1273 run_stdout = add_suffix(name,'run.stdout')
1274 run_stderr = add_suffix(name,'run.stderr')
1275
1276 rm_no_fail(qualify(name,'run.stdout'))
1277 rm_no_fail(qualify(name,'run.stderr'))
1278 rm_no_fail(qualify(name, 'hp'))
1279 rm_no_fail(qualify(name,'ps'))
1280 rm_no_fail(qualify(name, 'prof'))
1281
1282 my_rts_flags = rts_flags(way)
1283
1284 stats_file = name + '.stats'
1285 if len(opts.stats_range_fields) > 0:
1286 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1287
1288 if opts.no_stdin:
1289 stdin_comes_from = ''
1290 else:
1291 stdin_comes_from = ' <' + use_stdin
1292
1293 if opts.combined_output:
1294 redirection = ' >' + run_stdout \
1295 + ' 2>&1'
1296 else:
1297 redirection = ' >' + run_stdout \
1298 + ' 2>' + run_stderr
1299
1300 cmd = prog + ' ' + args + ' ' \
1301 + my_rts_flags + ' ' \
1302 + stdin_comes_from \
1303 + redirection
1304
1305 if opts.cmd_wrapper != None:
1306 cmd = opts.cmd_wrapper(cmd);
1307
1308 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1309
1310 # run the command
1311 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1312
1313 exit_code = result >> 8
1314 signal = result & 0xff
1315
1316 # check the exit code
1317 if exit_code != opts.exit_code:
1318 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1319 dump_stdout(name)
1320 dump_stderr(name)
1321 return failBecause('bad exit code')
1322
1323 check_hp = my_rts_flags.find("-h") != -1
1324 check_prof = my_rts_flags.find("-p") != -1
1325
1326 if not opts.ignore_output:
1327 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1328 bad_stdout = not check_stdout_ok(name)
1329 if bad_stderr:
1330 return failBecause('bad stderr')
1331 if bad_stdout:
1332 return failBecause('bad stdout')
1333 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1334 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1335 return failBecause('bad heap profile')
1336 if check_prof and not check_prof_ok(name):
1337 return failBecause('bad profile')
1338
1339 return checkStats(stats_file, opts.stats_range_fields)
1340
1341 def rts_flags(way):
1342 if (way == ''):
1343 return ''
1344 else:
1345 args = config.way_rts_flags[way]
1346
1347 if args == []:
1348 return ''
1349 else:
1350 return '+RTS ' + join(args,' ') + ' -RTS'
1351
1352 # -----------------------------------------------------------------------------
1353 # Run a program in the interpreter and check its output
1354
1355 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1356 outname = add_suffix(name, 'interp.stdout')
1357 errname = add_suffix(name, 'interp.stderr')
1358 rm_no_fail(outname)
1359 rm_no_fail(errname)
1360 rm_no_fail(name)
1361
1362 if (top_mod == ''):
1363 srcname = add_hs_lhs_suffix(name)
1364 else:
1365 srcname = top_mod
1366
1367 scriptname = add_suffix(name, 'genscript')
1368 qscriptname = in_testdir(scriptname)
1369 rm_no_fail(qscriptname)
1370
1371 delimiter = '===== program output begins here\n'
1372
1373 script = open(qscriptname, 'w')
1374 if not compile_only:
1375 # set the prog name and command-line args to match the compiled
1376 # environment.
1377 script.write(':set prog ' + name + '\n')
1378 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1379 # Add marker lines to the stdout and stderr output files, so we
1380 # can separate GHCi's output from the program's.
1381 script.write(':! echo ' + delimiter)
1382 script.write(':! echo 1>&2 ' + delimiter)
1383 # Set stdout to be line-buffered to match the compiled environment.
1384 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1385 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1386 # in the event of an exception as for the compiled program.
1387 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1388 script.close()
1389
1390 # figure out what to use for stdin
1391 if getTestOpts().stdin != '':
1392 stdin_file = in_testdir(getTestOpts().stdin)
1393 else:
1394 stdin_file = qualify(name, 'stdin')
1395
1396 if os.path.exists(stdin_file):
1397 stdin = open(stdin_file, 'r')
1398 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1399
1400 script.close()
1401
1402 flags = copy.copy(getTestOpts().compiler_always_flags)
1403 if getTestOpts().outputdir != None:
1404 flags.extend(["-outputdir", getTestOpts().outputdir])
1405
1406 cmd = "'" + config.compiler + "' " \
1407 + join(flags,' ') + ' ' \
1408 + srcname + ' ' \
1409 + join(config.way_flags(name)[way],' ') + ' ' \
1410 + extra_hc_opts + ' ' \
1411 + getTestOpts().extra_hc_opts + ' ' \
1412 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1413
1414 if getTestOpts().cmd_wrapper != None:
1415 cmd = getTestOpts().cmd_wrapper(cmd);
1416
1417 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1418
1419 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1420
1421 exit_code = result >> 8
1422 signal = result & 0xff
1423
1424 # split the stdout into compilation/program output
1425 split_file(in_testdir(outname), delimiter,
1426 qualify(name, 'comp.stdout'),
1427 qualify(name, 'run.stdout'))
1428 split_file(in_testdir(errname), delimiter,
1429 qualify(name, 'comp.stderr'),
1430 qualify(name, 'run.stderr'))
1431
1432 # check the exit code
1433 if exit_code != getTestOpts().exit_code:
1434 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1435 dump_stdout(name)
1436 dump_stderr(name)
1437 return failBecause('bad exit code')
1438
1439 # ToDo: if the sub-shell was killed by ^C, then exit
1440
1441 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1442 check_stdout_ok(name)):
1443 return passed()
1444 else:
1445 return failBecause('bad stdout or stderr')
1446
1447
1448 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1449 infile = open(in_fn)
1450 out1 = open(out1_fn, 'w')
1451 out2 = open(out2_fn, 'w')
1452
1453 line = infile.readline()
1454 line = re.sub('\r', '', line) # ignore Windows EOL
1455 while (re.sub('^\s*','',line) != delimiter and line != ''):
1456 out1.write(line)
1457 line = infile.readline()
1458 line = re.sub('\r', '', line)
1459 out1.close()
1460
1461 line = infile.readline()
1462 while (line != ''):
1463 out2.write(line)
1464 line = infile.readline()
1465 out2.close()
1466
1467 # -----------------------------------------------------------------------------
1468 # Generate External Core for the given program, then compile the resulting Core
1469 # and compare its output to the expected output
1470
1471 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1472
1473 depsfilename = qualify(name, 'deps')
1474 errname = add_suffix(name, 'comp.stderr')
1475 qerrname = qualify(errname,'')
1476
1477 hcname = qualify(name, 'hc')
1478 oname = qualify(name, 'o')
1479
1480 rm_no_fail( qerrname )
1481 rm_no_fail( qualify(name, '') )
1482
1483 if (top_mod == ''):
1484 srcname = add_hs_lhs_suffix(name)
1485 else:
1486 srcname = top_mod
1487
1488 qcorefilename = qualify(name, 'hcr')
1489 corefilename = add_suffix(name, 'hcr')
1490 rm_no_fail(qcorefilename)
1491
1492 # Generate External Core
1493
1494 if (top_mod == ''):
1495 to_do = ' ' + srcname + ' '
1496 else:
1497 to_do = ' --make ' + top_mod + ' '
1498
1499 flags = copy.copy(getTestOpts().compiler_always_flags)
1500 if getTestOpts().outputdir != None:
1501 flags.extend(["-outputdir", getTestOpts().outputdir])
1502 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1503 + config.compiler + "' " \
1504 + join(flags,' ') + ' ' \
1505 + join(config.way_flags(name)[way],' ') + ' ' \
1506 + extra_hc_opts + ' ' \
1507 + getTestOpts().extra_hc_opts \
1508 + to_do \
1509 + '>' + errname + ' 2>&1'
1510 result = runCmdFor(name, cmd)
1511
1512 exit_code = result >> 8
1513
1514 if exit_code != 0:
1515 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1516 if_verbose_dump(1,qerrname)
1517 return failBecause('ext core exit code non-0')
1518
1519 # Compile the resulting files -- if there's more than one module, we need to read the output
1520 # of the previous compilation in order to find the dependencies
1521 if (top_mod == ''):
1522 to_compile = corefilename
1523 else:
1524 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1525 deps = open(depsfilename).read()
1526 deplist = string.replace(deps, '\n',' ');
1527 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1528 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1529
1530 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1531 if getTestOpts().outputdir != None:
1532 flags.extend(["-outputdir", getTestOpts().outputdir])
1533
1534 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1535 + config.compiler + "' " \
1536 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1537 + to_compile + ' ' \
1538 + extra_hc_opts + ' ' \
1539 + getTestOpts().extra_hc_opts + ' ' \
1540 + flags \
1541 + ' -fglasgow-exts -o ' + name \
1542 + '>' + errname + ' 2>&1'
1543
1544 result = runCmdFor(name, cmd)
1545 exit_code = result >> 8
1546
1547 if exit_code != 0:
1548 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1549 if_verbose_dump(1,qerrname)
1550 return failBecause('ext core exit code non-0')
1551
1552 # Clean up
1553 rm_no_fail ( oname )
1554 rm_no_fail ( hcname )
1555 rm_no_fail ( qcorefilename )
1556 rm_no_fail ( depsfilename )
1557
1558 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1559
1560 # -----------------------------------------------------------------------------
1561 # Utils
1562
1563 def check_stdout_ok( name ):
1564 if getTestOpts().with_namebase == None:
1565 namebase = name
1566 else:
1567 namebase = getTestOpts().with_namebase
1568
1569 actual_stdout_file = qualify(name, 'run.stdout')
1570 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1571
1572 def norm(str):
1573 if platform_specific:
1574 return str
1575 else:
1576 return normalise_output(str)
1577
1578 return compare_outputs('stdout', \
1579 two_normalisers(norm, getTestOpts().extra_normaliser), \
1580 expected_stdout_file, actual_stdout_file)
1581
1582 def dump_stdout( name ):
1583 print 'Stdout:'
1584 print read_no_crs(qualify(name, 'run.stdout'))
1585
1586 def check_stderr_ok( name ):
1587 if getTestOpts().with_namebase == None:
1588 namebase = name
1589 else:
1590 namebase = getTestOpts().with_namebase
1591
1592 actual_stderr_file = qualify(name, 'run.stderr')
1593 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1594
1595 def norm(str):
1596 if platform_specific:
1597 return str
1598 else:
1599 return normalise_errmsg(str)
1600
1601 return compare_outputs('stderr', \
1602 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1603 expected_stderr_file, actual_stderr_file)
1604
1605 def dump_stderr( name ):
1606 print "Stderr:"
1607 print read_no_crs(qualify(name, 'run.stderr'))
1608
1609 def read_no_crs(file):
1610 str = ''
1611 try:
1612 h = open(file)
1613 str = h.read()
1614 h.close
1615 except:
1616 # On Windows, if the program fails very early, it seems the
1617 # files stdout/stderr are redirected to may not get created
1618 pass
1619 return re.sub('\r', '', str)
1620
1621 def write_file(file, str):
1622 h = open(file, 'w')
1623 h.write(str)
1624 h.close
1625
1626 def check_hp_ok(name):
1627
1628 # do not qualify for hp2ps because we should be in the right directory
1629 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1630
1631 hp2psResult = runCmdExitCode(hp2psCmd)
1632
1633 actual_ps_file = qualify(name, 'ps')
1634
1635 if(hp2psResult == 0):
1636 if (os.path.exists(actual_ps_file)):
1637 if gs_working:
1638 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1639 if (gsResult == 0):
1640 return (True)
1641 else:
1642 print "hp2ps output for " + name + "is not valid PostScript"
1643 else: return (True) # assume postscript is valid without ghostscript
1644 else:
1645 print "hp2ps did not generate PostScript for " + name
1646 return (False)
1647 else:
1648 print "hp2ps error when processing heap profile for " + name
1649 return(False)
1650
1651 def check_prof_ok(name):
1652
1653 prof_file = qualify(name,'prof')
1654
1655 if not os.path.exists(prof_file):
1656 print prof_file + " does not exist"
1657 return(False)
1658
1659 if os.path.getsize(qualify(name,'prof')) == 0:
1660 print prof_file + " is empty"
1661 return(False)
1662
1663 if getTestOpts().with_namebase == None:
1664 namebase = name
1665 else:
1666 namebase = getTestOpts().with_namebase
1667
1668 (platform_specific, expected_prof_file) = \
1669 platform_wordsize_qualify(namebase, 'prof.sample')
1670
1671 # sample prof file is not required
1672 if not os.path.exists(expected_prof_file):
1673 return True
1674 else:
1675 return compare_outputs('prof', \
1676 two_normalisers(normalise_whitespace,normalise_prof), \
1677 expected_prof_file, prof_file)
1678
1679 # Compare expected output to actual output, and optionally accept the
1680 # new output. Returns true if output matched or was accepted, false
1681 # otherwise.
1682 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1683 if os.path.exists(expected_file):
1684 expected_raw = read_no_crs(expected_file)
1685 # print "norm:", normaliser(expected_raw)
1686 expected_str = normaliser(expected_raw)
1687 expected_file_for_diff = expected_file
1688 else:
1689 expected_str = ''
1690 expected_file_for_diff = '/dev/null'
1691
1692 actual_raw = read_no_crs(actual_file)
1693 actual_str = normaliser(actual_raw)
1694
1695 if expected_str == actual_str:
1696 return 1
1697 else:
1698 print 'Actual ' + kind + ' output differs from expected:'
1699
1700 if expected_file_for_diff == '/dev/null':
1701 expected_normalised_file = '/dev/null'
1702 else:
1703 expected_normalised_file = expected_file + ".normalised"
1704 write_file(expected_normalised_file, expected_str)
1705
1706 actual_normalised_file = actual_file + ".normalised"
1707 write_file(actual_normalised_file, actual_str)
1708
1709 # Ignore whitespace when diffing. We should only get to this
1710 # point if there are non-whitespace differences
1711 #
1712 # Note we are diffing the *actual* output, not the normalised
1713 # output. The normalised output may have whitespace squashed
1714 # (including newlines) so the diff would be hard to read.
1715 # This does mean that the diff might contain changes that
1716 # would be normalised away.
1717 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1718 ' ' + actual_file )
1719
1720 # If for some reason there were no non-whitespace differences,
1721 # then do a full diff
1722 if r == 0:
1723 r = os.system( 'diff -u ' + expected_file_for_diff + \
1724 ' ' + actual_file )
1725
1726 if config.accept:
1727 print 'Accepting new output.'
1728 write_file(expected_file, actual_raw)
1729 return 1
1730 else:
1731 return 0
1732
1733
1734 def normalise_whitespace( str ):
1735 # Merge contiguous whitespace characters into a single space.
1736 str = re.sub('[ \t\n]+', ' ', str)
1737 return str
1738
1739 def normalise_errmsg( str ):
1740 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1741 # the colon is there because it appears in error messages; this
1742 # hacky solution is used in place of more sophisticated filename
1743 # mangling
1744 str = re.sub('([^\\s])\\.exe', '\\1', str)
1745 # normalise slashes, minimise Windows/Unix filename differences
1746 str = re.sub('\\\\', '/', str)
1747 # The inplace ghc's are called ghc-stage[123] to avoid filename
1748 # collisions, so we need to normalise that to just "ghc"
1749 str = re.sub('ghc-stage[123]', 'ghc', str)
1750 # We sometimes see the name of the integer-gmp package on stderr,
1751 # but this can change (either the implementation name or the
1752 # version number), so we canonicalise it here
1753 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1754 return str
1755
1756 # normalise a .prof file, so that we can reasonably compare it against
1757 # a sample. This doesn't compare any of the actual profiling data,
1758 # only the shape of the profile and the number of entries.
1759 def normalise_prof (str):
1760 # strip everything up to the line beginning "COST CENTRE"
1761 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1762
1763 # strip results for CAFs, these tend to change unpredictably
1764 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1765
1766 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1767 # sometimes under MAIN.
1768 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1769
1770 # We have somthing like this:
1771
1772 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1773 # k Main 204 1 0.0 0.0 0.0 0.0
1774 # foo Main 205 1 0.0 0.0 0.0 0.0
1775 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1776
1777 # then we remove all the specific profiling data, leaving only the
1778 # cost centre name, module, and entries, to end up with this:
1779
1780 # MAIN MAIN 0
1781 # k Main 1
1782 # foo Main 1
1783 # foo.bar Main 1
1784
1785 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1786 return str
1787
1788 def normalise_slashes_( str ):
1789 str = re.sub('\\\\', '/', str)
1790 return str
1791
1792 def normalise_exe_( str ):
1793 str = re.sub('\.exe', '', str)
1794 return str
1795
1796 def normalise_output( str ):
1797 # Remove a .exe extension (for Windows)
1798 # This can occur in error messages generated by the program.
1799 str = re.sub('([^\\s])\\.exe', '\\1', str)
1800 return str
1801
1802 def normalise_asm( str ):
1803 lines = str.split('\n')
1804 # Only keep instructions and labels not starting with a dot.
1805 metadata = re.compile('^[ \t]*\\..*$')
1806 out = []
1807 for line in lines:
1808 # Drop metadata directives (e.g. ".type")
1809 if not metadata.match(line):
1810 line = re.sub('@plt', '', line)
1811 instr = line.lstrip().split()
1812 # Drop empty lines.
1813 if not instr:
1814 continue
1815 # Drop operands, except for call instructions.
1816 elif instr[0] == 'call':
1817 out.append(instr[0] + ' ' + instr[1])
1818 else:
1819 out.append(instr[0])
1820 out = '\n'.join(out)
1821 return out
1822
1823 def if_verbose( n, str ):
1824 if config.verbose >= n:
1825 print str
1826
1827 def if_verbose_dump( n, f ):
1828 if config.verbose >= n:
1829 try:
1830 print open(f).read()
1831 except:
1832 print ''
1833
1834 def rawSystem(cmd_and_args):
1835 # We prefer subprocess.call to os.spawnv as the latter
1836 # seems to send its arguments through a shell or something
1837 # with the Windows (non-cygwin) python. An argument "a b c"
1838 # turns into three arguments ["a", "b", "c"].
1839
1840 # However, subprocess is new in python 2.4, so fall back to
1841 # using spawnv if we don't have it
1842
1843 if have_subprocess:
1844 return subprocess.call(cmd_and_args)
1845 else:
1846 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1847
1848 # Note that this doesn't handle the timeout itself; it is just used for
1849 # commands that have timeout handling built-in.
1850 def rawSystemWithTimeout(cmd_and_args):
1851 r = rawSystem(cmd_and_args)
1852 if r == 98:
1853 # The python timeout program uses 98 to signal that ^C was pressed
1854 stopNow()
1855 return r
1856
1857 # cmd is a complex command in Bourne-shell syntax
1858 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1859 # Hence it must ultimately be run by a Bourne shell
1860 #
1861 # Mostly it invokes the command wrapped in 'timeout' thus
1862 # timeout 300 'cd . && ...blah blah'
1863 # so it's timeout's job to invoke the Bourne shell
1864 #
1865 # But watch out for the case when there is no timeout program!
1866 # Then, when using the native Python, os.system will invoke the cmd shell
1867
1868 def runCmd( cmd ):
1869 if_verbose( 1, cmd )
1870 r = 0
1871 if config.os == 'mingw32':
1872 # On MinGW, we will always have timeout
1873 assert config.timeout_prog!=''
1874
1875 if config.timeout_prog != '':
1876 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1877 else:
1878 r = os.system(cmd)
1879 return r << 8
1880
1881 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1882 if_verbose( 1, cmd )
1883 r = 0
1884 if config.os == 'mingw32':
1885 # On MinGW, we will always have timeout
1886 assert config.timeout_prog!=''
1887 timeout = int(ceil(config.timeout * timeout_multiplier))
1888
1889 if config.timeout_prog != '':
1890 if config.check_files_written:
1891 fn = name + ".strace"
1892 r = rawSystemWithTimeout(
1893 ["strace", "-o", fn, "-fF",
1894 "-e", "creat,open,chdir,clone,vfork",
1895 config.timeout_prog, str(timeout), cmd])
1896 addTestFilesWritten(name, fn)
1897 rm_no_fail(fn)
1898 else:
1899 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1900 else:
1901 r = os.system(cmd)
1902 return r << 8
1903
1904 def runCmdExitCode( cmd ):
1905 return (runCmd(cmd) >> 8);
1906
1907
1908 # -----------------------------------------------------------------------------
1909 # checking for files being written to by multiple tests
1910
1911 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1912 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1913 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1914 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1915 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1916 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1917 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1918 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1919 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1920 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1921 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1922 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1923 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1924 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1925 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1926 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1927 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1928
1929 # Files that are read or written but shouldn't be:
1930 # * ghci_history shouldn't be read or written by tests
1931 # * things under package.conf.d shouldn't be written by tests
1932 bad_file_usages = {}
1933
1934 # Mapping from tests to the list of files that they write
1935 files_written = {}
1936
1937 # Mapping from tests to the list of files that they write but don't clean
1938 files_written_not_removed = {}
1939
1940 def add_bad_file_usage(name, file):
1941 try:
1942 if not file in bad_file_usages[name]:
1943 bad_file_usages[name].append(file)
1944 except:
1945 bad_file_usages[name] = [file]
1946
1947 def mkPath(curdir, path):
1948 # Given the current full directory is 'curdir', what is the full
1949 # path to 'path'?
1950 return os.path.realpath(os.path.join(curdir, path))
1951
1952 def addTestFilesWritten(name, fn):
1953 if config.use_threads:
1954 with t.lockFilesWritten:
1955 addTestFilesWrittenHelper(name, fn)
1956 else:
1957 addTestFilesWrittenHelper(name, fn)
1958
1959 def addTestFilesWrittenHelper(name, fn):
1960 started = False
1961 working_directories = {}
1962
1963 with open(fn, 'r') as f:
1964 for line in f:
1965 m_pid = re_strace_pid.match(line)
1966 if m_pid:
1967 pid = m_pid.group(1)
1968 content = m_pid.group(2)
1969 elif re_strace_unavailable.match(line):
1970 next
1971 else:
1972 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1973
1974 m_open = re_strace_open.match(content)
1975 m_chdir = re_strace_chdir.match(content)
1976 m_clone = re_strace_clone.match(content)
1977 m_vfork = re_strace_vfork.match(content)
1978
1979 if not started:
1980 working_directories[pid] = os.getcwd()
1981 started = True
1982
1983 if m_open:
1984 file = m_open.group(1)
1985 file = mkPath(working_directories[pid], file)
1986 if file.endswith("ghci_history"):
1987 add_bad_file_usage(name, file)
1988 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1989 flags = m_open.group(2).split('|')
1990 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1991 if re.match('package\.conf\.d', file):
1992 add_bad_file_usage(name, file)
1993 else:
1994 try:
1995 if not file in files_written[name]:
1996 files_written[name].append(file)
1997 except:
1998 files_written[name] = [file]
1999 elif 'O_RDONLY' in flags:
2000 pass
2001 else:
2002 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2003 elif m_chdir:
2004 # We optimistically assume that unfinished chdir's are going to succeed
2005 dir = m_chdir.group(1)
2006 working_directories[pid] = mkPath(working_directories[pid], dir)
2007 elif m_clone:
2008 working_directories[m_clone.group(2)] = working_directories[pid]
2009 elif m_vfork:
2010 working_directories[m_vfork.group(2)] = working_directories[pid]
2011 elif re_strace_open_resumed.match(content):
2012 pass
2013 elif re_strace_chdir_resumed.match(content):
2014 pass
2015 elif re_strace_vfork_unfinished.match(content):
2016 pass
2017 elif re_strace_clone_unfinished.match(content):
2018 pass
2019 elif re_strace_ignore_sigchild.match(content):
2020 pass
2021 elif re_strace_ignore_sigvtalarm.match(content):
2022 pass
2023 elif re_strace_ignore_sigint.match(content):
2024 pass
2025 elif re_strace_ignore_sigfpe.match(content):
2026 pass
2027 elif re_strace_ignore_sigsegv.match(content):
2028 pass
2029 elif re_strace_ignore_sigpipe.match(content):
2030 pass
2031 else:
2032 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2033
2034 def checkForFilesWrittenProblems(file):
2035 foundProblem = False
2036
2037 files_written_inverted = {}
2038 for t in files_written.keys():
2039 for f in files_written[t]:
2040 try:
2041 files_written_inverted[f].append(t)
2042 except:
2043 files_written_inverted[f] = [t]
2044
2045 for f in files_written_inverted.keys():
2046 if len(files_written_inverted[f]) > 1:
2047 if not foundProblem:
2048 foundProblem = True
2049 file.write("\n")
2050 file.write("\nSome files are written by multiple tests:\n")
2051 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2052 if foundProblem:
2053 file.write("\n")
2054
2055 # -----
2056
2057 if len(files_written_not_removed) > 0:
2058 file.write("\n")
2059 file.write("\nSome files written but not removed:\n")
2060 tests = files_written_not_removed.keys()
2061 tests.sort()
2062 for t in tests:
2063 for f in files_written_not_removed[t]:
2064 file.write(" " + t + ": " + f + "\n")
2065 file.write("\n")
2066
2067 # -----
2068
2069 if len(bad_file_usages) > 0:
2070 file.write("\n")
2071 file.write("\nSome bad file usages:\n")
2072 tests = bad_file_usages.keys()
2073 tests.sort()
2074 for t in tests:
2075 for f in bad_file_usages[t]:
2076 file.write(" " + t + ": " + f + "\n")
2077 file.write("\n")
2078
2079 # -----------------------------------------------------------------------------
2080 # checking if ghostscript is available for checking the output of hp2ps
2081
2082 def genGSCmd(psfile):
2083 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2084
2085 def gsNotWorking():
2086 global gs_working
2087 print "GhostScript not available for hp2ps tests"
2088
2089 global gs_working
2090 gs_working = 0
2091 if config.have_profiling:
2092 if config.gs != '':
2093 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2094 if resultGood == 0:
2095 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2096 if resultBad != 0:
2097 print "GhostScript available for hp2ps tests"
2098 gs_working = 1;
2099 else:
2100 gsNotWorking();
2101 else:
2102 gsNotWorking();
2103 else:
2104 gsNotWorking();
2105
2106 def rm_no_fail( file ):
2107 try:
2108 os.remove( file )
2109 finally:
2110 return
2111
2112 def add_suffix( name, suffix ):
2113 if suffix == '':
2114 return name
2115 else:
2116 return name + '.' + suffix
2117
2118 def add_hs_lhs_suffix(name):
2119 if getTestOpts().c_src:
2120 return add_suffix(name, 'c')
2121 elif getTestOpts().cmm_src:
2122 return add_suffix(name, 'cmm')
2123 elif getTestOpts().objc_src:
2124 return add_suffix(name, 'm')
2125 elif getTestOpts().objcpp_src:
2126 return add_suffix(name, 'mm')
2127 elif getTestOpts().literate:
2128 return add_suffix(name, 'lhs')
2129 else:
2130 return add_suffix(name, 'hs')
2131
2132 def replace_suffix( name, suffix ):
2133 base, suf = os.path.splitext(name)
2134 return base + '.' + suffix
2135
2136 def in_testdir( name ):
2137 return (getTestOpts().testdir + '/' + name)
2138
2139 def qualify( name, suff ):
2140 return in_testdir(add_suffix(name, suff))
2141
2142
2143 # Finding the sample output. The filename is of the form
2144 #
2145 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2146 #
2147 # and we pick the most specific version available. The <version> is
2148 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2149 # more fine-grained control use if_compiler_lt().
2150 #
2151 def platform_wordsize_qualify( name, suff ):
2152
2153 basepath = qualify(name, suff)
2154
2155 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2156 for (platformSpecific, plat) in [(1, '-' + config.platform),
2157 (1, '-' + config.os),
2158 (0, '')]
2159 for ws in ['-ws-' + config.wordsize, '']
2160 for comp in ['-' + config.compiler_type, '']
2161 for vers in ['-' + config.compiler_maj_version, '']]
2162
2163 dir = glob.glob(basepath + '*')
2164 dir = map (lambda d: normalise_slashes_(d), dir)
2165
2166 for (platformSpecific, f) in paths:
2167 if f in dir:
2168 return (platformSpecific,f)
2169
2170 return (0, basepath)
2171
2172 # Clean up prior to the test, so that we can't spuriously conclude
2173 # that it passed on the basis of old run outputs.
2174 def pretest_cleanup(name):
2175 if getTestOpts().outputdir != None:
2176 odir = in_testdir(getTestOpts().outputdir)
2177 try:
2178 shutil.rmtree(odir)
2179 except:
2180 pass
2181 os.mkdir(odir)
2182
2183 rm_no_fail(qualify(name,'comp.stderr'))
2184 rm_no_fail(qualify(name,'run.stderr'))
2185 rm_no_fail(qualify(name,'run.stdout'))
2186 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2187 # simple_build zaps the following:
2188 # rm_nofail(qualify("o"))
2189 # rm_nofail(qualify(""))
2190 # not interested in the return code
2191
2192 # -----------------------------------------------------------------------------
2193 # Return a list of all the files ending in '.T' below the directory dir.
2194
2195 def findTFiles(roots):
2196 return concat(map(findTFiles_,roots))
2197
2198 def findTFiles_(path):
2199 if os.path.isdir(path):
2200 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2201 return findTFiles(paths)
2202 elif path[-2:] == '.T':
2203 return [path]
2204 else:
2205 return []
2206
2207 # -----------------------------------------------------------------------------
2208 # Output a test summary to the specified file object
2209
2210 def summary(t, file):
2211
2212 file.write('\n')
2213 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2214 file.write('OVERALL SUMMARY for test run started at ' \
2215 + t.start_time + '\n'\
2216 + string.rjust(`t.total_tests`, 8) \
2217 + ' total tests, which gave rise to\n' \
2218 + string.rjust(`t.total_test_cases`, 8) \
2219 + ' test cases, of which\n' \
2220 + string.rjust(`t.n_framework_failures`, 8) \
2221 + ' caused framework failures\n' \
2222 + string.rjust(`t.n_tests_skipped`, 8)
2223 + ' were skipped\n\n' \
2224 + string.rjust(`t.n_expected_passes`, 8)
2225 + ' expected passes\n' \
2226 + string.rjust(`t.n_missing_libs`, 8)
2227 + ' had missing libraries\n' \
2228 + string.rjust(`t.n_expected_failures`, 8) \
2229 + ' expected failures\n' \
2230 + string.rjust(`t.n_unexpected_passes`, 8) \
2231 + ' unexpected passes\n'
2232 + string.rjust(`t.n_unexpected_failures`, 8) \
2233 + ' unexpected failures\n'
2234 + '\n')
2235
2236 if t.n_unexpected_passes > 0:
2237 file.write('Unexpected passes:\n')
2238 printPassingTestInfosSummary(file, t.unexpected_passes)
2239
2240 if t.n_unexpected_failures > 0:
2241 file.write('Unexpected failures:\n')
2242 printFailingTestInfosSummary(file, t.unexpected_failures)
2243
2244 if config.check_files_written:
2245 checkForFilesWrittenProblems(file)
2246
2247 if stopping():
2248 file.write('WARNING: Testsuite run was terminated early\n')
2249
2250 def printUnexpectedTests(file, testInfoss):
2251 unexpected = []
2252 for testInfos in testInfoss:
2253 directories = testInfos.keys()
2254 for directory in directories:
2255 tests = testInfos[directory].keys()
2256 unexpected += tests
2257 if unexpected != []:
2258 file.write('Unexpected results from:\n')
2259 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2260 file.write('\n')
2261
2262 def printPassingTestInfosSummary(file, testInfos):
2263 directories = testInfos.keys()
2264 directories.sort()
2265 maxDirLen = max(map ((lambda x : len(x)), directories))
2266 for directory in directories:
2267 tests = testInfos[directory].keys()
2268 tests.sort()
2269 for test in tests:
2270 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2271 ' (' + join(testInfos[directory][test],',') + ')\n')
2272 file.write('\n')
2273
2274 def printFailingTestInfosSummary(file, testInfos):
2275 directories = testInfos.keys()
2276 directories.sort()
2277 maxDirLen = max(map ((lambda x : len(x)), directories))
2278 for directory in directories:
2279 tests = testInfos[directory].keys()
2280 tests.sort()
2281 for test in tests:
2282 reasons = testInfos[directory][test].keys()
2283 for reason in reasons:
2284 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2285 ' [' + reason + ']' + \
2286 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2287 file.write('\n')
2288
2289 def getStdout(cmd):
2290 if have_subprocess:
2291 p = subprocess.Popen(cmd,
2292 stdout=subprocess.PIPE,
2293 stderr=subprocess.PIPE)
2294 (stdout, stderr) = p.communicate()
2295 r = p.wait()
2296 if r != 0:
2297 raise Exception("Command failed: " + str(cmd))
2298 if stderr != '':
2299 raise Exception("stderr from command: " + str(cmd))
2300 return stdout
2301 else:
2302 raise Exception("Need subprocess to get stdout, but don't have it")