Change how the testsuite driver handles copying the test options
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for all the tests in the current "directory". After
44 # each test, we reset the options to these. To change the options for
45 # multiple tests, the function setTestOpts() below can be used to alter
46 # these options.
47 global thisdir_testopts
48 thisdir_testopts = TestOptions()
49
50 def getThisDirTestOpts():
51 return thisdir_testopts
52
53 # Options valid for the current test only (these get reset to
54 # testdir_testopts after each test).
55
56 global testopts_local
57 if config.use_threads:
58 testopts_local = threading.local()
59 else:
60 class TestOpts_Local:
61 pass
62 testopts_local = TestOpts_Local()
63
64 def getTestOpts():
65 return testopts_local.x
66
67 def setLocalTestOpts(opts):
68 global testopts_local
69 testopts_local.x=opts
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 f( thisdir_testopts );
75
76 # -----------------------------------------------------------------------------
77 # Canned setup functions for common cases. eg. for a test you might say
78 #
79 # test('test001', normal, compile, [''])
80 #
81 # to run it without any options, but change it to
82 #
83 # test('test001', expect_fail, compile, [''])
84 #
85 # to expect failure for this test.
86
87 def normal( opts ):
88 return;
89
90 def skip( opts ):
91 opts.skip = 1
92
93 def expect_fail( opts ):
94 opts.expect = 'fail';
95
96 def reqlib( lib ):
97 return lambda opts, l=lib: _reqlib (opts, l )
98
99 # Cache the results of looking to see if we have a library or not.
100 # This makes quite a difference, especially on Windows.
101 have_lib = {}
102
103 def _reqlib( opts, lib ):
104 if have_lib.has_key(lib):
105 got_it = have_lib[lib]
106 else:
107 if have_subprocess:
108 # By preference we use subprocess, as the alternative uses
109 # /dev/null which mingw doesn't have.
110 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' describe ' + lib
119 + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def expect_broken( bug ):
143 return lambda opts, b=bug: _expect_broken (opts, b )
144
145 def _expect_broken( opts, bug ):
146 opts.expect = 'fail';
147
148 def ignore_output( opts ):
149 opts.ignore_output = 1
150
151 def no_stdin( opts ):
152 opts.no_stdin = 1
153
154 def combined_output( opts ):
155 opts.combined_output = True
156
157 # -----
158
159 def expect_fail_for( ways ):
160 return lambda opts, w=ways: _expect_fail_for( opts, w )
161
162 def _expect_fail_for( opts, ways ):
163 opts.expect_fail_for = ways
164
165 def expect_broken_for( bug, ways ):
166 return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )
167
168 def _expect_broken_for( opts, bug, ways ):
169 opts.expect_fail_for = ways
170
171 # -----
172
173 def omit_ways( ways ):
174 return lambda opts, w=ways: _omit_ways( opts, w )
175
176 def _omit_ways( opts, ways ):
177 opts.omit_ways = ways
178
179 # -----
180
181 def only_ways( ways ):
182 return lambda opts, w=ways: _only_ways( opts, w )
183
184 def _only_ways( opts, ways ):
185 opts.only_ways = ways
186
187 # -----
188
189 def extra_ways( ways ):
190 return lambda opts, w=ways: _extra_ways( opts, w )
191
192 def _extra_ways( opts, ways ):
193 opts.extra_ways = ways
194
195 # -----
196
197 def omit_compiler_types( compiler_types ):
198 return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)
199
200 def _omit_compiler_types( opts, compiler_types ):
201 if config.compiler_type in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def only_compiler_types( compiler_types ):
207 return lambda opts, c=compiler_types: _only_compiler_types(opts, c)
208
209 def _only_compiler_types( opts, compiler_types ):
210 if config.compiler_type not in compiler_types:
211 opts.skip = 1
212
213 # -----
214
215 def set_stdin( file ):
216 return lambda opts, f=file: _set_stdin(opts, f);
217
218 def _set_stdin( opts, f ):
219 opts.stdin = f
220
221 # -----
222
223 def exit_code( val ):
224 return lambda opts, v=val: _exit_code(opts, v);
225
226 def _exit_code( opts, v ):
227 opts.exit_code = v
228
229 # -----
230
231 def timeout_multiplier( val ):
232 return lambda opts, v=val: _timeout_multiplier(opts, v)
233
234 def _timeout_multiplier( opts, v ):
235 opts.timeout_multiplier = v
236
237 # -----
238
239 def extra_run_opts( val ):
240 return lambda opts, v=val: _extra_run_opts(opts, v);
241
242 def _extra_run_opts( opts, v ):
243 opts.extra_run_opts = v
244
245 # -----
246
247 def extra_hc_opts( val ):
248 return lambda opts, v=val: _extra_hc_opts(opts, v);
249
250 def _extra_hc_opts( opts, v ):
251 opts.extra_hc_opts = v
252
253 # -----
254
255 def extra_clean( files ):
256 return lambda opts, v=files: _extra_clean(opts, v);
257
258 def _extra_clean( opts, v ):
259 opts.clean_files = v
260
261 # -----
262
263 def stats_range_field( field, expected, dev ):
264 return lambda opts, f=field, x=expected, y=dev: _stats_range_field(opts, f, x, y);
265
266 def _stats_range_field( opts, f, x, y ):
267 opts.stats_range_fields[f] = (x, y)
268
269 def compiler_stats_range_field( field, expected, dev ):
270 return lambda opts, f=field, x=expected, y=dev: _compiler_stats_range_field(opts, f, x, y);
271
272 def _compiler_stats_range_field( opts, f, x, y ):
273 opts.compiler_stats_range_fields[f] = (x, y)
274
275 # -----
276
277 def skip_if_no_ghci(opts):
278 if not ('ghci' in config.run_ways):
279 opts.skip = 1
280
281 # ----
282
283 def skip_if_fast(opts):
284 if config.fast:
285 opts.skip = 1
286
287 # -----
288
289 def when(b, f):
290 if b:
291 return f
292 else:
293 return normal
294
295 def unless(b, f):
296 return when(not b, f)
297
298 def if_platform( plat, f ):
299 if config.platform == plat:
300 return f
301 else:
302 return normal
303
304 def unless_platform( plat, f ):
305 if config.platform != plat:
306 return f
307 else:
308 return normal
309
310 def if_os( os, f ):
311 if config.os == os:
312 return f
313 else:
314 return normal
315
316 def unless_os( os, f ):
317 if config.os == os:
318 return normal
319 else:
320 return f
321
322 def if_arch( arch, f ):
323 if config.arch == arch:
324 return f
325 else:
326 return normal
327
328 def unless_arch( arch, f ):
329 if config.arch == arch:
330 return normal
331 else:
332 return f
333
334 def wordsize( ws ):
335 return config.wordsize == str(ws)
336
337 def if_unregisterised( f ):
338 if config.unregisterised:
339 return f
340 else:
341 return normal
342
343 def unless_unregisterised( f ):
344 if config.unregisterised:
345 return normal
346 else:
347 return f
348
349 def if_msys( f ):
350 if config.msys:
351 return f
352 else:
353 return normal
354
355 def if_cygwin( f ):
356 if config.cygwin:
357 return f
358 else:
359 return normal
360
361 def when_have_vanilla( f ):
362 if config.have_vanilla:
363 return f
364 else:
365 return normal
366
367 def unless_have_vanilla( f ):
368 if config.have_vanilla:
369 return normal
370 else:
371 return f
372
373 def when_have_dynamic( f ):
374 if config.have_dynamic:
375 return f
376 else:
377 return normal
378
379 def unless_have_dynamic( f ):
380 if config.have_dynamic:
381 return normal
382 else:
383 return f
384
385 def when_have_profiling( f ):
386 if config.have_profiling:
387 return f
388 else:
389 return normal
390
391 def unless_have_profiling( f ):
392 if config.have_profiling:
393 return normal
394 else:
395 return f
396
397 # ---
398
399 def if_ghci_dynamic( f ):
400 if config.ghc_dynamic_by_default:
401 return f
402 else:
403 return normal
404
405 def if_in_tree_compiler( f ):
406 if config.in_tree_compiler:
407 return f
408 else:
409 return normal
410
411 def unless_in_tree_compiler( f ):
412 if config.in_tree_compiler:
413 return normal
414 else:
415 return f
416
417 def if_compiler_type( compiler, f ):
418 if config.compiler_type == compiler:
419 return f
420 else:
421 return normal
422
423 def if_compiler_profiled( f ):
424 if config.compiler_profiled:
425 return f
426 else:
427 return normal
428
429 def unless_compiler_profiled( f ):
430 if config.compiler_profiled:
431 return normal
432 else:
433 return f
434
435 def if_compiler_lt( compiler, version, f ):
436 if config.compiler_type == compiler and \
437 version_lt(config.compiler_version, version):
438 return f
439 else:
440 return normal
441
442 def if_compiler_le( compiler, version, f ):
443 if config.compiler_type == compiler and \
444 version_le(config.compiler_version, version):
445 return f
446 else:
447 return normal
448
449 def if_compiler_gt( compiler, version, f ):
450 if config.compiler_type == compiler and \
451 version_gt(config.compiler_version, version):
452 return f
453 else:
454 return normal
455
456 def if_compiler_ge( compiler, version, f ):
457 if config.compiler_type == compiler and \
458 version_ge(config.compiler_version, version):
459 return f
460 else:
461 return normal
462
463 def if_compiler_debugged( f ):
464 if config.compiler_debugged:
465 return f
466 else:
467 return normal
468
469 def namebase( nb ):
470 return lambda opts, nb=nb: _namebase(opts, nb)
471
472 def _namebase( opts, nb ):
473 opts.with_namebase = nb
474
475 # ---
476
477 def if_tag( tag, f ):
478 if tag in config.compiler_tags:
479 return f
480 else:
481 return normal
482
483 def unless_tag( tag, f ):
484 if not (tag in config.compiler_tags):
485 return f
486 else:
487 return normal
488
489 # ---
490 def high_memory_usage(opts):
491 opts.alone = True
492
493 # ---
494 def literate( opts ):
495 opts.literate = 1;
496
497 def c_src( opts ):
498 opts.c_src = 1;
499
500 def objc_src( opts ):
501 opts.objc_src = 1;
502
503 def objcpp_src( opts ):
504 opts.objcpp_src = 1;
505
506 def cmm_src( opts ):
507 opts.cmm_src = 1;
508
509 def outputdir( odir ):
510 return lambda opts, d=odir: _outputdir(opts, d)
511
512 def _outputdir( opts, odir ):
513 opts.outputdir = odir;
514
515 # ----
516
517 def pre_cmd( cmd ):
518 return lambda opts, c=cmd: _pre_cmd(opts, cmd)
519
520 def _pre_cmd( opts, cmd ):
521 opts.pre_cmd = cmd
522
523 # ----
524
525 def clean_cmd( cmd ):
526 return lambda opts, c=cmd: _clean_cmd(opts, cmd)
527
528 def _clean_cmd( opts, cmd ):
529 opts.clean_cmd = cmd
530
531 # ----
532
533 def cmd_prefix( prefix ):
534 return lambda opts, p=prefix: _cmd_prefix(opts, prefix)
535
536 def _cmd_prefix( opts, prefix ):
537 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
538
539 # ----
540
541 def cmd_wrapper( fun ):
542 return lambda opts, f=fun: _cmd_wrapper(opts, fun)
543
544 def _cmd_wrapper( opts, fun ):
545 opts.cmd_wrapper = fun
546
547 # ----
548
549 def compile_cmd_prefix( prefix ):
550 return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)
551
552 def _compile_cmd_prefix( opts, prefix ):
553 opts.compile_cmd_prefix = prefix
554
555 # ----
556
557 def normalise_slashes( opts ):
558 opts.extra_normaliser = normalise_slashes_
559
560 def normalise_exe( opts ):
561 opts.extra_normaliser = normalise_exe_
562
563 def normalise_fun( fun ):
564 return lambda opts, f=fun: _normalise_fun(opts, f)
565
566 def _normalise_fun( opts, f ):
567 opts.extra_normaliser = f
568
569 def normalise_errmsg_fun( fun ):
570 return lambda opts, f=fun: _normalise_errmsg_fun(opts, f)
571
572 def _normalise_errmsg_fun( opts, f ):
573 opts.extra_errmsg_normaliser = f
574
575 def two_normalisers(f, g):
576 return lambda x, f=f, g=g: f(g(x))
577
578 # ----
579 # Function for composing two opt-fns together
580
581 def composes( fs ):
582 return reduce(lambda f, g: compose(f, g), fs)
583
584 def compose( f, g ):
585 return lambda opts, f=f, g=g: _compose(opts,f,g)
586
587 def _compose( opts, f, g ):
588 f(opts)
589 g(opts)
590
591 # -----------------------------------------------------------------------------
592 # The current directory of tests
593
594 def newTestDir( dir ):
595 global thisdir_testopts
596 # reset the options for this test directory
597 thisdir_testopts = copy.copy(default_testopts)
598 thisdir_testopts.testdir = dir
599 thisdir_testopts.compiler_always_flags = config.compiler_always_flags
600
601 # -----------------------------------------------------------------------------
602 # Actually doing tests
603
604 parallelTests = []
605 aloneTests = []
606 allTestNames = set([])
607
608 def runTest (opts, name, func, args):
609 ok = 0
610
611 if config.use_threads:
612 t.thread_pool.acquire()
613 try:
614 while config.threads<(t.running_threads+1):
615 t.thread_pool.wait()
616 t.running_threads = t.running_threads+1
617 ok=1
618 t.thread_pool.release()
619 thread.start_new_thread(test_common_thread, (name, opts, func, args))
620 except:
621 if not ok:
622 t.thread_pool.release()
623 else:
624 test_common_work (name, opts, func, args)
625
626 # name :: String
627 # setup :: TestOpts -> IO ()
628 def test (name, setup, func, args):
629 global aloneTests
630 global parallelTests
631 global allTestNames
632 if name in allTestNames:
633 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
634 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
635 framework_fail(name, 'bad_name', 'This test has an invalid name')
636 # We need a deepcopy so that dictionarys, such as the stats_range_fields
637 # dictionary, get copied too.
638 myTestOpts = copy.deepcopy(thisdir_testopts)
639
640 if type(setup) is types.ListType:
641 setup = composes(setup)
642
643 setup(myTestOpts)
644
645 thisTest = lambda : runTest(myTestOpts, name, func, args)
646 if myTestOpts.alone:
647 aloneTests.append(thisTest)
648 else:
649 parallelTests.append(thisTest)
650 allTestNames.add(name)
651
652 if config.use_threads:
653 def test_common_thread(name, opts, func, args):
654 t.lock.acquire()
655 try:
656 test_common_work(name,opts,func,args)
657 finally:
658 t.lock.release()
659 t.thread_pool.acquire()
660 t.running_threads = t.running_threads - 1
661 t.thread_pool.notify()
662 t.thread_pool.release()
663
664 def get_package_cache_timestamp():
665 if config.package_conf_cache_file == '':
666 return 0.0
667 else:
668 try:
669 return os.stat(config.package_conf_cache_file).st_mtime
670 except:
671 return 0.0
672
673
674 def test_common_work (name, opts, func, args):
675 try:
676 t.total_tests = t.total_tests+1
677 setLocalTestOpts(opts)
678
679 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
680
681 # All the ways we might run this test
682 if func == compile or func == multimod_compile:
683 all_ways = config.compile_ways
684 elif func == compile_and_run or func == multimod_compile_and_run:
685 all_ways = config.run_ways
686 elif func == ghci_script:
687 if 'ghci' in config.run_ways:
688 all_ways = ['ghci']
689 else:
690 all_ways = []
691 else:
692 all_ways = ['normal']
693
694 # A test itself can request extra ways by setting opts.extra_ways
695 all_ways = all_ways + filter(lambda way: way not in all_ways,
696 opts.extra_ways)
697
698 t.total_test_cases = t.total_test_cases + len(all_ways)
699
700 ok_way = lambda way: \
701 not getTestOpts().skip \
702 and (config.only == [] or name in config.only) \
703 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
704 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
705 and way not in getTestOpts().omit_ways
706
707 # Which ways we are asked to skip
708 do_ways = filter (ok_way,all_ways)
709
710 # In fast mode, we skip all but one way
711 if config.fast and len(do_ways) > 0:
712 do_ways = [do_ways[0]]
713
714 if not config.clean_only:
715 # Run the required tests...
716 for way in do_ways:
717 if stopping():
718 break
719 do_test (name, way, func, args)
720
721 for way in all_ways:
722 if way not in do_ways:
723 skiptest (name,way)
724
725 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
726 clean(map (lambda suff: name + suff,
727 ['', '.exe', '.exe.manifest', '.genscript',
728 '.stderr.normalised', '.stdout.normalised',
729 '.run.stderr', '.run.stdout',
730 '.run.stderr.normalised', '.run.stdout.normalised',
731 '.comp.stderr', '.comp.stdout',
732 '.comp.stderr.normalised', '.comp.stdout.normalised',
733 '.interp.stderr', '.interp.stdout',
734 '.interp.stderr.normalised', '.interp.stdout.normalised',
735 '.stats', '.comp.stats',
736 '.hi', '.o', '.prof', '.exe.prof', '.hc',
737 '_stub.h', '_stub.c', '_stub.o',
738 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
739
740 if func == multi_compile or func == multi_compile_fail:
741 extra_mods = args[1]
742 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
743 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
744
745 clean(getTestOpts().clean_files)
746
747 if getTestOpts().outputdir != None:
748 odir = in_testdir(getTestOpts().outputdir)
749 try:
750 shutil.rmtree(odir)
751 except:
752 pass
753
754 try:
755 shutil.rmtree(in_testdir('.hpc.' + name))
756 except:
757 pass
758
759 try:
760 cleanCmd = getTestOpts().clean_cmd
761 if cleanCmd != None:
762 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
763 if result != 0:
764 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
765 except e:
766 framework_fail(name, 'cleaning', 'clean-command exception')
767
768 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
769
770 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
771 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
772
773 try:
774 for f in files_written[name]:
775 if os.path.exists(f):
776 try:
777 if not f in files_written_not_removed[name]:
778 files_written_not_removed[name].append(f)
779 except:
780 files_written_not_removed[name] = [f]
781 except:
782 pass
783 except Exception, e:
784 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
785
786 def clean(strs):
787 for str in strs:
788 for name in glob.glob(in_testdir(str)):
789 clean_full_path(name)
790
791 def clean_full_path(name):
792 try:
793 # Remove files...
794 os.remove(name)
795 except OSError, e1:
796 try:
797 # ... and empty directories
798 os.rmdir(name)
799 except OSError, e2:
800 # We don't want to fail here, but we do want to know
801 # what went wrong, so print out the exceptions.
802 # ENOENT isn't a problem, though, as we clean files
803 # that don't necessarily exist.
804 if e1.errno != errno.ENOENT:
805 print e1
806 if e2.errno != errno.ENOENT:
807 print e2
808
809 def do_test(name, way, func, args):
810 full_name = name + '(' + way + ')'
811
812 try:
813 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
814 str([t.n_unexpected_passes, \
815 t.n_unexpected_failures, \
816 t.n_framework_failures])
817
818 if config.use_threads:
819 t.lock.release()
820
821 try:
822 preCmd = getTestOpts().pre_cmd
823 if preCmd != None:
824 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
825 if result != 0:
826 framework_fail(name, way, 'pre-command failed: ' + str(result))
827 except e:
828 framework_fail(name, way, 'pre-command exception')
829
830 try:
831 result = apply(func, [name,way] + args)
832 finally:
833 if config.use_threads:
834 t.lock.acquire()
835
836 if getTestOpts().expect != 'pass' and \
837 getTestOpts().expect != 'fail' and \
838 getTestOpts().expect != 'missing-lib':
839 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
840
841 try:
842 passFail = result['passFail']
843 except:
844 passFail = 'No passFail found'
845
846 if passFail == 'pass':
847 if getTestOpts().expect == 'pass' \
848 and way not in getTestOpts().expect_fail_for:
849 t.n_expected_passes = t.n_expected_passes + 1
850 if name in t.expected_passes:
851 t.expected_passes[name].append(way)
852 else:
853 t.expected_passes[name] = [way]
854 else:
855 print '*** unexpected pass for', full_name
856 t.n_unexpected_passes = t.n_unexpected_passes + 1
857 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
858 elif passFail == 'fail':
859 if getTestOpts().expect == 'pass' \
860 and way not in getTestOpts().expect_fail_for:
861 print '*** unexpected failure for', full_name
862 t.n_unexpected_failures = t.n_unexpected_failures + 1
863 reason = result['reason']
864 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
865 else:
866 if getTestOpts().expect == 'missing-lib':
867 t.n_missing_libs = t.n_missing_libs + 1
868 if name in t.missing_libs:
869 t.missing_libs[name].append(way)
870 else:
871 t.missing_libs[name] = [way]
872 else:
873 t.n_expected_failures = t.n_expected_failures + 1
874 if name in t.expected_failures:
875 t.expected_failures[name].append(way)
876 else:
877 t.expected_failures[name] = [way]
878 else:
879 framework_fail(name, way, 'bad result ' + passFail)
880 except KeyboardInterrupt:
881 stopNow()
882 except:
883 framework_fail(name, way, 'do_test exception')
884 traceback.print_exc()
885
886 def addPassingTestInfo (testInfos, directory, name, way):
887 directory = re.sub('^\\.[/\\\\]', '', directory)
888
889 if not directory in testInfos:
890 testInfos[directory] = {}
891
892 if not name in testInfos[directory]:
893 testInfos[directory][name] = []
894
895 testInfos[directory][name].append(way)
896
897 def addFailingTestInfo (testInfos, directory, name, reason, way):
898 directory = re.sub('^\\.[/\\\\]', '', directory)
899
900 if not directory in testInfos:
901 testInfos[directory] = {}
902
903 if not name in testInfos[directory]:
904 testInfos[directory][name] = {}
905
906 if not reason in testInfos[directory][name]:
907 testInfos[directory][name][reason] = []
908
909 testInfos[directory][name][reason].append(way)
910
911 def skiptest (name, way):
912 # print 'Skipping test \"', name, '\"'
913 t.n_tests_skipped = t.n_tests_skipped + 1
914 if name in t.tests_skipped:
915 t.tests_skipped[name].append(way)
916 else:
917 t.tests_skipped[name] = [way]
918
919 def framework_fail( name, way, reason ):
920 full_name = name + '(' + way + ')'
921 print '*** framework failure for', full_name, reason
922 t.n_framework_failures = t.n_framework_failures + 1
923 if name in t.framework_failures:
924 t.framework_failures[name].append(way)
925 else:
926 t.framework_failures[name] = [way]
927
928 def badResult(result):
929 try:
930 if result['passFail'] == 'pass':
931 return False
932 return True
933 except:
934 return True
935
936 def passed():
937 return {'passFail': 'pass'}
938
939 def failBecause(reason):
940 return {'passFail': 'fail', 'reason': reason}
941
942 # -----------------------------------------------------------------------------
943 # Generic command tests
944
945 # A generic command test is expected to run and exit successfully.
946 #
947 # The expected exit code can be changed via exit_code() as normal, and
948 # the expected stdout/stderr are stored in <testname>.stdout and
949 # <testname>.stderr. The output of the command can be ignored
950 # altogether by using run_command_ignore_output instead of
951 # run_command.
952
953 def run_command( name, way, cmd ):
954 return simple_run( name, '', cmd, '' )
955
956 # -----------------------------------------------------------------------------
957 # GHCi tests
958
959 def ghci_script( name, way, script ):
960 # filter out -fforce-recomp from compiler_always_flags, because we're
961 # actually testing the recompilation behaviour in the GHCi tests.
962 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
963 flags.append(getTestOpts().extra_hc_opts)
964 if getTestOpts().outputdir != None:
965 flags.extend(["-outputdir", getTestOpts().outputdir])
966
967 # We pass HC and HC_OPTS as environment variables, so that the
968 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
969 cmd = "HC='" + config.compiler + "' " + \
970 "HC_OPTS='" + join(flags,' ') + "' " + \
971 "'" + config.compiler + "'" + \
972 ' --interactive -v0 -ignore-dot-ghci ' + \
973 join(flags,' ')
974
975 getTestOpts().stdin = script
976 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
977
978 # -----------------------------------------------------------------------------
979 # Compile-only tests
980
981 def compile( name, way, extra_hc_opts ):
982 return do_compile( name, way, 0, '', [], extra_hc_opts )
983
984 def compile_fail( name, way, extra_hc_opts ):
985 return do_compile( name, way, 1, '', [], extra_hc_opts )
986
987 def multimod_compile( name, way, top_mod, extra_hc_opts ):
988 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
989
990 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
991 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
992
993 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
994 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
995
996 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
997 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
998
999 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1000 # print 'Compile only, extra args = ', extra_hc_opts
1001 pretest_cleanup(name)
1002
1003 result = extras_build( way, extra_mods, extra_hc_opts )
1004 if badResult(result):
1005 return result
1006 extra_hc_opts = result['hc_opts']
1007
1008 force = 0
1009 if extra_mods:
1010 force = 1
1011 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1012
1013 if badResult(result):
1014 return result
1015
1016 # the actual stderr should always match the expected, regardless
1017 # of whether we expected the compilation to fail or not (successful
1018 # compilations may generate warnings).
1019
1020 if getTestOpts().with_namebase == None:
1021 namebase = name
1022 else:
1023 namebase = getTestOpts().with_namebase
1024
1025 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1026 actual_stderr_file = qualify(name, 'comp.stderr')
1027
1028 if not compare_outputs('stderr', \
1029 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1030 expected_stderr_file, actual_stderr_file):
1031 return failBecause('stderr mismatch')
1032
1033 # no problems found, this test passed
1034 return passed()
1035
1036 def compile_cmp_asm( name, way, extra_hc_opts ):
1037 print 'Compile only, extra args = ', extra_hc_opts
1038 pretest_cleanup(name)
1039 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1040
1041 if badResult(result):
1042 return result
1043
1044 # the actual stderr should always match the expected, regardless
1045 # of whether we expected the compilation to fail or not (successful
1046 # compilations may generate warnings).
1047
1048 if getTestOpts().with_namebase == None:
1049 namebase = name
1050 else:
1051 namebase = getTestOpts().with_namebase
1052
1053 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1054 actual_asm_file = qualify(name, 's')
1055
1056 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1057 expected_asm_file, actual_asm_file):
1058 return failBecause('asm mismatch')
1059
1060 # no problems found, this test passed
1061 return passed()
1062
1063 # -----------------------------------------------------------------------------
1064 # Compile-and-run tests
1065
1066 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1067 # print 'Compile and run, extra args = ', extra_hc_opts
1068 pretest_cleanup(name)
1069
1070 result = extras_build( way, extra_mods, extra_hc_opts )
1071 if badResult(result):
1072 return result
1073 extra_hc_opts = result['hc_opts']
1074
1075 if way == 'ghci': # interpreted...
1076 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1077 elif way == 'extcore' or way == 'optextcore' :
1078 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1079 else: # compiled...
1080 force = 0
1081 if extra_mods:
1082 force = 1
1083
1084 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1085 if badResult(result):
1086 return result
1087
1088 cmd = './' + name;
1089
1090 # we don't check the compiler's stderr for a compile-and-run test
1091 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1092
1093 def compile_and_run( name, way, extra_hc_opts ):
1094 return compile_and_run__( name, way, '', [], extra_hc_opts)
1095
1096 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1097 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1098
1099 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1100 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1101
1102 def stats( name, way, stats_file ):
1103 opts = getTestOpts()
1104 return checkStats(stats_file, opts.stats_range_fields)
1105
1106 # -----------------------------------------------------------------------------
1107 # Check -t stats info
1108
1109 def checkStats(stats_file, range_fields):
1110 result = passed()
1111 if len(range_fields) > 0:
1112 f = open(in_testdir(stats_file))
1113 contents = f.read()
1114 f.close()
1115
1116 for (field, (expected, dev)) in range_fields.items():
1117 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1118 if m == None:
1119 print 'Failed to find field: ', field
1120 result = failBecause('no such stats field')
1121 val = int(m.group(1))
1122
1123 lowerBound = trunc( expected * ((100 - float(dev))/100));
1124 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1125
1126 if val < lowerBound:
1127 print field, 'value is too low:'
1128 print '(If this is because you have improved GHC, please'
1129 print 'update the test so that GHC doesn\'t regress again)'
1130 result = failBecause('stat too good')
1131 if val > upperBound:
1132 print field, 'value is too high:'
1133 result = failBecause('stat not good enough')
1134
1135 if val < lowerBound or val > upperBound:
1136 valStr = str(val)
1137 valLen = len(valStr)
1138 expectedStr = str(expected)
1139 expectedLen = len(expectedStr)
1140 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1141 def display(descr, val, extra):
1142 print descr, string.rjust(str(val), length), extra
1143 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1144 display(' Lower bound ' + field + ':', lowerBound, '')
1145 display(' Upper bound ' + field + ':', upperBound, '')
1146 display(' Actual ' + field + ':', val, '')
1147
1148 return result
1149
1150 # -----------------------------------------------------------------------------
1151 # Build a single-module program
1152
1153 def extras_build( way, extra_mods, extra_hc_opts ):
1154 for modopts in extra_mods:
1155 mod, opts = modopts
1156 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1157 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1158 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1159 if badResult(result):
1160 return result
1161
1162 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1163
1164
1165 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1166 opts = getTestOpts()
1167 errname = add_suffix(name, 'comp.stderr')
1168 rm_no_fail( qualify(errname, '') )
1169
1170 if top_mod != '':
1171 srcname = top_mod
1172 rm_no_fail( qualify(name, '') )
1173 base, suf = os.path.splitext(top_mod)
1174 rm_no_fail( qualify(base, '') )
1175 rm_no_fail( qualify(base, 'exe') )
1176 elif addsuf:
1177 srcname = add_hs_lhs_suffix(name)
1178 rm_no_fail( qualify(name, '') )
1179 else:
1180 srcname = name
1181 rm_no_fail( qualify(name, 'o') )
1182
1183 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1184
1185 to_do = ''
1186 if top_mod != '':
1187 to_do = '--make '
1188 if link:
1189 to_do = to_do + '-o ' + name
1190 elif link:
1191 to_do = '-o ' + name
1192 elif opts.compile_to_hc:
1193 to_do = '-C'
1194 else:
1195 to_do = '-c' # just compile
1196
1197 stats_file = name + '.comp.stats'
1198 if len(opts.compiler_stats_range_fields) > 0:
1199 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1200
1201 # Required by GHC 7.3+, harmless for earlier versions:
1202 if (getTestOpts().c_src or
1203 getTestOpts().objc_src or
1204 getTestOpts().objcpp_src or
1205 getTestOpts().cmm_src):
1206 extra_hc_opts += ' -no-hs-main '
1207
1208 if getTestOpts().compile_cmd_prefix == '':
1209 cmd_prefix = ''
1210 else:
1211 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1212
1213 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1214 if noforce:
1215 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1216 if getTestOpts().outputdir != None:
1217 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1218
1219 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1220 + config.compiler + "' " \
1221 + join(comp_flags,' ') + ' ' \
1222 + to_do + ' ' + srcname + ' ' \
1223 + join(config.way_flags(name)[way],' ') + ' ' \
1224 + extra_hc_opts + ' ' \
1225 + opts.extra_hc_opts + ' ' \
1226 + '>' + errname + ' 2>&1'
1227
1228 result = runCmdFor(name, cmd)
1229
1230 if result != 0 and not should_fail:
1231 actual_stderr = qualify(name, 'comp.stderr')
1232 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1233 if_verbose_dump(1,actual_stderr)
1234
1235 # ToDo: if the sub-shell was killed by ^C, then exit
1236
1237 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1238
1239 if badResult(statsResult):
1240 return statsResult
1241
1242 if should_fail:
1243 if result == 0:
1244 return failBecause('exit code 0')
1245 else:
1246 if result != 0:
1247 return failBecause('exit code non-0')
1248
1249 return passed()
1250
1251 # -----------------------------------------------------------------------------
1252 # Run a program and check its output
1253 #
1254 # If testname.stdin exists, route input from that, else
1255 # from /dev/null. Route output to testname.run.stdout and
1256 # testname.run.stderr. Returns the exit code of the run.
1257
1258 def simple_run( name, way, prog, args ):
1259 opts = getTestOpts()
1260
1261 # figure out what to use for stdin
1262 if opts.stdin != '':
1263 use_stdin = opts.stdin
1264 else:
1265 stdin_file = add_suffix(name, 'stdin')
1266 if os.path.exists(in_testdir(stdin_file)):
1267 use_stdin = stdin_file
1268 else:
1269 use_stdin = '/dev/null'
1270
1271 run_stdout = add_suffix(name,'run.stdout')
1272 run_stderr = add_suffix(name,'run.stderr')
1273
1274 rm_no_fail(qualify(name,'run.stdout'))
1275 rm_no_fail(qualify(name,'run.stderr'))
1276 rm_no_fail(qualify(name, 'hp'))
1277 rm_no_fail(qualify(name,'ps'))
1278 rm_no_fail(qualify(name, 'prof'))
1279
1280 my_rts_flags = rts_flags(way)
1281
1282 stats_file = name + '.stats'
1283 if len(opts.stats_range_fields) > 0:
1284 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1285
1286 if opts.no_stdin:
1287 stdin_comes_from = ''
1288 else:
1289 stdin_comes_from = ' <' + use_stdin
1290
1291 if opts.combined_output:
1292 redirection = ' >' + run_stdout \
1293 + ' 2>&1'
1294 else:
1295 redirection = ' >' + run_stdout \
1296 + ' 2>' + run_stderr
1297
1298 cmd = prog + ' ' + args + ' ' \
1299 + my_rts_flags + ' ' \
1300 + stdin_comes_from \
1301 + redirection
1302
1303 if opts.cmd_wrapper != None:
1304 cmd = opts.cmd_wrapper(cmd);
1305
1306 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1307
1308 # run the command
1309 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1310
1311 exit_code = result >> 8
1312 signal = result & 0xff
1313
1314 # check the exit code
1315 if exit_code != opts.exit_code:
1316 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1317 dump_stdout(name)
1318 dump_stderr(name)
1319 return failBecause('bad exit code')
1320
1321 check_hp = my_rts_flags.find("-h") != -1
1322 check_prof = my_rts_flags.find("-p") != -1
1323
1324 if not opts.ignore_output:
1325 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1326 bad_stdout = not check_stdout_ok(name)
1327 if bad_stderr:
1328 return failBecause('bad stderr')
1329 if bad_stdout:
1330 return failBecause('bad stdout')
1331 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1332 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1333 return failBecause('bad heap profile')
1334 if check_prof and not check_prof_ok(name):
1335 return failBecause('bad profile')
1336
1337 return checkStats(stats_file, opts.stats_range_fields)
1338
1339 def rts_flags(way):
1340 if (way == ''):
1341 return ''
1342 else:
1343 args = config.way_rts_flags[way]
1344
1345 if args == []:
1346 return ''
1347 else:
1348 return '+RTS ' + join(args,' ') + ' -RTS'
1349
1350 # -----------------------------------------------------------------------------
1351 # Run a program in the interpreter and check its output
1352
1353 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1354 outname = add_suffix(name, 'interp.stdout')
1355 errname = add_suffix(name, 'interp.stderr')
1356 rm_no_fail(outname)
1357 rm_no_fail(errname)
1358 rm_no_fail(name)
1359
1360 if (top_mod == ''):
1361 srcname = add_hs_lhs_suffix(name)
1362 else:
1363 srcname = top_mod
1364
1365 scriptname = add_suffix(name, 'genscript')
1366 qscriptname = in_testdir(scriptname)
1367 rm_no_fail(qscriptname)
1368
1369 delimiter = '===== program output begins here\n'
1370
1371 script = open(qscriptname, 'w')
1372 if not compile_only:
1373 # set the prog name and command-line args to match the compiled
1374 # environment.
1375 script.write(':set prog ' + name + '\n')
1376 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1377 # Add marker lines to the stdout and stderr output files, so we
1378 # can separate GHCi's output from the program's.
1379 script.write(':! echo ' + delimiter)
1380 script.write(':! echo 1>&2 ' + delimiter)
1381 # Set stdout to be line-buffered to match the compiled environment.
1382 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1383 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1384 # in the event of an exception as for the compiled program.
1385 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1386 script.close()
1387
1388 # figure out what to use for stdin
1389 if getTestOpts().stdin != '':
1390 stdin_file = in_testdir(getTestOpts().stdin)
1391 else:
1392 stdin_file = qualify(name, 'stdin')
1393
1394 if os.path.exists(stdin_file):
1395 stdin = open(stdin_file, 'r')
1396 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1397
1398 script.close()
1399
1400 flags = copy.copy(getTestOpts().compiler_always_flags)
1401 if getTestOpts().outputdir != None:
1402 flags.extend(["-outputdir", getTestOpts().outputdir])
1403
1404 cmd = "'" + config.compiler + "' " \
1405 + join(flags,' ') + ' ' \
1406 + srcname + ' ' \
1407 + join(config.way_flags(name)[way],' ') + ' ' \
1408 + extra_hc_opts + ' ' \
1409 + getTestOpts().extra_hc_opts + ' ' \
1410 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1411
1412 if getTestOpts().cmd_wrapper != None:
1413 cmd = getTestOpts().cmd_wrapper(cmd);
1414
1415 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1416
1417 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1418
1419 exit_code = result >> 8
1420 signal = result & 0xff
1421
1422 # split the stdout into compilation/program output
1423 split_file(in_testdir(outname), delimiter,
1424 qualify(name, 'comp.stdout'),
1425 qualify(name, 'run.stdout'))
1426 split_file(in_testdir(errname), delimiter,
1427 qualify(name, 'comp.stderr'),
1428 qualify(name, 'run.stderr'))
1429
1430 # check the exit code
1431 if exit_code != getTestOpts().exit_code:
1432 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1433 dump_stdout(name)
1434 dump_stderr(name)
1435 return failBecause('bad exit code')
1436
1437 # ToDo: if the sub-shell was killed by ^C, then exit
1438
1439 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1440 check_stdout_ok(name)):
1441 return passed()
1442 else:
1443 return failBecause('bad stdout or stderr')
1444
1445
1446 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1447 infile = open(in_fn)
1448 out1 = open(out1_fn, 'w')
1449 out2 = open(out2_fn, 'w')
1450
1451 line = infile.readline()
1452 line = re.sub('\r', '', line) # ignore Windows EOL
1453 while (re.sub('^\s*','',line) != delimiter and line != ''):
1454 out1.write(line)
1455 line = infile.readline()
1456 line = re.sub('\r', '', line)
1457 out1.close()
1458
1459 line = infile.readline()
1460 while (line != ''):
1461 out2.write(line)
1462 line = infile.readline()
1463 out2.close()
1464
1465 # -----------------------------------------------------------------------------
1466 # Generate External Core for the given program, then compile the resulting Core
1467 # and compare its output to the expected output
1468
1469 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1470
1471 depsfilename = qualify(name, 'deps')
1472 errname = add_suffix(name, 'comp.stderr')
1473 qerrname = qualify(errname,'')
1474
1475 hcname = qualify(name, 'hc')
1476 oname = qualify(name, 'o')
1477
1478 rm_no_fail( qerrname )
1479 rm_no_fail( qualify(name, '') )
1480
1481 if (top_mod == ''):
1482 srcname = add_hs_lhs_suffix(name)
1483 else:
1484 srcname = top_mod
1485
1486 qcorefilename = qualify(name, 'hcr')
1487 corefilename = add_suffix(name, 'hcr')
1488 rm_no_fail(qcorefilename)
1489
1490 # Generate External Core
1491
1492 if (top_mod == ''):
1493 to_do = ' ' + srcname + ' '
1494 else:
1495 to_do = ' --make ' + top_mod + ' '
1496
1497 flags = copy.copy(getTestOpts().compiler_always_flags)
1498 if getTestOpts().outputdir != None:
1499 flags.extend(["-outputdir", getTestOpts().outputdir])
1500 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1501 + config.compiler + "' " \
1502 + join(flags,' ') + ' ' \
1503 + join(config.way_flags(name)[way],' ') + ' ' \
1504 + extra_hc_opts + ' ' \
1505 + getTestOpts().extra_hc_opts \
1506 + to_do \
1507 + '>' + errname + ' 2>&1'
1508 result = runCmdFor(name, cmd)
1509
1510 exit_code = result >> 8
1511
1512 if exit_code != 0:
1513 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1514 if_verbose_dump(1,qerrname)
1515 return failBecause('ext core exit code non-0')
1516
1517 # Compile the resulting files -- if there's more than one module, we need to read the output
1518 # of the previous compilation in order to find the dependencies
1519 if (top_mod == ''):
1520 to_compile = corefilename
1521 else:
1522 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1523 deps = open(depsfilename).read()
1524 deplist = string.replace(deps, '\n',' ');
1525 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1526 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1527
1528 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1529 if getTestOpts().outputdir != None:
1530 flags.extend(["-outputdir", getTestOpts().outputdir])
1531
1532 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1533 + config.compiler + "' " \
1534 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1535 + to_compile + ' ' \
1536 + extra_hc_opts + ' ' \
1537 + getTestOpts().extra_hc_opts + ' ' \
1538 + flags \
1539 + ' -fglasgow-exts -o ' + name \
1540 + '>' + errname + ' 2>&1'
1541
1542 result = runCmdFor(name, cmd)
1543 exit_code = result >> 8
1544
1545 if exit_code != 0:
1546 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1547 if_verbose_dump(1,qerrname)
1548 return failBecause('ext core exit code non-0')
1549
1550 # Clean up
1551 rm_no_fail ( oname )
1552 rm_no_fail ( hcname )
1553 rm_no_fail ( qcorefilename )
1554 rm_no_fail ( depsfilename )
1555
1556 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1557
1558 # -----------------------------------------------------------------------------
1559 # Utils
1560
1561 def check_stdout_ok( name ):
1562 if getTestOpts().with_namebase == None:
1563 namebase = name
1564 else:
1565 namebase = getTestOpts().with_namebase
1566
1567 actual_stdout_file = qualify(name, 'run.stdout')
1568 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1569
1570 def norm(str):
1571 if platform_specific:
1572 return str
1573 else:
1574 return normalise_output(str)
1575
1576 return compare_outputs('stdout', \
1577 two_normalisers(norm, getTestOpts().extra_normaliser), \
1578 expected_stdout_file, actual_stdout_file)
1579
1580 def dump_stdout( name ):
1581 print 'Stdout:'
1582 print read_no_crs(qualify(name, 'run.stdout'))
1583
1584 def check_stderr_ok( name ):
1585 if getTestOpts().with_namebase == None:
1586 namebase = name
1587 else:
1588 namebase = getTestOpts().with_namebase
1589
1590 actual_stderr_file = qualify(name, 'run.stderr')
1591 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1592
1593 def norm(str):
1594 if platform_specific:
1595 return str
1596 else:
1597 return normalise_errmsg(str)
1598
1599 return compare_outputs('stderr', \
1600 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1601 expected_stderr_file, actual_stderr_file)
1602
1603 def dump_stderr( name ):
1604 print "Stderr:"
1605 print read_no_crs(qualify(name, 'run.stderr'))
1606
1607 def read_no_crs(file):
1608 str = ''
1609 try:
1610 h = open(file)
1611 str = h.read()
1612 h.close
1613 except:
1614 # On Windows, if the program fails very early, it seems the
1615 # files stdout/stderr are redirected to may not get created
1616 pass
1617 return re.sub('\r', '', str)
1618
1619 def write_file(file, str):
1620 h = open(file, 'w')
1621 h.write(str)
1622 h.close
1623
1624 def check_hp_ok(name):
1625
1626 # do not qualify for hp2ps because we should be in the right directory
1627 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1628
1629 hp2psResult = runCmdExitCode(hp2psCmd)
1630
1631 actual_ps_file = qualify(name, 'ps')
1632
1633 if(hp2psResult == 0):
1634 if (os.path.exists(actual_ps_file)):
1635 if gs_working:
1636 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1637 if (gsResult == 0):
1638 return (True)
1639 else:
1640 print "hp2ps output for " + name + "is not valid PostScript"
1641 else: return (True) # assume postscript is valid without ghostscript
1642 else:
1643 print "hp2ps did not generate PostScript for " + name
1644 return (False)
1645 else:
1646 print "hp2ps error when processing heap profile for " + name
1647 return(False)
1648
1649 def check_prof_ok(name):
1650
1651 prof_file = qualify(name,'prof')
1652
1653 if not os.path.exists(prof_file):
1654 print prof_file + " does not exist"
1655 return(False)
1656
1657 if os.path.getsize(qualify(name,'prof')) == 0:
1658 print prof_file + " is empty"
1659 return(False)
1660
1661 if getTestOpts().with_namebase == None:
1662 namebase = name
1663 else:
1664 namebase = getTestOpts().with_namebase
1665
1666 (platform_specific, expected_prof_file) = \
1667 platform_wordsize_qualify(namebase, 'prof.sample')
1668
1669 # sample prof file is not required
1670 if not os.path.exists(expected_prof_file):
1671 return True
1672 else:
1673 return compare_outputs('prof', \
1674 two_normalisers(normalise_whitespace,normalise_prof), \
1675 expected_prof_file, prof_file)
1676
1677 # Compare expected output to actual output, and optionally accept the
1678 # new output. Returns true if output matched or was accepted, false
1679 # otherwise.
1680 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1681 if os.path.exists(expected_file):
1682 expected_raw = read_no_crs(expected_file)
1683 # print "norm:", normaliser(expected_raw)
1684 expected_str = normaliser(expected_raw)
1685 expected_file_for_diff = expected_file
1686 else:
1687 expected_str = ''
1688 expected_file_for_diff = '/dev/null'
1689
1690 actual_raw = read_no_crs(actual_file)
1691 actual_str = normaliser(actual_raw)
1692
1693 if expected_str == actual_str:
1694 return 1
1695 else:
1696 print 'Actual ' + kind + ' output differs from expected:'
1697
1698 if expected_file_for_diff == '/dev/null':
1699 expected_normalised_file = '/dev/null'
1700 else:
1701 expected_normalised_file = expected_file + ".normalised"
1702 write_file(expected_normalised_file, expected_str)
1703
1704 actual_normalised_file = actual_file + ".normalised"
1705 write_file(actual_normalised_file, actual_str)
1706
1707 # Ignore whitespace when diffing. We should only get to this
1708 # point if there are non-whitespace differences
1709 #
1710 # Note we are diffing the *actual* output, not the normalised
1711 # output. The normalised output may have whitespace squashed
1712 # (including newlines) so the diff would be hard to read.
1713 # This does mean that the diff might contain changes that
1714 # would be normalised away.
1715 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1716 ' ' + actual_file )
1717
1718 # If for some reason there were no non-whitespace differences,
1719 # then do a full diff
1720 if r == 0:
1721 r = os.system( 'diff -u ' + expected_file_for_diff + \
1722 ' ' + actual_file )
1723
1724 if config.accept:
1725 print 'Accepting new output.'
1726 write_file(expected_file, actual_raw)
1727 return 1
1728 else:
1729 return 0
1730
1731
1732 def normalise_whitespace( str ):
1733 # Merge contiguous whitespace characters into a single space.
1734 str = re.sub('[ \t\n]+', ' ', str)
1735 return str
1736
1737 def normalise_errmsg( str ):
1738 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1739 # the colon is there because it appears in error messages; this
1740 # hacky solution is used in place of more sophisticated filename
1741 # mangling
1742 str = re.sub('([^\\s])\\.exe', '\\1', str)
1743 # normalise slashes, minimise Windows/Unix filename differences
1744 str = re.sub('\\\\', '/', str)
1745 # The inplace ghc's are called ghc-stage[123] to avoid filename
1746 # collisions, so we need to normalise that to just "ghc"
1747 str = re.sub('ghc-stage[123]', 'ghc', str)
1748 # We sometimes see the name of the integer-gmp package on stderr,
1749 # but this can change (either the implementation name or the
1750 # version number), so we canonicalise it here
1751 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1752 return str
1753
1754 # normalise a .prof file, so that we can reasonably compare it against
1755 # a sample. This doesn't compare any of the actual profiling data,
1756 # only the shape of the profile and the number of entries.
1757 def normalise_prof (str):
1758 # strip everything up to the line beginning "COST CENTRE"
1759 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1760
1761 # strip results for CAFs, these tend to change unpredictably
1762 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1763
1764 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1765 # sometimes under MAIN.
1766 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1767
1768 # We have somthing like this:
1769
1770 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1771 # k Main 204 1 0.0 0.0 0.0 0.0
1772 # foo Main 205 1 0.0 0.0 0.0 0.0
1773 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1774
1775 # then we remove all the specific profiling data, leaving only the
1776 # cost centre name, module, and entries, to end up with this:
1777
1778 # MAIN MAIN 0
1779 # k Main 1
1780 # foo Main 1
1781 # foo.bar Main 1
1782
1783 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1784 return str
1785
1786 def normalise_slashes_( str ):
1787 str = re.sub('\\\\', '/', str)
1788 return str
1789
1790 def normalise_exe_( str ):
1791 str = re.sub('\.exe', '', str)
1792 return str
1793
1794 def normalise_output( str ):
1795 # Remove a .exe extension (for Windows)
1796 # This can occur in error messages generated by the program.
1797 str = re.sub('([^\\s])\\.exe', '\\1', str)
1798 return str
1799
1800 def normalise_asm( str ):
1801 lines = str.split('\n')
1802 # Only keep instructions and labels not starting with a dot.
1803 metadata = re.compile('^[ \t]*\\..*$')
1804 out = []
1805 for line in lines:
1806 # Drop metadata directives (e.g. ".type")
1807 if not metadata.match(line):
1808 line = re.sub('@plt', '', line)
1809 instr = line.lstrip().split()
1810 # Drop empty lines.
1811 if not instr:
1812 continue
1813 # Drop operands, except for call instructions.
1814 elif instr[0] == 'call':
1815 out.append(instr[0] + ' ' + instr[1])
1816 else:
1817 out.append(instr[0])
1818 out = '\n'.join(out)
1819 return out
1820
1821 def if_verbose( n, str ):
1822 if config.verbose >= n:
1823 print str
1824
1825 def if_verbose_dump( n, f ):
1826 if config.verbose >= n:
1827 try:
1828 print open(f).read()
1829 except:
1830 print ''
1831
1832 def rawSystem(cmd_and_args):
1833 # We prefer subprocess.call to os.spawnv as the latter
1834 # seems to send its arguments through a shell or something
1835 # with the Windows (non-cygwin) python. An argument "a b c"
1836 # turns into three arguments ["a", "b", "c"].
1837
1838 # However, subprocess is new in python 2.4, so fall back to
1839 # using spawnv if we don't have it
1840
1841 if have_subprocess:
1842 return subprocess.call(cmd_and_args)
1843 else:
1844 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1845
1846 # Note that this doesn't handle the timeout itself; it is just used for
1847 # commands that have timeout handling built-in.
1848 def rawSystemWithTimeout(cmd_and_args):
1849 r = rawSystem(cmd_and_args)
1850 if r == 98:
1851 # The python timeout program uses 98 to signal that ^C was pressed
1852 stopNow()
1853 return r
1854
1855 # cmd is a complex command in Bourne-shell syntax
1856 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1857 # Hence it must ultimately be run by a Bourne shell
1858 #
1859 # Mostly it invokes the command wrapped in 'timeout' thus
1860 # timeout 300 'cd . && ...blah blah'
1861 # so it's timeout's job to invoke the Bourne shell
1862 #
1863 # But watch out for the case when there is no timeout program!
1864 # Then, when using the native Python, os.system will invoke the cmd shell
1865
1866 def runCmd( cmd ):
1867 if_verbose( 1, cmd )
1868 r = 0
1869 if config.os == 'mingw32':
1870 # On MinGW, we will always have timeout
1871 assert config.timeout_prog!=''
1872
1873 if config.timeout_prog != '':
1874 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1875 else:
1876 r = os.system(cmd)
1877 return r << 8
1878
1879 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1880 if_verbose( 1, cmd )
1881 r = 0
1882 if config.os == 'mingw32':
1883 # On MinGW, we will always have timeout
1884 assert config.timeout_prog!=''
1885 timeout = int(ceil(config.timeout * timeout_multiplier))
1886
1887 if config.timeout_prog != '':
1888 if config.check_files_written:
1889 fn = name + ".strace"
1890 r = rawSystemWithTimeout(
1891 ["strace", "-o", fn, "-fF",
1892 "-e", "creat,open,chdir,clone,vfork",
1893 config.timeout_prog, str(timeout), cmd])
1894 addTestFilesWritten(name, fn)
1895 rm_no_fail(fn)
1896 else:
1897 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1898 else:
1899 r = os.system(cmd)
1900 return r << 8
1901
1902 def runCmdExitCode( cmd ):
1903 return (runCmd(cmd) >> 8);
1904
1905
1906 # -----------------------------------------------------------------------------
1907 # checking for files being written to by multiple tests
1908
1909 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1910 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1911 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1912 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1913 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1914 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1915 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1916 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1917 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1918 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1919 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1920 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1921 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1922 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1923 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1924 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1925 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1926
1927 # Files that are read or written but shouldn't be:
1928 # * ghci_history shouldn't be read or written by tests
1929 # * things under package.conf.d shouldn't be written by tests
1930 bad_file_usages = {}
1931
1932 # Mapping from tests to the list of files that they write
1933 files_written = {}
1934
1935 # Mapping from tests to the list of files that they write but don't clean
1936 files_written_not_removed = {}
1937
1938 def add_bad_file_usage(name, file):
1939 try:
1940 if not file in bad_file_usages[name]:
1941 bad_file_usages[name].append(file)
1942 except:
1943 bad_file_usages[name] = [file]
1944
1945 def mkPath(curdir, path):
1946 # Given the current full directory is 'curdir', what is the full
1947 # path to 'path'?
1948 return os.path.realpath(os.path.join(curdir, path))
1949
1950 def addTestFilesWritten(name, fn):
1951 if config.use_threads:
1952 with t.lockFilesWritten:
1953 addTestFilesWrittenHelper(name, fn)
1954 else:
1955 addTestFilesWrittenHelper(name, fn)
1956
1957 def addTestFilesWrittenHelper(name, fn):
1958 started = False
1959 working_directories = {}
1960
1961 with open(fn, 'r') as f:
1962 for line in f:
1963 m_pid = re_strace_pid.match(line)
1964 if m_pid:
1965 pid = m_pid.group(1)
1966 content = m_pid.group(2)
1967 elif re_strace_unavailable.match(line):
1968 next
1969 else:
1970 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1971
1972 m_open = re_strace_open.match(content)
1973 m_chdir = re_strace_chdir.match(content)
1974 m_clone = re_strace_clone.match(content)
1975 m_vfork = re_strace_vfork.match(content)
1976
1977 if not started:
1978 working_directories[pid] = os.getcwd()
1979 started = True
1980
1981 if m_open:
1982 file = m_open.group(1)
1983 file = mkPath(working_directories[pid], file)
1984 if file.endswith("ghci_history"):
1985 add_bad_file_usage(name, file)
1986 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1987 flags = m_open.group(2).split('|')
1988 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1989 if re.match('package\.conf\.d', file):
1990 add_bad_file_usage(name, file)
1991 else:
1992 try:
1993 if not file in files_written[name]:
1994 files_written[name].append(file)
1995 except:
1996 files_written[name] = [file]
1997 elif 'O_RDONLY' in flags:
1998 pass
1999 else:
2000 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2001 elif m_chdir:
2002 # We optimistically assume that unfinished chdir's are going to succeed
2003 dir = m_chdir.group(1)
2004 working_directories[pid] = mkPath(working_directories[pid], dir)
2005 elif m_clone:
2006 working_directories[m_clone.group(2)] = working_directories[pid]
2007 elif m_vfork:
2008 working_directories[m_vfork.group(2)] = working_directories[pid]
2009 elif re_strace_open_resumed.match(content):
2010 pass
2011 elif re_strace_chdir_resumed.match(content):
2012 pass
2013 elif re_strace_vfork_unfinished.match(content):
2014 pass
2015 elif re_strace_clone_unfinished.match(content):
2016 pass
2017 elif re_strace_ignore_sigchild.match(content):
2018 pass
2019 elif re_strace_ignore_sigvtalarm.match(content):
2020 pass
2021 elif re_strace_ignore_sigint.match(content):
2022 pass
2023 elif re_strace_ignore_sigfpe.match(content):
2024 pass
2025 elif re_strace_ignore_sigsegv.match(content):
2026 pass
2027 elif re_strace_ignore_sigpipe.match(content):
2028 pass
2029 else:
2030 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2031
2032 def checkForFilesWrittenProblems(file):
2033 foundProblem = False
2034
2035 files_written_inverted = {}
2036 for t in files_written.keys():
2037 for f in files_written[t]:
2038 try:
2039 files_written_inverted[f].append(t)
2040 except:
2041 files_written_inverted[f] = [t]
2042
2043 for f in files_written_inverted.keys():
2044 if len(files_written_inverted[f]) > 1:
2045 if not foundProblem:
2046 foundProblem = True
2047 file.write("\n")
2048 file.write("\nSome files are written by multiple tests:\n")
2049 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2050 if foundProblem:
2051 file.write("\n")
2052
2053 # -----
2054
2055 if len(files_written_not_removed) > 0:
2056 file.write("\n")
2057 file.write("\nSome files written but not removed:\n")
2058 tests = files_written_not_removed.keys()
2059 tests.sort()
2060 for t in tests:
2061 for f in files_written_not_removed[t]:
2062 file.write(" " + t + ": " + f + "\n")
2063 file.write("\n")
2064
2065 # -----
2066
2067 if len(bad_file_usages) > 0:
2068 file.write("\n")
2069 file.write("\nSome bad file usages:\n")
2070 tests = bad_file_usages.keys()
2071 tests.sort()
2072 for t in tests:
2073 for f in bad_file_usages[t]:
2074 file.write(" " + t + ": " + f + "\n")
2075 file.write("\n")
2076
2077 # -----------------------------------------------------------------------------
2078 # checking if ghostscript is available for checking the output of hp2ps
2079
2080 def genGSCmd(psfile):
2081 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2082
2083 def gsNotWorking():
2084 global gs_working
2085 print "GhostScript not available for hp2ps tests"
2086
2087 global gs_working
2088 gs_working = 0
2089 if config.have_profiling:
2090 if config.gs != '':
2091 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2092 if resultGood == 0:
2093 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2094 if resultBad != 0:
2095 print "GhostScript available for hp2ps tests"
2096 gs_working = 1;
2097 else:
2098 gsNotWorking();
2099 else:
2100 gsNotWorking();
2101 else:
2102 gsNotWorking();
2103
2104 def rm_no_fail( file ):
2105 try:
2106 os.remove( file )
2107 finally:
2108 return
2109
2110 def add_suffix( name, suffix ):
2111 if suffix == '':
2112 return name
2113 else:
2114 return name + '.' + suffix
2115
2116 def add_hs_lhs_suffix(name):
2117 if getTestOpts().c_src:
2118 return add_suffix(name, 'c')
2119 elif getTestOpts().cmm_src:
2120 return add_suffix(name, 'cmm')
2121 elif getTestOpts().objc_src:
2122 return add_suffix(name, 'm')
2123 elif getTestOpts().objcpp_src:
2124 return add_suffix(name, 'mm')
2125 elif getTestOpts().literate:
2126 return add_suffix(name, 'lhs')
2127 else:
2128 return add_suffix(name, 'hs')
2129
2130 def replace_suffix( name, suffix ):
2131 base, suf = os.path.splitext(name)
2132 return base + '.' + suffix
2133
2134 def in_testdir( name ):
2135 return (getTestOpts().testdir + '/' + name)
2136
2137 def qualify( name, suff ):
2138 return in_testdir(add_suffix(name, suff))
2139
2140
2141 # Finding the sample output. The filename is of the form
2142 #
2143 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2144 #
2145 # and we pick the most specific version available. The <version> is
2146 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2147 # more fine-grained control use if_compiler_lt().
2148 #
2149 def platform_wordsize_qualify( name, suff ):
2150
2151 basepath = qualify(name, suff)
2152
2153 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2154 for (platformSpecific, plat) in [(1, '-' + config.platform),
2155 (1, '-' + config.os),
2156 (0, '')]
2157 for ws in ['-ws-' + config.wordsize, '']
2158 for comp in ['-' + config.compiler_type, '']
2159 for vers in ['-' + config.compiler_maj_version, '']]
2160
2161 dir = glob.glob(basepath + '*')
2162 dir = map (lambda d: normalise_slashes_(d), dir)
2163
2164 for (platformSpecific, f) in paths:
2165 if f in dir:
2166 return (platformSpecific,f)
2167
2168 return (0, basepath)
2169
2170 # Clean up prior to the test, so that we can't spuriously conclude
2171 # that it passed on the basis of old run outputs.
2172 def pretest_cleanup(name):
2173 if getTestOpts().outputdir != None:
2174 odir = in_testdir(getTestOpts().outputdir)
2175 try:
2176 shutil.rmtree(odir)
2177 except:
2178 pass
2179 os.mkdir(odir)
2180
2181 rm_no_fail(qualify(name,'comp.stderr'))
2182 rm_no_fail(qualify(name,'run.stderr'))
2183 rm_no_fail(qualify(name,'run.stdout'))
2184 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2185 # simple_build zaps the following:
2186 # rm_nofail(qualify("o"))
2187 # rm_nofail(qualify(""))
2188 # not interested in the return code
2189
2190 # -----------------------------------------------------------------------------
2191 # Return a list of all the files ending in '.T' below the directory dir.
2192
2193 def findTFiles(roots):
2194 return concat(map(findTFiles_,roots))
2195
2196 def findTFiles_(path):
2197 if os.path.isdir(path):
2198 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2199 return findTFiles(paths)
2200 elif path[-2:] == '.T':
2201 return [path]
2202 else:
2203 return []
2204
2205 # -----------------------------------------------------------------------------
2206 # Output a test summary to the specified file object
2207
2208 def summary(t, file):
2209
2210 file.write('\n')
2211 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2212 file.write('OVERALL SUMMARY for test run started at ' \
2213 + t.start_time + '\n'\
2214 + string.rjust(`t.total_tests`, 8) \
2215 + ' total tests, which gave rise to\n' \
2216 + string.rjust(`t.total_test_cases`, 8) \
2217 + ' test cases, of which\n' \
2218 + string.rjust(`t.n_framework_failures`, 8) \
2219 + ' caused framework failures\n' \
2220 + string.rjust(`t.n_tests_skipped`, 8)
2221 + ' were skipped\n\n' \
2222 + string.rjust(`t.n_expected_passes`, 8)
2223 + ' expected passes\n' \
2224 + string.rjust(`t.n_missing_libs`, 8)
2225 + ' had missing libraries\n' \
2226 + string.rjust(`t.n_expected_failures`, 8) \
2227 + ' expected failures\n' \
2228 + string.rjust(`t.n_unexpected_passes`, 8) \
2229 + ' unexpected passes\n'
2230 + string.rjust(`t.n_unexpected_failures`, 8) \
2231 + ' unexpected failures\n'
2232 + '\n')
2233
2234 if t.n_unexpected_passes > 0:
2235 file.write('Unexpected passes:\n')
2236 printPassingTestInfosSummary(file, t.unexpected_passes)
2237
2238 if t.n_unexpected_failures > 0:
2239 file.write('Unexpected failures:\n')
2240 printFailingTestInfosSummary(file, t.unexpected_failures)
2241
2242 if config.check_files_written:
2243 checkForFilesWrittenProblems(file)
2244
2245 if stopping():
2246 file.write('WARNING: Testsuite run was terminated early\n')
2247
2248 def printUnexpectedTests(file, testInfoss):
2249 unexpected = []
2250 for testInfos in testInfoss:
2251 directories = testInfos.keys()
2252 for directory in directories:
2253 tests = testInfos[directory].keys()
2254 unexpected += tests
2255 if unexpected != []:
2256 file.write('Unexpected results from:\n')
2257 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2258 file.write('\n')
2259
2260 def printPassingTestInfosSummary(file, testInfos):
2261 directories = testInfos.keys()
2262 directories.sort()
2263 maxDirLen = max(map ((lambda x : len(x)), directories))
2264 for directory in directories:
2265 tests = testInfos[directory].keys()
2266 tests.sort()
2267 for test in tests:
2268 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2269 ' (' + join(testInfos[directory][test],',') + ')\n')
2270 file.write('\n')
2271
2272 def printFailingTestInfosSummary(file, testInfos):
2273 directories = testInfos.keys()
2274 directories.sort()
2275 maxDirLen = max(map ((lambda x : len(x)), directories))
2276 for directory in directories:
2277 tests = testInfos[directory].keys()
2278 tests.sort()
2279 for test in tests:
2280 reasons = testInfos[directory][test].keys()
2281 for reason in reasons:
2282 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2283 ' [' + reason + ']' + \
2284 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2285 file.write('\n')
2286
2287 def getStdout(cmd):
2288 if have_subprocess:
2289 p = subprocess.Popen(cmd,
2290 stdout=subprocess.PIPE,
2291 stderr=subprocess.PIPE)
2292 (stdout, stderr) = p.communicate()
2293 r = p.wait()
2294 if r != 0:
2295 raise Exception("Command failed: " + str(cmd))
2296 if stderr != '':
2297 raise Exception("stderr from command: " + str(cmd))
2298 return stdout
2299 else:
2300 raise Exception("Need subprocess to get stdout, but don't have it")