expect_broken_for should also be added to the list of brokens
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 global brokens
155 brokens.append((bug, name))
156 opts.expect = 'fail';
157
158 def expect_broken_for( bug, ways ):
159 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
160
161 def _expect_broken_for( name, opts, bug, ways ):
162 global brokens
163 brokens.append((bug, name))
164 opts.expect_fail_for = ways
165
166 # -----
167
168 def omit_ways( ways ):
169 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
170
171 def _omit_ways( name, opts, ways ):
172 opts.omit_ways = ways
173
174 # -----
175
176 def only_ways( ways ):
177 return lambda name, opts, w=ways: _only_ways( name, opts, w )
178
179 def _only_ways( name, opts, ways ):
180 opts.only_ways = ways
181
182 # -----
183
184 def extra_ways( ways ):
185 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
186
187 def _extra_ways( name, opts, ways ):
188 opts.extra_ways = ways
189
190 # -----
191
192 def omit_compiler_types( compiler_types ):
193 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
194
195 def _omit_compiler_types( name, opts, compiler_types ):
196 if config.compiler_type in compiler_types:
197 opts.skip = 1
198
199 # -----
200
201 def only_compiler_types( compiler_types ):
202 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
203
204 def _only_compiler_types( name, opts, compiler_types ):
205 if config.compiler_type not in compiler_types:
206 opts.skip = 1
207
208 # -----
209
210 def set_stdin( file ):
211 return lambda name, opts, f=file: _set_stdin(name, opts, f);
212
213 def _set_stdin( name, opts, f ):
214 opts.stdin = f
215
216 # -----
217
218 def exit_code( val ):
219 return lambda name, opts, v=val: _exit_code(name, opts, v);
220
221 def _exit_code( name, opts, v ):
222 opts.exit_code = v
223
224 # -----
225
226 def timeout_multiplier( val ):
227 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
228
229 def _timeout_multiplier( name, opts, v ):
230 opts.timeout_multiplier = v
231
232 # -----
233
234 def extra_run_opts( val ):
235 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
236
237 def _extra_run_opts( name, opts, v ):
238 opts.extra_run_opts = v
239
240 # -----
241
242 def extra_hc_opts( val ):
243 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
244
245 def _extra_hc_opts( name, opts, v ):
246 opts.extra_hc_opts = v
247
248 # -----
249
250 def extra_clean( files ):
251 return lambda name, opts, v=files: _extra_clean(name, opts, v);
252
253 def _extra_clean( name, opts, v ):
254 opts.clean_files = v
255
256 # -----
257
258 def stats_num_field( field, expecteds ):
259 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
260
261 def _stats_num_field( name, opts, field, expecteds ):
262 if field in opts.stats_range_fields:
263 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
264
265 if type(expecteds) is types.ListType:
266 for (b, expected, dev) in expecteds:
267 if b:
268 opts.stats_range_fields[field] = (expected, dev)
269 return
270 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
271
272 else:
273 (expected, dev) = expecteds
274 opts.stats_range_fields[field] = (expected, dev)
275
276 def compiler_stats_num_field( field, expecteds ):
277 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
278
279 def _compiler_stats_num_field( name, opts, field, expecteds ):
280 if field in opts.compiler_stats_range_fields:
281 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
282
283 for (b, expected, dev) in expecteds:
284 if b:
285 opts.compiler_stats_range_fields[field] = (expected, dev)
286 return
287
288 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
289
290 # -----
291
292 def skip_if_no_ghci(name, opts):
293 if not ('ghci' in config.run_ways):
294 opts.skip = 1
295
296 # ----
297
298 def skip_if_fast(name, opts):
299 if config.fast:
300 opts.skip = 1
301
302 # -----
303
304 def when(b, f):
305 # When list_brokens is on, we want to see all expect_broken calls,
306 # so we always do f
307 if b or config.list_broken:
308 return f
309 else:
310 return normal
311
312 def unless(b, f):
313 return when(not b, f)
314
315 def platform( plat ):
316 return config.platform == plat
317
318 def opsys( os ):
319 return config.os == os
320
321 def if_arch( arch, f ):
322 if config.arch == arch:
323 return f
324 else:
325 return normal
326
327 def unless_arch( arch, f ):
328 if config.arch == arch:
329 return normal
330 else:
331 return f
332
333 def wordsize( ws ):
334 return config.wordsize == str(ws)
335
336 def if_unregisterised( f ):
337 if config.unregisterised:
338 return f
339 else:
340 return normal
341
342 def unless_unregisterised( f ):
343 if config.unregisterised:
344 return normal
345 else:
346 return f
347
348 def if_msys( f ):
349 if config.msys:
350 return f
351 else:
352 return normal
353
354 def if_cygwin( f ):
355 if config.cygwin:
356 return f
357 else:
358 return normal
359
360 def when_have_vanilla( f ):
361 if config.have_vanilla:
362 return f
363 else:
364 return normal
365
366 def unless_have_vanilla( f ):
367 if config.have_vanilla:
368 return normal
369 else:
370 return f
371
372 def when_have_dynamic( f ):
373 if config.have_dynamic:
374 return f
375 else:
376 return normal
377
378 def unless_have_dynamic( f ):
379 if config.have_dynamic:
380 return normal
381 else:
382 return f
383
384 def when_have_profiling( f ):
385 if config.have_profiling:
386 return f
387 else:
388 return normal
389
390 def unless_have_profiling( f ):
391 if config.have_profiling:
392 return normal
393 else:
394 return f
395
396 # ---
397
398 def if_ghci_dynamic( f ):
399 if config.ghc_dynamic_by_default:
400 return f
401 else:
402 return normal
403
404 def if_in_tree_compiler( f ):
405 if config.in_tree_compiler:
406 return f
407 else:
408 return normal
409
410 def unless_in_tree_compiler( f ):
411 if config.in_tree_compiler:
412 return normal
413 else:
414 return f
415
416 def if_compiler_type( compiler, f ):
417 if config.compiler_type == compiler:
418 return f
419 else:
420 return normal
421
422 def if_compiler_profiled( f ):
423 if config.compiler_profiled:
424 return f
425 else:
426 return normal
427
428 def unless_compiler_profiled( f ):
429 if config.compiler_profiled:
430 return normal
431 else:
432 return f
433
434 def if_compiler_lt( compiler, version, f ):
435 if config.compiler_type == compiler and \
436 version_lt(config.compiler_version, version):
437 return f
438 else:
439 return normal
440
441 def if_compiler_le( compiler, version, f ):
442 if config.compiler_type == compiler and \
443 version_le(config.compiler_version, version):
444 return f
445 else:
446 return normal
447
448 def if_compiler_gt( compiler, version, f ):
449 if config.compiler_type == compiler and \
450 version_gt(config.compiler_version, version):
451 return f
452 else:
453 return normal
454
455 def if_compiler_ge( compiler, version, f ):
456 if config.compiler_type == compiler and \
457 version_ge(config.compiler_version, version):
458 return f
459 else:
460 return normal
461
462 def if_compiler_debugged( f ):
463 if config.compiler_debugged:
464 return f
465 else:
466 return normal
467
468 def namebase( nb ):
469 return lambda opts, nb=nb: _namebase(opts, nb)
470
471 def _namebase( opts, nb ):
472 opts.with_namebase = nb
473
474 # ---
475
476 def if_tag( tag, f ):
477 if tag in config.compiler_tags:
478 return f
479 else:
480 return normal
481
482 def unless_tag( tag, f ):
483 if not (tag in config.compiler_tags):
484 return f
485 else:
486 return normal
487
488 # ---
489 def high_memory_usage(name, opts):
490 opts.alone = True
491
492 # ---
493 def literate( name, opts ):
494 opts.literate = 1;
495
496 def c_src( name, opts ):
497 opts.c_src = 1;
498
499 def objc_src( name, opts ):
500 opts.objc_src = 1;
501
502 def objcpp_src( name, opts ):
503 opts.objcpp_src = 1;
504
505 def cmm_src( name, opts ):
506 opts.cmm_src = 1;
507
508 def outputdir( odir ):
509 return lambda name, opts, d=odir: _outputdir(name, opts, d)
510
511 def _outputdir( name, opts, odir ):
512 opts.outputdir = odir;
513
514 # ----
515
516 def pre_cmd( cmd ):
517 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
518
519 def _pre_cmd( name, opts, cmd ):
520 opts.pre_cmd = cmd
521
522 # ----
523
524 def clean_cmd( cmd ):
525 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
526
527 def _clean_cmd( name, opts, cmd ):
528 opts.clean_cmd = cmd
529
530 # ----
531
532 def cmd_prefix( prefix ):
533 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
534
535 def _cmd_prefix( name, opts, prefix ):
536 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
537
538 # ----
539
540 def cmd_wrapper( fun ):
541 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
542
543 def _cmd_wrapper( name, opts, fun ):
544 opts.cmd_wrapper = fun
545
546 # ----
547
548 def compile_cmd_prefix( prefix ):
549 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
550
551 def _compile_cmd_prefix( name, opts, prefix ):
552 opts.compile_cmd_prefix = prefix
553
554 # ----
555
556 def normalise_slashes( name, opts ):
557 opts.extra_normaliser = normalise_slashes_
558
559 def normalise_exe( name, opts ):
560 opts.extra_normaliser = normalise_exe_
561
562 def normalise_fun( fun ):
563 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
564
565 def _normalise_fun( name, opts, f ):
566 opts.extra_normaliser = f
567
568 def normalise_errmsg_fun( fun ):
569 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
570
571 def _normalise_errmsg_fun( name, opts, f ):
572 opts.extra_errmsg_normaliser = f
573
574 def two_normalisers(f, g):
575 return lambda x, f=f, g=g: f(g(x))
576
577 # ----
578 # Function for composing two opt-fns together
579
580 def composes( fs ):
581 return reduce(lambda f, g: compose(f, g), fs)
582
583 def compose( f, g ):
584 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
585
586 def _compose( name, opts, f, g ):
587 f(name, opts)
588 g(name, opts)
589
590 # -----------------------------------------------------------------------------
591 # The current directory of tests
592
593 def newTestDir( dir ):
594 global thisdir_settings
595 # reset the options for this test directory
596 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
597
598 def _newTestDir( name, opts, dir ):
599 opts.testdir = dir
600 opts.compiler_always_flags = config.compiler_always_flags
601
602 # -----------------------------------------------------------------------------
603 # Actually doing tests
604
605 parallelTests = []
606 aloneTests = []
607 allTestNames = set([])
608
609 def runTest (opts, name, func, args):
610 ok = 0
611
612 if config.use_threads:
613 t.thread_pool.acquire()
614 try:
615 while config.threads<(t.running_threads+1):
616 t.thread_pool.wait()
617 t.running_threads = t.running_threads+1
618 ok=1
619 t.thread_pool.release()
620 thread.start_new_thread(test_common_thread, (name, opts, func, args))
621 except:
622 if not ok:
623 t.thread_pool.release()
624 else:
625 test_common_work (name, opts, func, args)
626
627 # name :: String
628 # setup :: TestOpts -> IO ()
629 def test (name, setup, func, args):
630 global aloneTests
631 global parallelTests
632 global allTestNames
633 global thisdir_settings
634 if name in allTestNames:
635 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
636 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
637 framework_fail(name, 'bad_name', 'This test has an invalid name')
638
639 # Make a deep copy of the default_testopts, as we need our own copy
640 # of any dictionaries etc inside it. Otherwise, if one test modifies
641 # them, all tests will see the modified version!
642 myTestOpts = copy.deepcopy(default_testopts)
643
644 if type(setup) is types.ListType:
645 setup = composes(setup)
646
647 setup = compose(thisdir_settings, setup)
648 setup(name, myTestOpts)
649
650 thisTest = lambda : runTest(myTestOpts, name, func, args)
651 if myTestOpts.alone:
652 aloneTests.append(thisTest)
653 else:
654 parallelTests.append(thisTest)
655 allTestNames.add(name)
656
657 if config.use_threads:
658 def test_common_thread(name, opts, func, args):
659 t.lock.acquire()
660 try:
661 test_common_work(name,opts,func,args)
662 finally:
663 t.lock.release()
664 t.thread_pool.acquire()
665 t.running_threads = t.running_threads - 1
666 t.thread_pool.notify()
667 t.thread_pool.release()
668
669 def get_package_cache_timestamp():
670 if config.package_conf_cache_file == '':
671 return 0.0
672 else:
673 try:
674 return os.stat(config.package_conf_cache_file).st_mtime
675 except:
676 return 0.0
677
678
679 def test_common_work (name, opts, func, args):
680 try:
681 t.total_tests = t.total_tests+1
682 setLocalTestOpts(opts)
683
684 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
685
686 # All the ways we might run this test
687 if func == compile or func == multimod_compile:
688 all_ways = config.compile_ways
689 elif func == compile_and_run or func == multimod_compile_and_run:
690 all_ways = config.run_ways
691 elif func == ghci_script:
692 if 'ghci' in config.run_ways:
693 all_ways = ['ghci']
694 else:
695 all_ways = []
696 else:
697 all_ways = ['normal']
698
699 # A test itself can request extra ways by setting opts.extra_ways
700 all_ways = all_ways + filter(lambda way: way not in all_ways,
701 opts.extra_ways)
702
703 t.total_test_cases = t.total_test_cases + len(all_ways)
704
705 ok_way = lambda way: \
706 not getTestOpts().skip \
707 and (config.only == [] or name in config.only) \
708 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
709 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
710 and way not in getTestOpts().omit_ways
711
712 # Which ways we are asked to skip
713 do_ways = filter (ok_way,all_ways)
714
715 # In fast mode, we skip all but one way
716 if config.fast and len(do_ways) > 0:
717 do_ways = [do_ways[0]]
718
719 if not config.clean_only:
720 # Run the required tests...
721 for way in do_ways:
722 if stopping():
723 break
724 do_test (name, way, func, args)
725
726 for way in all_ways:
727 if way not in do_ways:
728 skiptest (name,way)
729
730 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
731 clean(map (lambda suff: name + suff,
732 ['', '.exe', '.exe.manifest', '.genscript',
733 '.stderr.normalised', '.stdout.normalised',
734 '.run.stderr', '.run.stdout',
735 '.run.stderr.normalised', '.run.stdout.normalised',
736 '.comp.stderr', '.comp.stdout',
737 '.comp.stderr.normalised', '.comp.stdout.normalised',
738 '.interp.stderr', '.interp.stdout',
739 '.interp.stderr.normalised', '.interp.stdout.normalised',
740 '.stats', '.comp.stats',
741 '.hi', '.o', '.prof', '.exe.prof', '.hc',
742 '_stub.h', '_stub.c', '_stub.o',
743 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
744
745 if func == multi_compile or func == multi_compile_fail:
746 extra_mods = args[1]
747 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
748 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
749
750 clean(getTestOpts().clean_files)
751
752 if getTestOpts().outputdir != None:
753 odir = in_testdir(getTestOpts().outputdir)
754 try:
755 shutil.rmtree(odir)
756 except:
757 pass
758
759 try:
760 shutil.rmtree(in_testdir('.hpc.' + name))
761 except:
762 pass
763
764 try:
765 cleanCmd = getTestOpts().clean_cmd
766 if cleanCmd != None:
767 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
768 if result != 0:
769 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
770 except e:
771 framework_fail(name, 'cleaning', 'clean-command exception')
772
773 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
774
775 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
776 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
777
778 try:
779 for f in files_written[name]:
780 if os.path.exists(f):
781 try:
782 if not f in files_written_not_removed[name]:
783 files_written_not_removed[name].append(f)
784 except:
785 files_written_not_removed[name] = [f]
786 except:
787 pass
788 except Exception, e:
789 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
790
791 def clean(strs):
792 for str in strs:
793 for name in glob.glob(in_testdir(str)):
794 clean_full_path(name)
795
796 def clean_full_path(name):
797 try:
798 # Remove files...
799 os.remove(name)
800 except OSError, e1:
801 try:
802 # ... and empty directories
803 os.rmdir(name)
804 except OSError, e2:
805 # We don't want to fail here, but we do want to know
806 # what went wrong, so print out the exceptions.
807 # ENOENT isn't a problem, though, as we clean files
808 # that don't necessarily exist.
809 if e1.errno != errno.ENOENT:
810 print e1
811 if e2.errno != errno.ENOENT:
812 print e2
813
814 def do_test(name, way, func, args):
815 full_name = name + '(' + way + ')'
816
817 try:
818 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
819 str([t.n_unexpected_passes, \
820 t.n_unexpected_failures, \
821 t.n_framework_failures])
822
823 if config.use_threads:
824 t.lock.release()
825
826 try:
827 preCmd = getTestOpts().pre_cmd
828 if preCmd != None:
829 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
830 if result != 0:
831 framework_fail(name, way, 'pre-command failed: ' + str(result))
832 except e:
833 framework_fail(name, way, 'pre-command exception')
834
835 try:
836 result = apply(func, [name,way] + args)
837 finally:
838 if config.use_threads:
839 t.lock.acquire()
840
841 if getTestOpts().expect != 'pass' and \
842 getTestOpts().expect != 'fail' and \
843 getTestOpts().expect != 'missing-lib':
844 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
845
846 try:
847 passFail = result['passFail']
848 except:
849 passFail = 'No passFail found'
850
851 if passFail == 'pass':
852 if getTestOpts().expect == 'pass' \
853 and way not in getTestOpts().expect_fail_for:
854 t.n_expected_passes = t.n_expected_passes + 1
855 if name in t.expected_passes:
856 t.expected_passes[name].append(way)
857 else:
858 t.expected_passes[name] = [way]
859 else:
860 print '*** unexpected pass for', full_name
861 t.n_unexpected_passes = t.n_unexpected_passes + 1
862 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
863 elif passFail == 'fail':
864 if getTestOpts().expect == 'pass' \
865 and way not in getTestOpts().expect_fail_for:
866 print '*** unexpected failure for', full_name
867 t.n_unexpected_failures = t.n_unexpected_failures + 1
868 reason = result['reason']
869 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
870 else:
871 if getTestOpts().expect == 'missing-lib':
872 t.n_missing_libs = t.n_missing_libs + 1
873 if name in t.missing_libs:
874 t.missing_libs[name].append(way)
875 else:
876 t.missing_libs[name] = [way]
877 else:
878 t.n_expected_failures = t.n_expected_failures + 1
879 if name in t.expected_failures:
880 t.expected_failures[name].append(way)
881 else:
882 t.expected_failures[name] = [way]
883 else:
884 framework_fail(name, way, 'bad result ' + passFail)
885 except KeyboardInterrupt:
886 stopNow()
887 except:
888 framework_fail(name, way, 'do_test exception')
889 traceback.print_exc()
890
891 def addPassingTestInfo (testInfos, directory, name, way):
892 directory = re.sub('^\\.[/\\\\]', '', directory)
893
894 if not directory in testInfos:
895 testInfos[directory] = {}
896
897 if not name in testInfos[directory]:
898 testInfos[directory][name] = []
899
900 testInfos[directory][name].append(way)
901
902 def addFailingTestInfo (testInfos, directory, name, reason, way):
903 directory = re.sub('^\\.[/\\\\]', '', directory)
904
905 if not directory in testInfos:
906 testInfos[directory] = {}
907
908 if not name in testInfos[directory]:
909 testInfos[directory][name] = {}
910
911 if not reason in testInfos[directory][name]:
912 testInfos[directory][name][reason] = []
913
914 testInfos[directory][name][reason].append(way)
915
916 def skiptest (name, way):
917 # print 'Skipping test \"', name, '\"'
918 t.n_tests_skipped = t.n_tests_skipped + 1
919 if name in t.tests_skipped:
920 t.tests_skipped[name].append(way)
921 else:
922 t.tests_skipped[name] = [way]
923
924 def framework_fail( name, way, reason ):
925 full_name = name + '(' + way + ')'
926 print '*** framework failure for', full_name, reason
927 t.n_framework_failures = t.n_framework_failures + 1
928 if name in t.framework_failures:
929 t.framework_failures[name].append(way)
930 else:
931 t.framework_failures[name] = [way]
932
933 def badResult(result):
934 try:
935 if result['passFail'] == 'pass':
936 return False
937 return True
938 except:
939 return True
940
941 def passed():
942 return {'passFail': 'pass'}
943
944 def failBecause(reason):
945 return {'passFail': 'fail', 'reason': reason}
946
947 # -----------------------------------------------------------------------------
948 # Generic command tests
949
950 # A generic command test is expected to run and exit successfully.
951 #
952 # The expected exit code can be changed via exit_code() as normal, and
953 # the expected stdout/stderr are stored in <testname>.stdout and
954 # <testname>.stderr. The output of the command can be ignored
955 # altogether by using run_command_ignore_output instead of
956 # run_command.
957
958 def run_command( name, way, cmd ):
959 return simple_run( name, '', cmd, '' )
960
961 # -----------------------------------------------------------------------------
962 # GHCi tests
963
964 def ghci_script( name, way, script ):
965 # filter out -fforce-recomp from compiler_always_flags, because we're
966 # actually testing the recompilation behaviour in the GHCi tests.
967 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
968 flags.append(getTestOpts().extra_hc_opts)
969 if getTestOpts().outputdir != None:
970 flags.extend(["-outputdir", getTestOpts().outputdir])
971
972 # We pass HC and HC_OPTS as environment variables, so that the
973 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
974 cmd = "HC='" + config.compiler + "' " + \
975 "HC_OPTS='" + join(flags,' ') + "' " + \
976 "'" + config.compiler + "'" + \
977 ' --interactive -v0 -ignore-dot-ghci ' + \
978 join(flags,' ')
979
980 getTestOpts().stdin = script
981 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
982
983 # -----------------------------------------------------------------------------
984 # Compile-only tests
985
986 def compile( name, way, extra_hc_opts ):
987 return do_compile( name, way, 0, '', [], extra_hc_opts )
988
989 def compile_fail( name, way, extra_hc_opts ):
990 return do_compile( name, way, 1, '', [], extra_hc_opts )
991
992 def multimod_compile( name, way, top_mod, extra_hc_opts ):
993 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
994
995 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
996 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
997
998 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
999 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1000
1001 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1002 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1003
1004 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1005 # print 'Compile only, extra args = ', extra_hc_opts
1006 pretest_cleanup(name)
1007
1008 result = extras_build( way, extra_mods, extra_hc_opts )
1009 if badResult(result):
1010 return result
1011 extra_hc_opts = result['hc_opts']
1012
1013 force = 0
1014 if extra_mods:
1015 force = 1
1016 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1017
1018 if badResult(result):
1019 return result
1020
1021 # the actual stderr should always match the expected, regardless
1022 # of whether we expected the compilation to fail or not (successful
1023 # compilations may generate warnings).
1024
1025 if getTestOpts().with_namebase == None:
1026 namebase = name
1027 else:
1028 namebase = getTestOpts().with_namebase
1029
1030 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1031 actual_stderr_file = qualify(name, 'comp.stderr')
1032
1033 if not compare_outputs('stderr', \
1034 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1035 expected_stderr_file, actual_stderr_file):
1036 return failBecause('stderr mismatch')
1037
1038 # no problems found, this test passed
1039 return passed()
1040
1041 def compile_cmp_asm( name, way, extra_hc_opts ):
1042 print 'Compile only, extra args = ', extra_hc_opts
1043 pretest_cleanup(name)
1044 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1045
1046 if badResult(result):
1047 return result
1048
1049 # the actual stderr should always match the expected, regardless
1050 # of whether we expected the compilation to fail or not (successful
1051 # compilations may generate warnings).
1052
1053 if getTestOpts().with_namebase == None:
1054 namebase = name
1055 else:
1056 namebase = getTestOpts().with_namebase
1057
1058 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1059 actual_asm_file = qualify(name, 's')
1060
1061 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1062 expected_asm_file, actual_asm_file):
1063 return failBecause('asm mismatch')
1064
1065 # no problems found, this test passed
1066 return passed()
1067
1068 # -----------------------------------------------------------------------------
1069 # Compile-and-run tests
1070
1071 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1072 # print 'Compile and run, extra args = ', extra_hc_opts
1073 pretest_cleanup(name)
1074
1075 result = extras_build( way, extra_mods, extra_hc_opts )
1076 if badResult(result):
1077 return result
1078 extra_hc_opts = result['hc_opts']
1079
1080 if way == 'ghci': # interpreted...
1081 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1082 elif way == 'extcore' or way == 'optextcore' :
1083 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1084 else: # compiled...
1085 force = 0
1086 if extra_mods:
1087 force = 1
1088
1089 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1090 if badResult(result):
1091 return result
1092
1093 cmd = './' + name;
1094
1095 # we don't check the compiler's stderr for a compile-and-run test
1096 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1097
1098 def compile_and_run( name, way, extra_hc_opts ):
1099 return compile_and_run__( name, way, '', [], extra_hc_opts)
1100
1101 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1102 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1103
1104 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1105 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1106
1107 def stats( name, way, stats_file ):
1108 opts = getTestOpts()
1109 return checkStats(stats_file, opts.stats_range_fields)
1110
1111 # -----------------------------------------------------------------------------
1112 # Check -t stats info
1113
1114 def checkStats(stats_file, range_fields):
1115 result = passed()
1116 if len(range_fields) > 0:
1117 f = open(in_testdir(stats_file))
1118 contents = f.read()
1119 f.close()
1120
1121 for (field, (expected, dev)) in range_fields.items():
1122 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1123 if m == None:
1124 print 'Failed to find field: ', field
1125 result = failBecause('no such stats field')
1126 val = int(m.group(1))
1127
1128 lowerBound = trunc( expected * ((100 - float(dev))/100));
1129 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1130
1131 if val < lowerBound:
1132 print field, 'value is too low:'
1133 print '(If this is because you have improved GHC, please'
1134 print 'update the test so that GHC doesn\'t regress again)'
1135 result = failBecause('stat too good')
1136 if val > upperBound:
1137 print field, 'value is too high:'
1138 result = failBecause('stat not good enough')
1139
1140 if val < lowerBound or val > upperBound:
1141 valStr = str(val)
1142 valLen = len(valStr)
1143 expectedStr = str(expected)
1144 expectedLen = len(expectedStr)
1145 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1146 def display(descr, val, extra):
1147 print descr, string.rjust(str(val), length), extra
1148 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1149 display(' Lower bound ' + field + ':', lowerBound, '')
1150 display(' Upper bound ' + field + ':', upperBound, '')
1151 display(' Actual ' + field + ':', val, '')
1152
1153 return result
1154
1155 # -----------------------------------------------------------------------------
1156 # Build a single-module program
1157
1158 def extras_build( way, extra_mods, extra_hc_opts ):
1159 for modopts in extra_mods:
1160 mod, opts = modopts
1161 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1162 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1163 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1164 if badResult(result):
1165 return result
1166
1167 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1168
1169
1170 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1171 opts = getTestOpts()
1172 errname = add_suffix(name, 'comp.stderr')
1173 rm_no_fail( qualify(errname, '') )
1174
1175 if top_mod != '':
1176 srcname = top_mod
1177 rm_no_fail( qualify(name, '') )
1178 base, suf = os.path.splitext(top_mod)
1179 rm_no_fail( qualify(base, '') )
1180 rm_no_fail( qualify(base, 'exe') )
1181 elif addsuf:
1182 srcname = add_hs_lhs_suffix(name)
1183 rm_no_fail( qualify(name, '') )
1184 else:
1185 srcname = name
1186 rm_no_fail( qualify(name, 'o') )
1187
1188 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1189
1190 to_do = ''
1191 if top_mod != '':
1192 to_do = '--make '
1193 if link:
1194 to_do = to_do + '-o ' + name
1195 elif link:
1196 to_do = '-o ' + name
1197 elif opts.compile_to_hc:
1198 to_do = '-C'
1199 else:
1200 to_do = '-c' # just compile
1201
1202 stats_file = name + '.comp.stats'
1203 if len(opts.compiler_stats_range_fields) > 0:
1204 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1205
1206 # Required by GHC 7.3+, harmless for earlier versions:
1207 if (getTestOpts().c_src or
1208 getTestOpts().objc_src or
1209 getTestOpts().objcpp_src or
1210 getTestOpts().cmm_src):
1211 extra_hc_opts += ' -no-hs-main '
1212
1213 if getTestOpts().compile_cmd_prefix == '':
1214 cmd_prefix = ''
1215 else:
1216 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1217
1218 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1219 if noforce:
1220 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1221 if getTestOpts().outputdir != None:
1222 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1223
1224 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1225 + config.compiler + "' " \
1226 + join(comp_flags,' ') + ' ' \
1227 + to_do + ' ' + srcname + ' ' \
1228 + join(config.way_flags(name)[way],' ') + ' ' \
1229 + extra_hc_opts + ' ' \
1230 + opts.extra_hc_opts + ' ' \
1231 + '>' + errname + ' 2>&1'
1232
1233 result = runCmdFor(name, cmd)
1234
1235 if result != 0 and not should_fail:
1236 actual_stderr = qualify(name, 'comp.stderr')
1237 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1238 if_verbose_dump(1,actual_stderr)
1239
1240 # ToDo: if the sub-shell was killed by ^C, then exit
1241
1242 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1243
1244 if badResult(statsResult):
1245 return statsResult
1246
1247 if should_fail:
1248 if result == 0:
1249 return failBecause('exit code 0')
1250 else:
1251 if result != 0:
1252 return failBecause('exit code non-0')
1253
1254 return passed()
1255
1256 # -----------------------------------------------------------------------------
1257 # Run a program and check its output
1258 #
1259 # If testname.stdin exists, route input from that, else
1260 # from /dev/null. Route output to testname.run.stdout and
1261 # testname.run.stderr. Returns the exit code of the run.
1262
1263 def simple_run( name, way, prog, args ):
1264 opts = getTestOpts()
1265
1266 # figure out what to use for stdin
1267 if opts.stdin != '':
1268 use_stdin = opts.stdin
1269 else:
1270 stdin_file = add_suffix(name, 'stdin')
1271 if os.path.exists(in_testdir(stdin_file)):
1272 use_stdin = stdin_file
1273 else:
1274 use_stdin = '/dev/null'
1275
1276 run_stdout = add_suffix(name,'run.stdout')
1277 run_stderr = add_suffix(name,'run.stderr')
1278
1279 rm_no_fail(qualify(name,'run.stdout'))
1280 rm_no_fail(qualify(name,'run.stderr'))
1281 rm_no_fail(qualify(name, 'hp'))
1282 rm_no_fail(qualify(name,'ps'))
1283 rm_no_fail(qualify(name, 'prof'))
1284
1285 my_rts_flags = rts_flags(way)
1286
1287 stats_file = name + '.stats'
1288 if len(opts.stats_range_fields) > 0:
1289 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1290
1291 if opts.no_stdin:
1292 stdin_comes_from = ''
1293 else:
1294 stdin_comes_from = ' <' + use_stdin
1295
1296 if opts.combined_output:
1297 redirection = ' >' + run_stdout \
1298 + ' 2>&1'
1299 else:
1300 redirection = ' >' + run_stdout \
1301 + ' 2>' + run_stderr
1302
1303 cmd = prog + ' ' + args + ' ' \
1304 + my_rts_flags + ' ' \
1305 + stdin_comes_from \
1306 + redirection
1307
1308 if opts.cmd_wrapper != None:
1309 cmd = opts.cmd_wrapper(cmd);
1310
1311 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1312
1313 # run the command
1314 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1315
1316 exit_code = result >> 8
1317 signal = result & 0xff
1318
1319 # check the exit code
1320 if exit_code != opts.exit_code:
1321 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1322 dump_stdout(name)
1323 dump_stderr(name)
1324 return failBecause('bad exit code')
1325
1326 check_hp = my_rts_flags.find("-h") != -1
1327 check_prof = my_rts_flags.find("-p") != -1
1328
1329 if not opts.ignore_output:
1330 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1331 bad_stdout = not check_stdout_ok(name)
1332 if bad_stderr:
1333 return failBecause('bad stderr')
1334 if bad_stdout:
1335 return failBecause('bad stdout')
1336 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1337 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1338 return failBecause('bad heap profile')
1339 if check_prof and not check_prof_ok(name):
1340 return failBecause('bad profile')
1341
1342 return checkStats(stats_file, opts.stats_range_fields)
1343
1344 def rts_flags(way):
1345 if (way == ''):
1346 return ''
1347 else:
1348 args = config.way_rts_flags[way]
1349
1350 if args == []:
1351 return ''
1352 else:
1353 return '+RTS ' + join(args,' ') + ' -RTS'
1354
1355 # -----------------------------------------------------------------------------
1356 # Run a program in the interpreter and check its output
1357
1358 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1359 outname = add_suffix(name, 'interp.stdout')
1360 errname = add_suffix(name, 'interp.stderr')
1361 rm_no_fail(outname)
1362 rm_no_fail(errname)
1363 rm_no_fail(name)
1364
1365 if (top_mod == ''):
1366 srcname = add_hs_lhs_suffix(name)
1367 else:
1368 srcname = top_mod
1369
1370 scriptname = add_suffix(name, 'genscript')
1371 qscriptname = in_testdir(scriptname)
1372 rm_no_fail(qscriptname)
1373
1374 delimiter = '===== program output begins here\n'
1375
1376 script = open(qscriptname, 'w')
1377 if not compile_only:
1378 # set the prog name and command-line args to match the compiled
1379 # environment.
1380 script.write(':set prog ' + name + '\n')
1381 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1382 # Add marker lines to the stdout and stderr output files, so we
1383 # can separate GHCi's output from the program's.
1384 script.write(':! echo ' + delimiter)
1385 script.write(':! echo 1>&2 ' + delimiter)
1386 # Set stdout to be line-buffered to match the compiled environment.
1387 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1388 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1389 # in the event of an exception as for the compiled program.
1390 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1391 script.close()
1392
1393 # figure out what to use for stdin
1394 if getTestOpts().stdin != '':
1395 stdin_file = in_testdir(getTestOpts().stdin)
1396 else:
1397 stdin_file = qualify(name, 'stdin')
1398
1399 if os.path.exists(stdin_file):
1400 stdin = open(stdin_file, 'r')
1401 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1402
1403 script.close()
1404
1405 flags = copy.copy(getTestOpts().compiler_always_flags)
1406 if getTestOpts().outputdir != None:
1407 flags.extend(["-outputdir", getTestOpts().outputdir])
1408
1409 cmd = "'" + config.compiler + "' " \
1410 + join(flags,' ') + ' ' \
1411 + srcname + ' ' \
1412 + join(config.way_flags(name)[way],' ') + ' ' \
1413 + extra_hc_opts + ' ' \
1414 + getTestOpts().extra_hc_opts + ' ' \
1415 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1416
1417 if getTestOpts().cmd_wrapper != None:
1418 cmd = getTestOpts().cmd_wrapper(cmd);
1419
1420 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1421
1422 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1423
1424 exit_code = result >> 8
1425 signal = result & 0xff
1426
1427 # split the stdout into compilation/program output
1428 split_file(in_testdir(outname), delimiter,
1429 qualify(name, 'comp.stdout'),
1430 qualify(name, 'run.stdout'))
1431 split_file(in_testdir(errname), delimiter,
1432 qualify(name, 'comp.stderr'),
1433 qualify(name, 'run.stderr'))
1434
1435 # check the exit code
1436 if exit_code != getTestOpts().exit_code:
1437 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1438 dump_stdout(name)
1439 dump_stderr(name)
1440 return failBecause('bad exit code')
1441
1442 # ToDo: if the sub-shell was killed by ^C, then exit
1443
1444 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1445 check_stdout_ok(name)):
1446 return passed()
1447 else:
1448 return failBecause('bad stdout or stderr')
1449
1450
1451 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1452 infile = open(in_fn)
1453 out1 = open(out1_fn, 'w')
1454 out2 = open(out2_fn, 'w')
1455
1456 line = infile.readline()
1457 line = re.sub('\r', '', line) # ignore Windows EOL
1458 while (re.sub('^\s*','',line) != delimiter and line != ''):
1459 out1.write(line)
1460 line = infile.readline()
1461 line = re.sub('\r', '', line)
1462 out1.close()
1463
1464 line = infile.readline()
1465 while (line != ''):
1466 out2.write(line)
1467 line = infile.readline()
1468 out2.close()
1469
1470 # -----------------------------------------------------------------------------
1471 # Generate External Core for the given program, then compile the resulting Core
1472 # and compare its output to the expected output
1473
1474 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1475
1476 depsfilename = qualify(name, 'deps')
1477 errname = add_suffix(name, 'comp.stderr')
1478 qerrname = qualify(errname,'')
1479
1480 hcname = qualify(name, 'hc')
1481 oname = qualify(name, 'o')
1482
1483 rm_no_fail( qerrname )
1484 rm_no_fail( qualify(name, '') )
1485
1486 if (top_mod == ''):
1487 srcname = add_hs_lhs_suffix(name)
1488 else:
1489 srcname = top_mod
1490
1491 qcorefilename = qualify(name, 'hcr')
1492 corefilename = add_suffix(name, 'hcr')
1493 rm_no_fail(qcorefilename)
1494
1495 # Generate External Core
1496
1497 if (top_mod == ''):
1498 to_do = ' ' + srcname + ' '
1499 else:
1500 to_do = ' --make ' + top_mod + ' '
1501
1502 flags = copy.copy(getTestOpts().compiler_always_flags)
1503 if getTestOpts().outputdir != None:
1504 flags.extend(["-outputdir", getTestOpts().outputdir])
1505 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1506 + config.compiler + "' " \
1507 + join(flags,' ') + ' ' \
1508 + join(config.way_flags(name)[way],' ') + ' ' \
1509 + extra_hc_opts + ' ' \
1510 + getTestOpts().extra_hc_opts \
1511 + to_do \
1512 + '>' + errname + ' 2>&1'
1513 result = runCmdFor(name, cmd)
1514
1515 exit_code = result >> 8
1516
1517 if exit_code != 0:
1518 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1519 if_verbose_dump(1,qerrname)
1520 return failBecause('ext core exit code non-0')
1521
1522 # Compile the resulting files -- if there's more than one module, we need to read the output
1523 # of the previous compilation in order to find the dependencies
1524 if (top_mod == ''):
1525 to_compile = corefilename
1526 else:
1527 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1528 deps = open(depsfilename).read()
1529 deplist = string.replace(deps, '\n',' ');
1530 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1531 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1532
1533 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1534 if getTestOpts().outputdir != None:
1535 flags.extend(["-outputdir", getTestOpts().outputdir])
1536
1537 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1538 + config.compiler + "' " \
1539 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1540 + to_compile + ' ' \
1541 + extra_hc_opts + ' ' \
1542 + getTestOpts().extra_hc_opts + ' ' \
1543 + flags \
1544 + ' -fglasgow-exts -o ' + name \
1545 + '>' + errname + ' 2>&1'
1546
1547 result = runCmdFor(name, cmd)
1548 exit_code = result >> 8
1549
1550 if exit_code != 0:
1551 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1552 if_verbose_dump(1,qerrname)
1553 return failBecause('ext core exit code non-0')
1554
1555 # Clean up
1556 rm_no_fail ( oname )
1557 rm_no_fail ( hcname )
1558 rm_no_fail ( qcorefilename )
1559 rm_no_fail ( depsfilename )
1560
1561 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1562
1563 # -----------------------------------------------------------------------------
1564 # Utils
1565
1566 def check_stdout_ok( name ):
1567 if getTestOpts().with_namebase == None:
1568 namebase = name
1569 else:
1570 namebase = getTestOpts().with_namebase
1571
1572 actual_stdout_file = qualify(name, 'run.stdout')
1573 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1574
1575 def norm(str):
1576 if platform_specific:
1577 return str
1578 else:
1579 return normalise_output(str)
1580
1581 return compare_outputs('stdout', \
1582 two_normalisers(norm, getTestOpts().extra_normaliser), \
1583 expected_stdout_file, actual_stdout_file)
1584
1585 def dump_stdout( name ):
1586 print 'Stdout:'
1587 print read_no_crs(qualify(name, 'run.stdout'))
1588
1589 def check_stderr_ok( name ):
1590 if getTestOpts().with_namebase == None:
1591 namebase = name
1592 else:
1593 namebase = getTestOpts().with_namebase
1594
1595 actual_stderr_file = qualify(name, 'run.stderr')
1596 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1597
1598 def norm(str):
1599 if platform_specific:
1600 return str
1601 else:
1602 return normalise_errmsg(str)
1603
1604 return compare_outputs('stderr', \
1605 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1606 expected_stderr_file, actual_stderr_file)
1607
1608 def dump_stderr( name ):
1609 print "Stderr:"
1610 print read_no_crs(qualify(name, 'run.stderr'))
1611
1612 def read_no_crs(file):
1613 str = ''
1614 try:
1615 h = open(file)
1616 str = h.read()
1617 h.close
1618 except:
1619 # On Windows, if the program fails very early, it seems the
1620 # files stdout/stderr are redirected to may not get created
1621 pass
1622 return re.sub('\r', '', str)
1623
1624 def write_file(file, str):
1625 h = open(file, 'w')
1626 h.write(str)
1627 h.close
1628
1629 def check_hp_ok(name):
1630
1631 # do not qualify for hp2ps because we should be in the right directory
1632 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1633
1634 hp2psResult = runCmdExitCode(hp2psCmd)
1635
1636 actual_ps_file = qualify(name, 'ps')
1637
1638 if(hp2psResult == 0):
1639 if (os.path.exists(actual_ps_file)):
1640 if gs_working:
1641 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1642 if (gsResult == 0):
1643 return (True)
1644 else:
1645 print "hp2ps output for " + name + "is not valid PostScript"
1646 else: return (True) # assume postscript is valid without ghostscript
1647 else:
1648 print "hp2ps did not generate PostScript for " + name
1649 return (False)
1650 else:
1651 print "hp2ps error when processing heap profile for " + name
1652 return(False)
1653
1654 def check_prof_ok(name):
1655
1656 prof_file = qualify(name,'prof')
1657
1658 if not os.path.exists(prof_file):
1659 print prof_file + " does not exist"
1660 return(False)
1661
1662 if os.path.getsize(qualify(name,'prof')) == 0:
1663 print prof_file + " is empty"
1664 return(False)
1665
1666 if getTestOpts().with_namebase == None:
1667 namebase = name
1668 else:
1669 namebase = getTestOpts().with_namebase
1670
1671 (platform_specific, expected_prof_file) = \
1672 platform_wordsize_qualify(namebase, 'prof.sample')
1673
1674 # sample prof file is not required
1675 if not os.path.exists(expected_prof_file):
1676 return True
1677 else:
1678 return compare_outputs('prof', \
1679 two_normalisers(normalise_whitespace,normalise_prof), \
1680 expected_prof_file, prof_file)
1681
1682 # Compare expected output to actual output, and optionally accept the
1683 # new output. Returns true if output matched or was accepted, false
1684 # otherwise.
1685 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1686 if os.path.exists(expected_file):
1687 expected_raw = read_no_crs(expected_file)
1688 # print "norm:", normaliser(expected_raw)
1689 expected_str = normaliser(expected_raw)
1690 expected_file_for_diff = expected_file
1691 else:
1692 expected_str = ''
1693 expected_file_for_diff = '/dev/null'
1694
1695 actual_raw = read_no_crs(actual_file)
1696 actual_str = normaliser(actual_raw)
1697
1698 if expected_str == actual_str:
1699 return 1
1700 else:
1701 print 'Actual ' + kind + ' output differs from expected:'
1702
1703 if expected_file_for_diff == '/dev/null':
1704 expected_normalised_file = '/dev/null'
1705 else:
1706 expected_normalised_file = expected_file + ".normalised"
1707 write_file(expected_normalised_file, expected_str)
1708
1709 actual_normalised_file = actual_file + ".normalised"
1710 write_file(actual_normalised_file, actual_str)
1711
1712 # Ignore whitespace when diffing. We should only get to this
1713 # point if there are non-whitespace differences
1714 #
1715 # Note we are diffing the *actual* output, not the normalised
1716 # output. The normalised output may have whitespace squashed
1717 # (including newlines) so the diff would be hard to read.
1718 # This does mean that the diff might contain changes that
1719 # would be normalised away.
1720 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1721 ' ' + actual_file )
1722
1723 # If for some reason there were no non-whitespace differences,
1724 # then do a full diff
1725 if r == 0:
1726 r = os.system( 'diff -u ' + expected_file_for_diff + \
1727 ' ' + actual_file )
1728
1729 if config.accept:
1730 print 'Accepting new output.'
1731 write_file(expected_file, actual_raw)
1732 return 1
1733 else:
1734 return 0
1735
1736
1737 def normalise_whitespace( str ):
1738 # Merge contiguous whitespace characters into a single space.
1739 str = re.sub('[ \t\n]+', ' ', str)
1740 return str
1741
1742 def normalise_errmsg( str ):
1743 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1744 # the colon is there because it appears in error messages; this
1745 # hacky solution is used in place of more sophisticated filename
1746 # mangling
1747 str = re.sub('([^\\s])\\.exe', '\\1', str)
1748 # normalise slashes, minimise Windows/Unix filename differences
1749 str = re.sub('\\\\', '/', str)
1750 # The inplace ghc's are called ghc-stage[123] to avoid filename
1751 # collisions, so we need to normalise that to just "ghc"
1752 str = re.sub('ghc-stage[123]', 'ghc', str)
1753 # We sometimes see the name of the integer-gmp package on stderr,
1754 # but this can change (either the implementation name or the
1755 # version number), so we canonicalise it here
1756 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1757 return str
1758
1759 # normalise a .prof file, so that we can reasonably compare it against
1760 # a sample. This doesn't compare any of the actual profiling data,
1761 # only the shape of the profile and the number of entries.
1762 def normalise_prof (str):
1763 # strip everything up to the line beginning "COST CENTRE"
1764 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1765
1766 # strip results for CAFs, these tend to change unpredictably
1767 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1768
1769 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1770 # sometimes under MAIN.
1771 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1772
1773 # We have somthing like this:
1774
1775 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1776 # k Main 204 1 0.0 0.0 0.0 0.0
1777 # foo Main 205 1 0.0 0.0 0.0 0.0
1778 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1779
1780 # then we remove all the specific profiling data, leaving only the
1781 # cost centre name, module, and entries, to end up with this:
1782
1783 # MAIN MAIN 0
1784 # k Main 1
1785 # foo Main 1
1786 # foo.bar Main 1
1787
1788 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1789 return str
1790
1791 def normalise_slashes_( str ):
1792 str = re.sub('\\\\', '/', str)
1793 return str
1794
1795 def normalise_exe_( str ):
1796 str = re.sub('\.exe', '', str)
1797 return str
1798
1799 def normalise_output( str ):
1800 # Remove a .exe extension (for Windows)
1801 # This can occur in error messages generated by the program.
1802 str = re.sub('([^\\s])\\.exe', '\\1', str)
1803 return str
1804
1805 def normalise_asm( str ):
1806 lines = str.split('\n')
1807 # Only keep instructions and labels not starting with a dot.
1808 metadata = re.compile('^[ \t]*\\..*$')
1809 out = []
1810 for line in lines:
1811 # Drop metadata directives (e.g. ".type")
1812 if not metadata.match(line):
1813 line = re.sub('@plt', '', line)
1814 instr = line.lstrip().split()
1815 # Drop empty lines.
1816 if not instr:
1817 continue
1818 # Drop operands, except for call instructions.
1819 elif instr[0] == 'call':
1820 out.append(instr[0] + ' ' + instr[1])
1821 else:
1822 out.append(instr[0])
1823 out = '\n'.join(out)
1824 return out
1825
1826 def if_verbose( n, str ):
1827 if config.verbose >= n:
1828 print str
1829
1830 def if_verbose_dump( n, f ):
1831 if config.verbose >= n:
1832 try:
1833 print open(f).read()
1834 except:
1835 print ''
1836
1837 def rawSystem(cmd_and_args):
1838 # We prefer subprocess.call to os.spawnv as the latter
1839 # seems to send its arguments through a shell or something
1840 # with the Windows (non-cygwin) python. An argument "a b c"
1841 # turns into three arguments ["a", "b", "c"].
1842
1843 # However, subprocess is new in python 2.4, so fall back to
1844 # using spawnv if we don't have it
1845
1846 if have_subprocess:
1847 return subprocess.call(cmd_and_args)
1848 else:
1849 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1850
1851 # Note that this doesn't handle the timeout itself; it is just used for
1852 # commands that have timeout handling built-in.
1853 def rawSystemWithTimeout(cmd_and_args):
1854 r = rawSystem(cmd_and_args)
1855 if r == 98:
1856 # The python timeout program uses 98 to signal that ^C was pressed
1857 stopNow()
1858 return r
1859
1860 # cmd is a complex command in Bourne-shell syntax
1861 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1862 # Hence it must ultimately be run by a Bourne shell
1863 #
1864 # Mostly it invokes the command wrapped in 'timeout' thus
1865 # timeout 300 'cd . && ...blah blah'
1866 # so it's timeout's job to invoke the Bourne shell
1867 #
1868 # But watch out for the case when there is no timeout program!
1869 # Then, when using the native Python, os.system will invoke the cmd shell
1870
1871 def runCmd( cmd ):
1872 if_verbose( 1, cmd )
1873 r = 0
1874 if config.os == 'mingw32':
1875 # On MinGW, we will always have timeout
1876 assert config.timeout_prog!=''
1877
1878 if config.timeout_prog != '':
1879 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1880 else:
1881 r = os.system(cmd)
1882 return r << 8
1883
1884 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1885 if_verbose( 1, cmd )
1886 r = 0
1887 if config.os == 'mingw32':
1888 # On MinGW, we will always have timeout
1889 assert config.timeout_prog!=''
1890 timeout = int(ceil(config.timeout * timeout_multiplier))
1891
1892 if config.timeout_prog != '':
1893 if config.check_files_written:
1894 fn = name + ".strace"
1895 r = rawSystemWithTimeout(
1896 ["strace", "-o", fn, "-fF",
1897 "-e", "creat,open,chdir,clone,vfork",
1898 config.timeout_prog, str(timeout), cmd])
1899 addTestFilesWritten(name, fn)
1900 rm_no_fail(fn)
1901 else:
1902 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1903 else:
1904 r = os.system(cmd)
1905 return r << 8
1906
1907 def runCmdExitCode( cmd ):
1908 return (runCmd(cmd) >> 8);
1909
1910
1911 # -----------------------------------------------------------------------------
1912 # checking for files being written to by multiple tests
1913
1914 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1915 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1916 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1917 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1918 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1919 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1920 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1921 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1922 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1923 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1924 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1925 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1926 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1927 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1928 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1929 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1930 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1931
1932 # Files that are read or written but shouldn't be:
1933 # * ghci_history shouldn't be read or written by tests
1934 # * things under package.conf.d shouldn't be written by tests
1935 bad_file_usages = {}
1936
1937 # Mapping from tests to the list of files that they write
1938 files_written = {}
1939
1940 # Mapping from tests to the list of files that they write but don't clean
1941 files_written_not_removed = {}
1942
1943 def add_bad_file_usage(name, file):
1944 try:
1945 if not file in bad_file_usages[name]:
1946 bad_file_usages[name].append(file)
1947 except:
1948 bad_file_usages[name] = [file]
1949
1950 def mkPath(curdir, path):
1951 # Given the current full directory is 'curdir', what is the full
1952 # path to 'path'?
1953 return os.path.realpath(os.path.join(curdir, path))
1954
1955 def addTestFilesWritten(name, fn):
1956 if config.use_threads:
1957 with t.lockFilesWritten:
1958 addTestFilesWrittenHelper(name, fn)
1959 else:
1960 addTestFilesWrittenHelper(name, fn)
1961
1962 def addTestFilesWrittenHelper(name, fn):
1963 started = False
1964 working_directories = {}
1965
1966 with open(fn, 'r') as f:
1967 for line in f:
1968 m_pid = re_strace_pid.match(line)
1969 if m_pid:
1970 pid = m_pid.group(1)
1971 content = m_pid.group(2)
1972 elif re_strace_unavailable.match(line):
1973 next
1974 else:
1975 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1976
1977 m_open = re_strace_open.match(content)
1978 m_chdir = re_strace_chdir.match(content)
1979 m_clone = re_strace_clone.match(content)
1980 m_vfork = re_strace_vfork.match(content)
1981
1982 if not started:
1983 working_directories[pid] = os.getcwd()
1984 started = True
1985
1986 if m_open:
1987 file = m_open.group(1)
1988 file = mkPath(working_directories[pid], file)
1989 if file.endswith("ghci_history"):
1990 add_bad_file_usage(name, file)
1991 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1992 flags = m_open.group(2).split('|')
1993 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1994 if re.match('package\.conf\.d', file):
1995 add_bad_file_usage(name, file)
1996 else:
1997 try:
1998 if not file in files_written[name]:
1999 files_written[name].append(file)
2000 except:
2001 files_written[name] = [file]
2002 elif 'O_RDONLY' in flags:
2003 pass
2004 else:
2005 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2006 elif m_chdir:
2007 # We optimistically assume that unfinished chdir's are going to succeed
2008 dir = m_chdir.group(1)
2009 working_directories[pid] = mkPath(working_directories[pid], dir)
2010 elif m_clone:
2011 working_directories[m_clone.group(2)] = working_directories[pid]
2012 elif m_vfork:
2013 working_directories[m_vfork.group(2)] = working_directories[pid]
2014 elif re_strace_open_resumed.match(content):
2015 pass
2016 elif re_strace_chdir_resumed.match(content):
2017 pass
2018 elif re_strace_vfork_unfinished.match(content):
2019 pass
2020 elif re_strace_clone_unfinished.match(content):
2021 pass
2022 elif re_strace_ignore_sigchild.match(content):
2023 pass
2024 elif re_strace_ignore_sigvtalarm.match(content):
2025 pass
2026 elif re_strace_ignore_sigint.match(content):
2027 pass
2028 elif re_strace_ignore_sigfpe.match(content):
2029 pass
2030 elif re_strace_ignore_sigsegv.match(content):
2031 pass
2032 elif re_strace_ignore_sigpipe.match(content):
2033 pass
2034 else:
2035 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2036
2037 def checkForFilesWrittenProblems(file):
2038 foundProblem = False
2039
2040 files_written_inverted = {}
2041 for t in files_written.keys():
2042 for f in files_written[t]:
2043 try:
2044 files_written_inverted[f].append(t)
2045 except:
2046 files_written_inverted[f] = [t]
2047
2048 for f in files_written_inverted.keys():
2049 if len(files_written_inverted[f]) > 1:
2050 if not foundProblem:
2051 foundProblem = True
2052 file.write("\n")
2053 file.write("\nSome files are written by multiple tests:\n")
2054 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2055 if foundProblem:
2056 file.write("\n")
2057
2058 # -----
2059
2060 if len(files_written_not_removed) > 0:
2061 file.write("\n")
2062 file.write("\nSome files written but not removed:\n")
2063 tests = files_written_not_removed.keys()
2064 tests.sort()
2065 for t in tests:
2066 for f in files_written_not_removed[t]:
2067 file.write(" " + t + ": " + f + "\n")
2068 file.write("\n")
2069
2070 # -----
2071
2072 if len(bad_file_usages) > 0:
2073 file.write("\n")
2074 file.write("\nSome bad file usages:\n")
2075 tests = bad_file_usages.keys()
2076 tests.sort()
2077 for t in tests:
2078 for f in bad_file_usages[t]:
2079 file.write(" " + t + ": " + f + "\n")
2080 file.write("\n")
2081
2082 # -----------------------------------------------------------------------------
2083 # checking if ghostscript is available for checking the output of hp2ps
2084
2085 def genGSCmd(psfile):
2086 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2087
2088 def gsNotWorking():
2089 global gs_working
2090 print "GhostScript not available for hp2ps tests"
2091
2092 global gs_working
2093 gs_working = 0
2094 if config.have_profiling:
2095 if config.gs != '':
2096 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2097 if resultGood == 0:
2098 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2099 if resultBad != 0:
2100 print "GhostScript available for hp2ps tests"
2101 gs_working = 1;
2102 else:
2103 gsNotWorking();
2104 else:
2105 gsNotWorking();
2106 else:
2107 gsNotWorking();
2108
2109 def rm_no_fail( file ):
2110 try:
2111 os.remove( file )
2112 finally:
2113 return
2114
2115 def add_suffix( name, suffix ):
2116 if suffix == '':
2117 return name
2118 else:
2119 return name + '.' + suffix
2120
2121 def add_hs_lhs_suffix(name):
2122 if getTestOpts().c_src:
2123 return add_suffix(name, 'c')
2124 elif getTestOpts().cmm_src:
2125 return add_suffix(name, 'cmm')
2126 elif getTestOpts().objc_src:
2127 return add_suffix(name, 'm')
2128 elif getTestOpts().objcpp_src:
2129 return add_suffix(name, 'mm')
2130 elif getTestOpts().literate:
2131 return add_suffix(name, 'lhs')
2132 else:
2133 return add_suffix(name, 'hs')
2134
2135 def replace_suffix( name, suffix ):
2136 base, suf = os.path.splitext(name)
2137 return base + '.' + suffix
2138
2139 def in_testdir( name ):
2140 return (getTestOpts().testdir + '/' + name)
2141
2142 def qualify( name, suff ):
2143 return in_testdir(add_suffix(name, suff))
2144
2145
2146 # Finding the sample output. The filename is of the form
2147 #
2148 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2149 #
2150 # and we pick the most specific version available. The <version> is
2151 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2152 # more fine-grained control use if_compiler_lt().
2153 #
2154 def platform_wordsize_qualify( name, suff ):
2155
2156 basepath = qualify(name, suff)
2157
2158 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2159 for (platformSpecific, plat) in [(1, '-' + config.platform),
2160 (1, '-' + config.os),
2161 (0, '')]
2162 for ws in ['-ws-' + config.wordsize, '']
2163 for comp in ['-' + config.compiler_type, '']
2164 for vers in ['-' + config.compiler_maj_version, '']]
2165
2166 dir = glob.glob(basepath + '*')
2167 dir = map (lambda d: normalise_slashes_(d), dir)
2168
2169 for (platformSpecific, f) in paths:
2170 if f in dir:
2171 return (platformSpecific,f)
2172
2173 return (0, basepath)
2174
2175 # Clean up prior to the test, so that we can't spuriously conclude
2176 # that it passed on the basis of old run outputs.
2177 def pretest_cleanup(name):
2178 if getTestOpts().outputdir != None:
2179 odir = in_testdir(getTestOpts().outputdir)
2180 try:
2181 shutil.rmtree(odir)
2182 except:
2183 pass
2184 os.mkdir(odir)
2185
2186 rm_no_fail(qualify(name,'comp.stderr'))
2187 rm_no_fail(qualify(name,'run.stderr'))
2188 rm_no_fail(qualify(name,'run.stdout'))
2189 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2190 # simple_build zaps the following:
2191 # rm_nofail(qualify("o"))
2192 # rm_nofail(qualify(""))
2193 # not interested in the return code
2194
2195 # -----------------------------------------------------------------------------
2196 # Return a list of all the files ending in '.T' below the directory dir.
2197
2198 def findTFiles(roots):
2199 return concat(map(findTFiles_,roots))
2200
2201 def findTFiles_(path):
2202 if os.path.isdir(path):
2203 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2204 return findTFiles(paths)
2205 elif path[-2:] == '.T':
2206 return [path]
2207 else:
2208 return []
2209
2210 # -----------------------------------------------------------------------------
2211 # Output a test summary to the specified file object
2212
2213 def summary(t, file):
2214
2215 file.write('\n')
2216 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2217 file.write('OVERALL SUMMARY for test run started at '
2218 + t.start_time + '\n'
2219 + string.rjust(`t.total_tests`, 8)
2220 + ' total tests, which gave rise to\n'
2221 + string.rjust(`t.total_test_cases`, 8)
2222 + ' test cases, of which\n'
2223 + string.rjust(`t.n_tests_skipped`, 8)
2224 + ' were skipped\n'
2225 + '\n'
2226 + string.rjust(`t.n_missing_libs`, 8)
2227 + ' had missing libraries\n'
2228 + string.rjust(`t.n_expected_passes`, 8)
2229 + ' expected passes\n'
2230 + string.rjust(`t.n_expected_failures`, 8)
2231 + ' expected failures\n'
2232 + '\n'
2233 + string.rjust(`t.n_framework_failures`, 8)
2234 + ' caused framework failures\n'
2235 + string.rjust(`t.n_unexpected_passes`, 8)
2236 + ' unexpected passes\n'
2237 + string.rjust(`t.n_unexpected_failures`, 8)
2238 + ' unexpected failures\n'
2239 + '\n')
2240
2241 if t.n_unexpected_passes > 0:
2242 file.write('Unexpected passes:\n')
2243 printPassingTestInfosSummary(file, t.unexpected_passes)
2244
2245 if t.n_unexpected_failures > 0:
2246 file.write('Unexpected failures:\n')
2247 printFailingTestInfosSummary(file, t.unexpected_failures)
2248
2249 if config.check_files_written:
2250 checkForFilesWrittenProblems(file)
2251
2252 if stopping():
2253 file.write('WARNING: Testsuite run was terminated early\n')
2254
2255 def printUnexpectedTests(file, testInfoss):
2256 unexpected = []
2257 for testInfos in testInfoss:
2258 directories = testInfos.keys()
2259 for directory in directories:
2260 tests = testInfos[directory].keys()
2261 unexpected += tests
2262 if unexpected != []:
2263 file.write('Unexpected results from:\n')
2264 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2265 file.write('\n')
2266
2267 def printPassingTestInfosSummary(file, testInfos):
2268 directories = testInfos.keys()
2269 directories.sort()
2270 maxDirLen = max(map ((lambda x : len(x)), directories))
2271 for directory in directories:
2272 tests = testInfos[directory].keys()
2273 tests.sort()
2274 for test in tests:
2275 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2276 ' (' + join(testInfos[directory][test],',') + ')\n')
2277 file.write('\n')
2278
2279 def printFailingTestInfosSummary(file, testInfos):
2280 directories = testInfos.keys()
2281 directories.sort()
2282 maxDirLen = max(map ((lambda x : len(x)), directories))
2283 for directory in directories:
2284 tests = testInfos[directory].keys()
2285 tests.sort()
2286 for test in tests:
2287 reasons = testInfos[directory][test].keys()
2288 for reason in reasons:
2289 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2290 ' [' + reason + ']' + \
2291 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2292 file.write('\n')
2293
2294 def getStdout(cmd):
2295 if have_subprocess:
2296 p = subprocess.Popen(cmd,
2297 stdout=subprocess.PIPE,
2298 stderr=subprocess.PIPE)
2299 (stdout, stderr) = p.communicate()
2300 r = p.wait()
2301 if r != 0:
2302 raise Exception("Command failed: " + str(cmd))
2303 if stderr != '':
2304 raise Exception("stderr from command: " + str(cmd))
2305 return stdout
2306 else:
2307 raise Exception("Need subprocess to get stdout, but don't have it")