Add 'make list_brokens'
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 global brokens
138 brokens.append((bug, name))
139 opts.expect = 'fail';
140
141 def ignore_output( name, opts ):
142 opts.ignore_output = 1
143
144 def no_stdin( name, opts ):
145 opts.no_stdin = 1
146
147 def combined_output( name, opts ):
148 opts.combined_output = True
149
150 # -----
151
152 def expect_fail_for( ways ):
153 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
154
155 def _expect_fail_for( name, opts, ways ):
156 opts.expect_fail_for = ways
157
158 def expect_broken_for( bug, ways ):
159 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
160
161 def _expect_broken_for( name, opts, bug, ways ):
162 opts.expect_fail_for = ways
163
164 # -----
165
166 def omit_ways( ways ):
167 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
168
169 def _omit_ways( name, opts, ways ):
170 opts.omit_ways = ways
171
172 # -----
173
174 def only_ways( ways ):
175 return lambda name, opts, w=ways: _only_ways( name, opts, w )
176
177 def _only_ways( name, opts, ways ):
178 opts.only_ways = ways
179
180 # -----
181
182 def extra_ways( ways ):
183 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
184
185 def _extra_ways( name, opts, ways ):
186 opts.extra_ways = ways
187
188 # -----
189
190 def omit_compiler_types( compiler_types ):
191 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
192
193 def _omit_compiler_types( name, opts, compiler_types ):
194 if config.compiler_type in compiler_types:
195 opts.skip = 1
196
197 # -----
198
199 def only_compiler_types( compiler_types ):
200 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
201
202 def _only_compiler_types( name, opts, compiler_types ):
203 if config.compiler_type not in compiler_types:
204 opts.skip = 1
205
206 # -----
207
208 def set_stdin( file ):
209 return lambda name, opts, f=file: _set_stdin(name, opts, f);
210
211 def _set_stdin( name, opts, f ):
212 opts.stdin = f
213
214 # -----
215
216 def exit_code( val ):
217 return lambda name, opts, v=val: _exit_code(name, opts, v);
218
219 def _exit_code( name, opts, v ):
220 opts.exit_code = v
221
222 # -----
223
224 def timeout_multiplier( val ):
225 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
226
227 def _timeout_multiplier( name, opts, v ):
228 opts.timeout_multiplier = v
229
230 # -----
231
232 def extra_run_opts( val ):
233 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
234
235 def _extra_run_opts( name, opts, v ):
236 opts.extra_run_opts = v
237
238 # -----
239
240 def extra_hc_opts( val ):
241 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
242
243 def _extra_hc_opts( name, opts, v ):
244 opts.extra_hc_opts = v
245
246 # -----
247
248 def extra_clean( files ):
249 return lambda name, opts, v=files: _extra_clean(name, opts, v);
250
251 def _extra_clean( name, opts, v ):
252 opts.clean_files = v
253
254 # -----
255
256 def stats_num_field( field, expecteds ):
257 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
258
259 def _stats_num_field( name, opts, field, expecteds ):
260 if field in opts.stats_range_fields:
261 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
262
263 if type(expecteds) is types.ListType:
264 for (b, expected, dev) in expecteds:
265 if b:
266 opts.stats_range_fields[field] = (expected, dev)
267 return
268 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
269
270 else:
271 (expected, dev) = expecteds
272 opts.stats_range_fields[field] = (expected, dev)
273
274 def compiler_stats_num_field( field, expecteds ):
275 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
276
277 def _compiler_stats_num_field( name, opts, field, expecteds ):
278 if field in opts.compiler_stats_range_fields:
279 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
280
281 for (b, expected, dev) in expecteds:
282 if b:
283 opts.compiler_stats_range_fields[field] = (expected, dev)
284 return
285
286 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
287
288 # -----
289
290 def skip_if_no_ghci(name, opts):
291 if not ('ghci' in config.run_ways):
292 opts.skip = 1
293
294 # ----
295
296 def skip_if_fast(name, opts):
297 if config.fast:
298 opts.skip = 1
299
300 # -----
301
302 def when(b, f):
303 # When list_brokens is on, we want to see all expect_broken calls,
304 # so we always do f
305 if b or config.list_broken:
306 return f
307 else:
308 return normal
309
310 def unless(b, f):
311 return when(not b, f)
312
313 def platform( plat ):
314 return config.platform == plat
315
316 def if_os( os, f ):
317 if config.os == os:
318 return f
319 else:
320 return normal
321
322 def unless_os( os, f ):
323 if config.os == os:
324 return normal
325 else:
326 return f
327
328 def if_arch( arch, f ):
329 if config.arch == arch:
330 return f
331 else:
332 return normal
333
334 def unless_arch( arch, f ):
335 if config.arch == arch:
336 return normal
337 else:
338 return f
339
340 def wordsize( ws ):
341 return config.wordsize == str(ws)
342
343 def if_unregisterised( f ):
344 if config.unregisterised:
345 return f
346 else:
347 return normal
348
349 def unless_unregisterised( f ):
350 if config.unregisterised:
351 return normal
352 else:
353 return f
354
355 def if_msys( f ):
356 if config.msys:
357 return f
358 else:
359 return normal
360
361 def if_cygwin( f ):
362 if config.cygwin:
363 return f
364 else:
365 return normal
366
367 def when_have_vanilla( f ):
368 if config.have_vanilla:
369 return f
370 else:
371 return normal
372
373 def unless_have_vanilla( f ):
374 if config.have_vanilla:
375 return normal
376 else:
377 return f
378
379 def when_have_dynamic( f ):
380 if config.have_dynamic:
381 return f
382 else:
383 return normal
384
385 def unless_have_dynamic( f ):
386 if config.have_dynamic:
387 return normal
388 else:
389 return f
390
391 def when_have_profiling( f ):
392 if config.have_profiling:
393 return f
394 else:
395 return normal
396
397 def unless_have_profiling( f ):
398 if config.have_profiling:
399 return normal
400 else:
401 return f
402
403 # ---
404
405 def if_ghci_dynamic( f ):
406 if config.ghc_dynamic_by_default:
407 return f
408 else:
409 return normal
410
411 def if_in_tree_compiler( f ):
412 if config.in_tree_compiler:
413 return f
414 else:
415 return normal
416
417 def unless_in_tree_compiler( f ):
418 if config.in_tree_compiler:
419 return normal
420 else:
421 return f
422
423 def if_compiler_type( compiler, f ):
424 if config.compiler_type == compiler:
425 return f
426 else:
427 return normal
428
429 def if_compiler_profiled( f ):
430 if config.compiler_profiled:
431 return f
432 else:
433 return normal
434
435 def unless_compiler_profiled( f ):
436 if config.compiler_profiled:
437 return normal
438 else:
439 return f
440
441 def if_compiler_lt( compiler, version, f ):
442 if config.compiler_type == compiler and \
443 version_lt(config.compiler_version, version):
444 return f
445 else:
446 return normal
447
448 def if_compiler_le( compiler, version, f ):
449 if config.compiler_type == compiler and \
450 version_le(config.compiler_version, version):
451 return f
452 else:
453 return normal
454
455 def if_compiler_gt( compiler, version, f ):
456 if config.compiler_type == compiler and \
457 version_gt(config.compiler_version, version):
458 return f
459 else:
460 return normal
461
462 def if_compiler_ge( compiler, version, f ):
463 if config.compiler_type == compiler and \
464 version_ge(config.compiler_version, version):
465 return f
466 else:
467 return normal
468
469 def if_compiler_debugged( f ):
470 if config.compiler_debugged:
471 return f
472 else:
473 return normal
474
475 def namebase( nb ):
476 return lambda opts, nb=nb: _namebase(opts, nb)
477
478 def _namebase( opts, nb ):
479 opts.with_namebase = nb
480
481 # ---
482
483 def if_tag( tag, f ):
484 if tag in config.compiler_tags:
485 return f
486 else:
487 return normal
488
489 def unless_tag( tag, f ):
490 if not (tag in config.compiler_tags):
491 return f
492 else:
493 return normal
494
495 # ---
496 def high_memory_usage(name, opts):
497 opts.alone = True
498
499 # ---
500 def literate( name, opts ):
501 opts.literate = 1;
502
503 def c_src( name, opts ):
504 opts.c_src = 1;
505
506 def objc_src( name, opts ):
507 opts.objc_src = 1;
508
509 def objcpp_src( name, opts ):
510 opts.objcpp_src = 1;
511
512 def cmm_src( name, opts ):
513 opts.cmm_src = 1;
514
515 def outputdir( odir ):
516 return lambda name, opts, d=odir: _outputdir(name, opts, d)
517
518 def _outputdir( name, opts, odir ):
519 opts.outputdir = odir;
520
521 # ----
522
523 def pre_cmd( cmd ):
524 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
525
526 def _pre_cmd( name, opts, cmd ):
527 opts.pre_cmd = cmd
528
529 # ----
530
531 def clean_cmd( cmd ):
532 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
533
534 def _clean_cmd( name, opts, cmd ):
535 opts.clean_cmd = cmd
536
537 # ----
538
539 def cmd_prefix( prefix ):
540 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
541
542 def _cmd_prefix( name, opts, prefix ):
543 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
544
545 # ----
546
547 def cmd_wrapper( fun ):
548 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
549
550 def _cmd_wrapper( name, opts, fun ):
551 opts.cmd_wrapper = fun
552
553 # ----
554
555 def compile_cmd_prefix( prefix ):
556 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
557
558 def _compile_cmd_prefix( name, opts, prefix ):
559 opts.compile_cmd_prefix = prefix
560
561 # ----
562
563 def normalise_slashes( name, opts ):
564 opts.extra_normaliser = normalise_slashes_
565
566 def normalise_exe( name, opts ):
567 opts.extra_normaliser = normalise_exe_
568
569 def normalise_fun( fun ):
570 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
571
572 def _normalise_fun( name, opts, f ):
573 opts.extra_normaliser = f
574
575 def normalise_errmsg_fun( fun ):
576 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
577
578 def _normalise_errmsg_fun( name, opts, f ):
579 opts.extra_errmsg_normaliser = f
580
581 def two_normalisers(f, g):
582 return lambda x, f=f, g=g: f(g(x))
583
584 # ----
585 # Function for composing two opt-fns together
586
587 def composes( fs ):
588 return reduce(lambda f, g: compose(f, g), fs)
589
590 def compose( f, g ):
591 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
592
593 def _compose( name, opts, f, g ):
594 f(name, opts)
595 g(name, opts)
596
597 # -----------------------------------------------------------------------------
598 # The current directory of tests
599
600 def newTestDir( dir ):
601 global thisdir_settings
602 # reset the options for this test directory
603 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
604
605 def _newTestDir( name, opts, dir ):
606 opts.testdir = dir
607 opts.compiler_always_flags = config.compiler_always_flags
608
609 # -----------------------------------------------------------------------------
610 # Actually doing tests
611
612 parallelTests = []
613 aloneTests = []
614 allTestNames = set([])
615
616 def runTest (opts, name, func, args):
617 ok = 0
618
619 if config.use_threads:
620 t.thread_pool.acquire()
621 try:
622 while config.threads<(t.running_threads+1):
623 t.thread_pool.wait()
624 t.running_threads = t.running_threads+1
625 ok=1
626 t.thread_pool.release()
627 thread.start_new_thread(test_common_thread, (name, opts, func, args))
628 except:
629 if not ok:
630 t.thread_pool.release()
631 else:
632 test_common_work (name, opts, func, args)
633
634 # name :: String
635 # setup :: TestOpts -> IO ()
636 def test (name, setup, func, args):
637 global aloneTests
638 global parallelTests
639 global allTestNames
640 global thisdir_settings
641 if name in allTestNames:
642 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
643 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
644 framework_fail(name, 'bad_name', 'This test has an invalid name')
645
646 # Make a deep copy of the default_testopts, as we need our own copy
647 # of any dictionaries etc inside it. Otherwise, if one test modifies
648 # them, all tests will see the modified version!
649 myTestOpts = copy.deepcopy(default_testopts)
650
651 if type(setup) is types.ListType:
652 setup = composes(setup)
653
654 setup = compose(thisdir_settings, setup)
655 setup(name, myTestOpts)
656
657 thisTest = lambda : runTest(myTestOpts, name, func, args)
658 if myTestOpts.alone:
659 aloneTests.append(thisTest)
660 else:
661 parallelTests.append(thisTest)
662 allTestNames.add(name)
663
664 if config.use_threads:
665 def test_common_thread(name, opts, func, args):
666 t.lock.acquire()
667 try:
668 test_common_work(name,opts,func,args)
669 finally:
670 t.lock.release()
671 t.thread_pool.acquire()
672 t.running_threads = t.running_threads - 1
673 t.thread_pool.notify()
674 t.thread_pool.release()
675
676 def get_package_cache_timestamp():
677 if config.package_conf_cache_file == '':
678 return 0.0
679 else:
680 try:
681 return os.stat(config.package_conf_cache_file).st_mtime
682 except:
683 return 0.0
684
685
686 def test_common_work (name, opts, func, args):
687 try:
688 t.total_tests = t.total_tests+1
689 setLocalTestOpts(opts)
690
691 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
692
693 # All the ways we might run this test
694 if func == compile or func == multimod_compile:
695 all_ways = config.compile_ways
696 elif func == compile_and_run or func == multimod_compile_and_run:
697 all_ways = config.run_ways
698 elif func == ghci_script:
699 if 'ghci' in config.run_ways:
700 all_ways = ['ghci']
701 else:
702 all_ways = []
703 else:
704 all_ways = ['normal']
705
706 # A test itself can request extra ways by setting opts.extra_ways
707 all_ways = all_ways + filter(lambda way: way not in all_ways,
708 opts.extra_ways)
709
710 t.total_test_cases = t.total_test_cases + len(all_ways)
711
712 ok_way = lambda way: \
713 not getTestOpts().skip \
714 and (config.only == [] or name in config.only) \
715 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
716 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
717 and way not in getTestOpts().omit_ways
718
719 # Which ways we are asked to skip
720 do_ways = filter (ok_way,all_ways)
721
722 # In fast mode, we skip all but one way
723 if config.fast and len(do_ways) > 0:
724 do_ways = [do_ways[0]]
725
726 if not config.clean_only:
727 # Run the required tests...
728 for way in do_ways:
729 if stopping():
730 break
731 do_test (name, way, func, args)
732
733 for way in all_ways:
734 if way not in do_ways:
735 skiptest (name,way)
736
737 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
738 clean(map (lambda suff: name + suff,
739 ['', '.exe', '.exe.manifest', '.genscript',
740 '.stderr.normalised', '.stdout.normalised',
741 '.run.stderr', '.run.stdout',
742 '.run.stderr.normalised', '.run.stdout.normalised',
743 '.comp.stderr', '.comp.stdout',
744 '.comp.stderr.normalised', '.comp.stdout.normalised',
745 '.interp.stderr', '.interp.stdout',
746 '.interp.stderr.normalised', '.interp.stdout.normalised',
747 '.stats', '.comp.stats',
748 '.hi', '.o', '.prof', '.exe.prof', '.hc',
749 '_stub.h', '_stub.c', '_stub.o',
750 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
751
752 if func == multi_compile or func == multi_compile_fail:
753 extra_mods = args[1]
754 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
755 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
756
757 clean(getTestOpts().clean_files)
758
759 if getTestOpts().outputdir != None:
760 odir = in_testdir(getTestOpts().outputdir)
761 try:
762 shutil.rmtree(odir)
763 except:
764 pass
765
766 try:
767 shutil.rmtree(in_testdir('.hpc.' + name))
768 except:
769 pass
770
771 try:
772 cleanCmd = getTestOpts().clean_cmd
773 if cleanCmd != None:
774 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
775 if result != 0:
776 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
777 except e:
778 framework_fail(name, 'cleaning', 'clean-command exception')
779
780 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
781
782 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
783 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
784
785 try:
786 for f in files_written[name]:
787 if os.path.exists(f):
788 try:
789 if not f in files_written_not_removed[name]:
790 files_written_not_removed[name].append(f)
791 except:
792 files_written_not_removed[name] = [f]
793 except:
794 pass
795 except Exception, e:
796 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
797
798 def clean(strs):
799 for str in strs:
800 for name in glob.glob(in_testdir(str)):
801 clean_full_path(name)
802
803 def clean_full_path(name):
804 try:
805 # Remove files...
806 os.remove(name)
807 except OSError, e1:
808 try:
809 # ... and empty directories
810 os.rmdir(name)
811 except OSError, e2:
812 # We don't want to fail here, but we do want to know
813 # what went wrong, so print out the exceptions.
814 # ENOENT isn't a problem, though, as we clean files
815 # that don't necessarily exist.
816 if e1.errno != errno.ENOENT:
817 print e1
818 if e2.errno != errno.ENOENT:
819 print e2
820
821 def do_test(name, way, func, args):
822 full_name = name + '(' + way + ')'
823
824 try:
825 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
826 str([t.n_unexpected_passes, \
827 t.n_unexpected_failures, \
828 t.n_framework_failures])
829
830 if config.use_threads:
831 t.lock.release()
832
833 try:
834 preCmd = getTestOpts().pre_cmd
835 if preCmd != None:
836 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
837 if result != 0:
838 framework_fail(name, way, 'pre-command failed: ' + str(result))
839 except e:
840 framework_fail(name, way, 'pre-command exception')
841
842 try:
843 result = apply(func, [name,way] + args)
844 finally:
845 if config.use_threads:
846 t.lock.acquire()
847
848 if getTestOpts().expect != 'pass' and \
849 getTestOpts().expect != 'fail' and \
850 getTestOpts().expect != 'missing-lib':
851 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
852
853 try:
854 passFail = result['passFail']
855 except:
856 passFail = 'No passFail found'
857
858 if passFail == 'pass':
859 if getTestOpts().expect == 'pass' \
860 and way not in getTestOpts().expect_fail_for:
861 t.n_expected_passes = t.n_expected_passes + 1
862 if name in t.expected_passes:
863 t.expected_passes[name].append(way)
864 else:
865 t.expected_passes[name] = [way]
866 else:
867 print '*** unexpected pass for', full_name
868 t.n_unexpected_passes = t.n_unexpected_passes + 1
869 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
870 elif passFail == 'fail':
871 if getTestOpts().expect == 'pass' \
872 and way not in getTestOpts().expect_fail_for:
873 print '*** unexpected failure for', full_name
874 t.n_unexpected_failures = t.n_unexpected_failures + 1
875 reason = result['reason']
876 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
877 else:
878 if getTestOpts().expect == 'missing-lib':
879 t.n_missing_libs = t.n_missing_libs + 1
880 if name in t.missing_libs:
881 t.missing_libs[name].append(way)
882 else:
883 t.missing_libs[name] = [way]
884 else:
885 t.n_expected_failures = t.n_expected_failures + 1
886 if name in t.expected_failures:
887 t.expected_failures[name].append(way)
888 else:
889 t.expected_failures[name] = [way]
890 else:
891 framework_fail(name, way, 'bad result ' + passFail)
892 except KeyboardInterrupt:
893 stopNow()
894 except:
895 framework_fail(name, way, 'do_test exception')
896 traceback.print_exc()
897
898 def addPassingTestInfo (testInfos, directory, name, way):
899 directory = re.sub('^\\.[/\\\\]', '', directory)
900
901 if not directory in testInfos:
902 testInfos[directory] = {}
903
904 if not name in testInfos[directory]:
905 testInfos[directory][name] = []
906
907 testInfos[directory][name].append(way)
908
909 def addFailingTestInfo (testInfos, directory, name, reason, way):
910 directory = re.sub('^\\.[/\\\\]', '', directory)
911
912 if not directory in testInfos:
913 testInfos[directory] = {}
914
915 if not name in testInfos[directory]:
916 testInfos[directory][name] = {}
917
918 if not reason in testInfos[directory][name]:
919 testInfos[directory][name][reason] = []
920
921 testInfos[directory][name][reason].append(way)
922
923 def skiptest (name, way):
924 # print 'Skipping test \"', name, '\"'
925 t.n_tests_skipped = t.n_tests_skipped + 1
926 if name in t.tests_skipped:
927 t.tests_skipped[name].append(way)
928 else:
929 t.tests_skipped[name] = [way]
930
931 def framework_fail( name, way, reason ):
932 full_name = name + '(' + way + ')'
933 print '*** framework failure for', full_name, reason
934 t.n_framework_failures = t.n_framework_failures + 1
935 if name in t.framework_failures:
936 t.framework_failures[name].append(way)
937 else:
938 t.framework_failures[name] = [way]
939
940 def badResult(result):
941 try:
942 if result['passFail'] == 'pass':
943 return False
944 return True
945 except:
946 return True
947
948 def passed():
949 return {'passFail': 'pass'}
950
951 def failBecause(reason):
952 return {'passFail': 'fail', 'reason': reason}
953
954 # -----------------------------------------------------------------------------
955 # Generic command tests
956
957 # A generic command test is expected to run and exit successfully.
958 #
959 # The expected exit code can be changed via exit_code() as normal, and
960 # the expected stdout/stderr are stored in <testname>.stdout and
961 # <testname>.stderr. The output of the command can be ignored
962 # altogether by using run_command_ignore_output instead of
963 # run_command.
964
965 def run_command( name, way, cmd ):
966 return simple_run( name, '', cmd, '' )
967
968 # -----------------------------------------------------------------------------
969 # GHCi tests
970
971 def ghci_script( name, way, script ):
972 # filter out -fforce-recomp from compiler_always_flags, because we're
973 # actually testing the recompilation behaviour in the GHCi tests.
974 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
975 flags.append(getTestOpts().extra_hc_opts)
976 if getTestOpts().outputdir != None:
977 flags.extend(["-outputdir", getTestOpts().outputdir])
978
979 # We pass HC and HC_OPTS as environment variables, so that the
980 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
981 cmd = "HC='" + config.compiler + "' " + \
982 "HC_OPTS='" + join(flags,' ') + "' " + \
983 "'" + config.compiler + "'" + \
984 ' --interactive -v0 -ignore-dot-ghci ' + \
985 join(flags,' ')
986
987 getTestOpts().stdin = script
988 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
989
990 # -----------------------------------------------------------------------------
991 # Compile-only tests
992
993 def compile( name, way, extra_hc_opts ):
994 return do_compile( name, way, 0, '', [], extra_hc_opts )
995
996 def compile_fail( name, way, extra_hc_opts ):
997 return do_compile( name, way, 1, '', [], extra_hc_opts )
998
999 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1000 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1001
1002 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1003 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1004
1005 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1006 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1007
1008 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1009 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1010
1011 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1012 # print 'Compile only, extra args = ', extra_hc_opts
1013 pretest_cleanup(name)
1014
1015 result = extras_build( way, extra_mods, extra_hc_opts )
1016 if badResult(result):
1017 return result
1018 extra_hc_opts = result['hc_opts']
1019
1020 force = 0
1021 if extra_mods:
1022 force = 1
1023 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1024
1025 if badResult(result):
1026 return result
1027
1028 # the actual stderr should always match the expected, regardless
1029 # of whether we expected the compilation to fail or not (successful
1030 # compilations may generate warnings).
1031
1032 if getTestOpts().with_namebase == None:
1033 namebase = name
1034 else:
1035 namebase = getTestOpts().with_namebase
1036
1037 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1038 actual_stderr_file = qualify(name, 'comp.stderr')
1039
1040 if not compare_outputs('stderr', \
1041 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1042 expected_stderr_file, actual_stderr_file):
1043 return failBecause('stderr mismatch')
1044
1045 # no problems found, this test passed
1046 return passed()
1047
1048 def compile_cmp_asm( name, way, extra_hc_opts ):
1049 print 'Compile only, extra args = ', extra_hc_opts
1050 pretest_cleanup(name)
1051 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1052
1053 if badResult(result):
1054 return result
1055
1056 # the actual stderr should always match the expected, regardless
1057 # of whether we expected the compilation to fail or not (successful
1058 # compilations may generate warnings).
1059
1060 if getTestOpts().with_namebase == None:
1061 namebase = name
1062 else:
1063 namebase = getTestOpts().with_namebase
1064
1065 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1066 actual_asm_file = qualify(name, 's')
1067
1068 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1069 expected_asm_file, actual_asm_file):
1070 return failBecause('asm mismatch')
1071
1072 # no problems found, this test passed
1073 return passed()
1074
1075 # -----------------------------------------------------------------------------
1076 # Compile-and-run tests
1077
1078 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1079 # print 'Compile and run, extra args = ', extra_hc_opts
1080 pretest_cleanup(name)
1081
1082 result = extras_build( way, extra_mods, extra_hc_opts )
1083 if badResult(result):
1084 return result
1085 extra_hc_opts = result['hc_opts']
1086
1087 if way == 'ghci': # interpreted...
1088 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1089 elif way == 'extcore' or way == 'optextcore' :
1090 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1091 else: # compiled...
1092 force = 0
1093 if extra_mods:
1094 force = 1
1095
1096 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1097 if badResult(result):
1098 return result
1099
1100 cmd = './' + name;
1101
1102 # we don't check the compiler's stderr for a compile-and-run test
1103 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1104
1105 def compile_and_run( name, way, extra_hc_opts ):
1106 return compile_and_run__( name, way, '', [], extra_hc_opts)
1107
1108 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1109 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1110
1111 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1112 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1113
1114 def stats( name, way, stats_file ):
1115 opts = getTestOpts()
1116 return checkStats(stats_file, opts.stats_range_fields)
1117
1118 # -----------------------------------------------------------------------------
1119 # Check -t stats info
1120
1121 def checkStats(stats_file, range_fields):
1122 result = passed()
1123 if len(range_fields) > 0:
1124 f = open(in_testdir(stats_file))
1125 contents = f.read()
1126 f.close()
1127
1128 for (field, (expected, dev)) in range_fields.items():
1129 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1130 if m == None:
1131 print 'Failed to find field: ', field
1132 result = failBecause('no such stats field')
1133 val = int(m.group(1))
1134
1135 lowerBound = trunc( expected * ((100 - float(dev))/100));
1136 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1137
1138 if val < lowerBound:
1139 print field, 'value is too low:'
1140 print '(If this is because you have improved GHC, please'
1141 print 'update the test so that GHC doesn\'t regress again)'
1142 result = failBecause('stat too good')
1143 if val > upperBound:
1144 print field, 'value is too high:'
1145 result = failBecause('stat not good enough')
1146
1147 if val < lowerBound or val > upperBound:
1148 valStr = str(val)
1149 valLen = len(valStr)
1150 expectedStr = str(expected)
1151 expectedLen = len(expectedStr)
1152 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1153 def display(descr, val, extra):
1154 print descr, string.rjust(str(val), length), extra
1155 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1156 display(' Lower bound ' + field + ':', lowerBound, '')
1157 display(' Upper bound ' + field + ':', upperBound, '')
1158 display(' Actual ' + field + ':', val, '')
1159
1160 return result
1161
1162 # -----------------------------------------------------------------------------
1163 # Build a single-module program
1164
1165 def extras_build( way, extra_mods, extra_hc_opts ):
1166 for modopts in extra_mods:
1167 mod, opts = modopts
1168 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1169 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1170 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1171 if badResult(result):
1172 return result
1173
1174 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1175
1176
1177 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1178 opts = getTestOpts()
1179 errname = add_suffix(name, 'comp.stderr')
1180 rm_no_fail( qualify(errname, '') )
1181
1182 if top_mod != '':
1183 srcname = top_mod
1184 rm_no_fail( qualify(name, '') )
1185 base, suf = os.path.splitext(top_mod)
1186 rm_no_fail( qualify(base, '') )
1187 rm_no_fail( qualify(base, 'exe') )
1188 elif addsuf:
1189 srcname = add_hs_lhs_suffix(name)
1190 rm_no_fail( qualify(name, '') )
1191 else:
1192 srcname = name
1193 rm_no_fail( qualify(name, 'o') )
1194
1195 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1196
1197 to_do = ''
1198 if top_mod != '':
1199 to_do = '--make '
1200 if link:
1201 to_do = to_do + '-o ' + name
1202 elif link:
1203 to_do = '-o ' + name
1204 elif opts.compile_to_hc:
1205 to_do = '-C'
1206 else:
1207 to_do = '-c' # just compile
1208
1209 stats_file = name + '.comp.stats'
1210 if len(opts.compiler_stats_range_fields) > 0:
1211 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1212
1213 # Required by GHC 7.3+, harmless for earlier versions:
1214 if (getTestOpts().c_src or
1215 getTestOpts().objc_src or
1216 getTestOpts().objcpp_src or
1217 getTestOpts().cmm_src):
1218 extra_hc_opts += ' -no-hs-main '
1219
1220 if getTestOpts().compile_cmd_prefix == '':
1221 cmd_prefix = ''
1222 else:
1223 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1224
1225 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1226 if noforce:
1227 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1228 if getTestOpts().outputdir != None:
1229 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1230
1231 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1232 + config.compiler + "' " \
1233 + join(comp_flags,' ') + ' ' \
1234 + to_do + ' ' + srcname + ' ' \
1235 + join(config.way_flags(name)[way],' ') + ' ' \
1236 + extra_hc_opts + ' ' \
1237 + opts.extra_hc_opts + ' ' \
1238 + '>' + errname + ' 2>&1'
1239
1240 result = runCmdFor(name, cmd)
1241
1242 if result != 0 and not should_fail:
1243 actual_stderr = qualify(name, 'comp.stderr')
1244 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1245 if_verbose_dump(1,actual_stderr)
1246
1247 # ToDo: if the sub-shell was killed by ^C, then exit
1248
1249 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1250
1251 if badResult(statsResult):
1252 return statsResult
1253
1254 if should_fail:
1255 if result == 0:
1256 return failBecause('exit code 0')
1257 else:
1258 if result != 0:
1259 return failBecause('exit code non-0')
1260
1261 return passed()
1262
1263 # -----------------------------------------------------------------------------
1264 # Run a program and check its output
1265 #
1266 # If testname.stdin exists, route input from that, else
1267 # from /dev/null. Route output to testname.run.stdout and
1268 # testname.run.stderr. Returns the exit code of the run.
1269
1270 def simple_run( name, way, prog, args ):
1271 opts = getTestOpts()
1272
1273 # figure out what to use for stdin
1274 if opts.stdin != '':
1275 use_stdin = opts.stdin
1276 else:
1277 stdin_file = add_suffix(name, 'stdin')
1278 if os.path.exists(in_testdir(stdin_file)):
1279 use_stdin = stdin_file
1280 else:
1281 use_stdin = '/dev/null'
1282
1283 run_stdout = add_suffix(name,'run.stdout')
1284 run_stderr = add_suffix(name,'run.stderr')
1285
1286 rm_no_fail(qualify(name,'run.stdout'))
1287 rm_no_fail(qualify(name,'run.stderr'))
1288 rm_no_fail(qualify(name, 'hp'))
1289 rm_no_fail(qualify(name,'ps'))
1290 rm_no_fail(qualify(name, 'prof'))
1291
1292 my_rts_flags = rts_flags(way)
1293
1294 stats_file = name + '.stats'
1295 if len(opts.stats_range_fields) > 0:
1296 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1297
1298 if opts.no_stdin:
1299 stdin_comes_from = ''
1300 else:
1301 stdin_comes_from = ' <' + use_stdin
1302
1303 if opts.combined_output:
1304 redirection = ' >' + run_stdout \
1305 + ' 2>&1'
1306 else:
1307 redirection = ' >' + run_stdout \
1308 + ' 2>' + run_stderr
1309
1310 cmd = prog + ' ' + args + ' ' \
1311 + my_rts_flags + ' ' \
1312 + stdin_comes_from \
1313 + redirection
1314
1315 if opts.cmd_wrapper != None:
1316 cmd = opts.cmd_wrapper(cmd);
1317
1318 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1319
1320 # run the command
1321 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1322
1323 exit_code = result >> 8
1324 signal = result & 0xff
1325
1326 # check the exit code
1327 if exit_code != opts.exit_code:
1328 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1329 dump_stdout(name)
1330 dump_stderr(name)
1331 return failBecause('bad exit code')
1332
1333 check_hp = my_rts_flags.find("-h") != -1
1334 check_prof = my_rts_flags.find("-p") != -1
1335
1336 if not opts.ignore_output:
1337 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1338 bad_stdout = not check_stdout_ok(name)
1339 if bad_stderr:
1340 return failBecause('bad stderr')
1341 if bad_stdout:
1342 return failBecause('bad stdout')
1343 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1344 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1345 return failBecause('bad heap profile')
1346 if check_prof and not check_prof_ok(name):
1347 return failBecause('bad profile')
1348
1349 return checkStats(stats_file, opts.stats_range_fields)
1350
1351 def rts_flags(way):
1352 if (way == ''):
1353 return ''
1354 else:
1355 args = config.way_rts_flags[way]
1356
1357 if args == []:
1358 return ''
1359 else:
1360 return '+RTS ' + join(args,' ') + ' -RTS'
1361
1362 # -----------------------------------------------------------------------------
1363 # Run a program in the interpreter and check its output
1364
1365 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1366 outname = add_suffix(name, 'interp.stdout')
1367 errname = add_suffix(name, 'interp.stderr')
1368 rm_no_fail(outname)
1369 rm_no_fail(errname)
1370 rm_no_fail(name)
1371
1372 if (top_mod == ''):
1373 srcname = add_hs_lhs_suffix(name)
1374 else:
1375 srcname = top_mod
1376
1377 scriptname = add_suffix(name, 'genscript')
1378 qscriptname = in_testdir(scriptname)
1379 rm_no_fail(qscriptname)
1380
1381 delimiter = '===== program output begins here\n'
1382
1383 script = open(qscriptname, 'w')
1384 if not compile_only:
1385 # set the prog name and command-line args to match the compiled
1386 # environment.
1387 script.write(':set prog ' + name + '\n')
1388 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1389 # Add marker lines to the stdout and stderr output files, so we
1390 # can separate GHCi's output from the program's.
1391 script.write(':! echo ' + delimiter)
1392 script.write(':! echo 1>&2 ' + delimiter)
1393 # Set stdout to be line-buffered to match the compiled environment.
1394 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1395 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1396 # in the event of an exception as for the compiled program.
1397 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1398 script.close()
1399
1400 # figure out what to use for stdin
1401 if getTestOpts().stdin != '':
1402 stdin_file = in_testdir(getTestOpts().stdin)
1403 else:
1404 stdin_file = qualify(name, 'stdin')
1405
1406 if os.path.exists(stdin_file):
1407 stdin = open(stdin_file, 'r')
1408 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1409
1410 script.close()
1411
1412 flags = copy.copy(getTestOpts().compiler_always_flags)
1413 if getTestOpts().outputdir != None:
1414 flags.extend(["-outputdir", getTestOpts().outputdir])
1415
1416 cmd = "'" + config.compiler + "' " \
1417 + join(flags,' ') + ' ' \
1418 + srcname + ' ' \
1419 + join(config.way_flags(name)[way],' ') + ' ' \
1420 + extra_hc_opts + ' ' \
1421 + getTestOpts().extra_hc_opts + ' ' \
1422 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1423
1424 if getTestOpts().cmd_wrapper != None:
1425 cmd = getTestOpts().cmd_wrapper(cmd);
1426
1427 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1428
1429 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1430
1431 exit_code = result >> 8
1432 signal = result & 0xff
1433
1434 # split the stdout into compilation/program output
1435 split_file(in_testdir(outname), delimiter,
1436 qualify(name, 'comp.stdout'),
1437 qualify(name, 'run.stdout'))
1438 split_file(in_testdir(errname), delimiter,
1439 qualify(name, 'comp.stderr'),
1440 qualify(name, 'run.stderr'))
1441
1442 # check the exit code
1443 if exit_code != getTestOpts().exit_code:
1444 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1445 dump_stdout(name)
1446 dump_stderr(name)
1447 return failBecause('bad exit code')
1448
1449 # ToDo: if the sub-shell was killed by ^C, then exit
1450
1451 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1452 check_stdout_ok(name)):
1453 return passed()
1454 else:
1455 return failBecause('bad stdout or stderr')
1456
1457
1458 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1459 infile = open(in_fn)
1460 out1 = open(out1_fn, 'w')
1461 out2 = open(out2_fn, 'w')
1462
1463 line = infile.readline()
1464 line = re.sub('\r', '', line) # ignore Windows EOL
1465 while (re.sub('^\s*','',line) != delimiter and line != ''):
1466 out1.write(line)
1467 line = infile.readline()
1468 line = re.sub('\r', '', line)
1469 out1.close()
1470
1471 line = infile.readline()
1472 while (line != ''):
1473 out2.write(line)
1474 line = infile.readline()
1475 out2.close()
1476
1477 # -----------------------------------------------------------------------------
1478 # Generate External Core for the given program, then compile the resulting Core
1479 # and compare its output to the expected output
1480
1481 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1482
1483 depsfilename = qualify(name, 'deps')
1484 errname = add_suffix(name, 'comp.stderr')
1485 qerrname = qualify(errname,'')
1486
1487 hcname = qualify(name, 'hc')
1488 oname = qualify(name, 'o')
1489
1490 rm_no_fail( qerrname )
1491 rm_no_fail( qualify(name, '') )
1492
1493 if (top_mod == ''):
1494 srcname = add_hs_lhs_suffix(name)
1495 else:
1496 srcname = top_mod
1497
1498 qcorefilename = qualify(name, 'hcr')
1499 corefilename = add_suffix(name, 'hcr')
1500 rm_no_fail(qcorefilename)
1501
1502 # Generate External Core
1503
1504 if (top_mod == ''):
1505 to_do = ' ' + srcname + ' '
1506 else:
1507 to_do = ' --make ' + top_mod + ' '
1508
1509 flags = copy.copy(getTestOpts().compiler_always_flags)
1510 if getTestOpts().outputdir != None:
1511 flags.extend(["-outputdir", getTestOpts().outputdir])
1512 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1513 + config.compiler + "' " \
1514 + join(flags,' ') + ' ' \
1515 + join(config.way_flags(name)[way],' ') + ' ' \
1516 + extra_hc_opts + ' ' \
1517 + getTestOpts().extra_hc_opts \
1518 + to_do \
1519 + '>' + errname + ' 2>&1'
1520 result = runCmdFor(name, cmd)
1521
1522 exit_code = result >> 8
1523
1524 if exit_code != 0:
1525 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1526 if_verbose_dump(1,qerrname)
1527 return failBecause('ext core exit code non-0')
1528
1529 # Compile the resulting files -- if there's more than one module, we need to read the output
1530 # of the previous compilation in order to find the dependencies
1531 if (top_mod == ''):
1532 to_compile = corefilename
1533 else:
1534 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1535 deps = open(depsfilename).read()
1536 deplist = string.replace(deps, '\n',' ');
1537 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1538 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1539
1540 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1541 if getTestOpts().outputdir != None:
1542 flags.extend(["-outputdir", getTestOpts().outputdir])
1543
1544 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1545 + config.compiler + "' " \
1546 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1547 + to_compile + ' ' \
1548 + extra_hc_opts + ' ' \
1549 + getTestOpts().extra_hc_opts + ' ' \
1550 + flags \
1551 + ' -fglasgow-exts -o ' + name \
1552 + '>' + errname + ' 2>&1'
1553
1554 result = runCmdFor(name, cmd)
1555 exit_code = result >> 8
1556
1557 if exit_code != 0:
1558 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1559 if_verbose_dump(1,qerrname)
1560 return failBecause('ext core exit code non-0')
1561
1562 # Clean up
1563 rm_no_fail ( oname )
1564 rm_no_fail ( hcname )
1565 rm_no_fail ( qcorefilename )
1566 rm_no_fail ( depsfilename )
1567
1568 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1569
1570 # -----------------------------------------------------------------------------
1571 # Utils
1572
1573 def check_stdout_ok( name ):
1574 if getTestOpts().with_namebase == None:
1575 namebase = name
1576 else:
1577 namebase = getTestOpts().with_namebase
1578
1579 actual_stdout_file = qualify(name, 'run.stdout')
1580 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1581
1582 def norm(str):
1583 if platform_specific:
1584 return str
1585 else:
1586 return normalise_output(str)
1587
1588 return compare_outputs('stdout', \
1589 two_normalisers(norm, getTestOpts().extra_normaliser), \
1590 expected_stdout_file, actual_stdout_file)
1591
1592 def dump_stdout( name ):
1593 print 'Stdout:'
1594 print read_no_crs(qualify(name, 'run.stdout'))
1595
1596 def check_stderr_ok( name ):
1597 if getTestOpts().with_namebase == None:
1598 namebase = name
1599 else:
1600 namebase = getTestOpts().with_namebase
1601
1602 actual_stderr_file = qualify(name, 'run.stderr')
1603 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1604
1605 def norm(str):
1606 if platform_specific:
1607 return str
1608 else:
1609 return normalise_errmsg(str)
1610
1611 return compare_outputs('stderr', \
1612 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1613 expected_stderr_file, actual_stderr_file)
1614
1615 def dump_stderr( name ):
1616 print "Stderr:"
1617 print read_no_crs(qualify(name, 'run.stderr'))
1618
1619 def read_no_crs(file):
1620 str = ''
1621 try:
1622 h = open(file)
1623 str = h.read()
1624 h.close
1625 except:
1626 # On Windows, if the program fails very early, it seems the
1627 # files stdout/stderr are redirected to may not get created
1628 pass
1629 return re.sub('\r', '', str)
1630
1631 def write_file(file, str):
1632 h = open(file, 'w')
1633 h.write(str)
1634 h.close
1635
1636 def check_hp_ok(name):
1637
1638 # do not qualify for hp2ps because we should be in the right directory
1639 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1640
1641 hp2psResult = runCmdExitCode(hp2psCmd)
1642
1643 actual_ps_file = qualify(name, 'ps')
1644
1645 if(hp2psResult == 0):
1646 if (os.path.exists(actual_ps_file)):
1647 if gs_working:
1648 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1649 if (gsResult == 0):
1650 return (True)
1651 else:
1652 print "hp2ps output for " + name + "is not valid PostScript"
1653 else: return (True) # assume postscript is valid without ghostscript
1654 else:
1655 print "hp2ps did not generate PostScript for " + name
1656 return (False)
1657 else:
1658 print "hp2ps error when processing heap profile for " + name
1659 return(False)
1660
1661 def check_prof_ok(name):
1662
1663 prof_file = qualify(name,'prof')
1664
1665 if not os.path.exists(prof_file):
1666 print prof_file + " does not exist"
1667 return(False)
1668
1669 if os.path.getsize(qualify(name,'prof')) == 0:
1670 print prof_file + " is empty"
1671 return(False)
1672
1673 if getTestOpts().with_namebase == None:
1674 namebase = name
1675 else:
1676 namebase = getTestOpts().with_namebase
1677
1678 (platform_specific, expected_prof_file) = \
1679 platform_wordsize_qualify(namebase, 'prof.sample')
1680
1681 # sample prof file is not required
1682 if not os.path.exists(expected_prof_file):
1683 return True
1684 else:
1685 return compare_outputs('prof', \
1686 two_normalisers(normalise_whitespace,normalise_prof), \
1687 expected_prof_file, prof_file)
1688
1689 # Compare expected output to actual output, and optionally accept the
1690 # new output. Returns true if output matched or was accepted, false
1691 # otherwise.
1692 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1693 if os.path.exists(expected_file):
1694 expected_raw = read_no_crs(expected_file)
1695 # print "norm:", normaliser(expected_raw)
1696 expected_str = normaliser(expected_raw)
1697 expected_file_for_diff = expected_file
1698 else:
1699 expected_str = ''
1700 expected_file_for_diff = '/dev/null'
1701
1702 actual_raw = read_no_crs(actual_file)
1703 actual_str = normaliser(actual_raw)
1704
1705 if expected_str == actual_str:
1706 return 1
1707 else:
1708 print 'Actual ' + kind + ' output differs from expected:'
1709
1710 if expected_file_for_diff == '/dev/null':
1711 expected_normalised_file = '/dev/null'
1712 else:
1713 expected_normalised_file = expected_file + ".normalised"
1714 write_file(expected_normalised_file, expected_str)
1715
1716 actual_normalised_file = actual_file + ".normalised"
1717 write_file(actual_normalised_file, actual_str)
1718
1719 # Ignore whitespace when diffing. We should only get to this
1720 # point if there are non-whitespace differences
1721 #
1722 # Note we are diffing the *actual* output, not the normalised
1723 # output. The normalised output may have whitespace squashed
1724 # (including newlines) so the diff would be hard to read.
1725 # This does mean that the diff might contain changes that
1726 # would be normalised away.
1727 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1728 ' ' + actual_file )
1729
1730 # If for some reason there were no non-whitespace differences,
1731 # then do a full diff
1732 if r == 0:
1733 r = os.system( 'diff -u ' + expected_file_for_diff + \
1734 ' ' + actual_file )
1735
1736 if config.accept:
1737 print 'Accepting new output.'
1738 write_file(expected_file, actual_raw)
1739 return 1
1740 else:
1741 return 0
1742
1743
1744 def normalise_whitespace( str ):
1745 # Merge contiguous whitespace characters into a single space.
1746 str = re.sub('[ \t\n]+', ' ', str)
1747 return str
1748
1749 def normalise_errmsg( str ):
1750 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1751 # the colon is there because it appears in error messages; this
1752 # hacky solution is used in place of more sophisticated filename
1753 # mangling
1754 str = re.sub('([^\\s])\\.exe', '\\1', str)
1755 # normalise slashes, minimise Windows/Unix filename differences
1756 str = re.sub('\\\\', '/', str)
1757 # The inplace ghc's are called ghc-stage[123] to avoid filename
1758 # collisions, so we need to normalise that to just "ghc"
1759 str = re.sub('ghc-stage[123]', 'ghc', str)
1760 # We sometimes see the name of the integer-gmp package on stderr,
1761 # but this can change (either the implementation name or the
1762 # version number), so we canonicalise it here
1763 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1764 return str
1765
1766 # normalise a .prof file, so that we can reasonably compare it against
1767 # a sample. This doesn't compare any of the actual profiling data,
1768 # only the shape of the profile and the number of entries.
1769 def normalise_prof (str):
1770 # strip everything up to the line beginning "COST CENTRE"
1771 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1772
1773 # strip results for CAFs, these tend to change unpredictably
1774 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1775
1776 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1777 # sometimes under MAIN.
1778 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1779
1780 # We have somthing like this:
1781
1782 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1783 # k Main 204 1 0.0 0.0 0.0 0.0
1784 # foo Main 205 1 0.0 0.0 0.0 0.0
1785 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1786
1787 # then we remove all the specific profiling data, leaving only the
1788 # cost centre name, module, and entries, to end up with this:
1789
1790 # MAIN MAIN 0
1791 # k Main 1
1792 # foo Main 1
1793 # foo.bar Main 1
1794
1795 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1796 return str
1797
1798 def normalise_slashes_( str ):
1799 str = re.sub('\\\\', '/', str)
1800 return str
1801
1802 def normalise_exe_( str ):
1803 str = re.sub('\.exe', '', str)
1804 return str
1805
1806 def normalise_output( str ):
1807 # Remove a .exe extension (for Windows)
1808 # This can occur in error messages generated by the program.
1809 str = re.sub('([^\\s])\\.exe', '\\1', str)
1810 return str
1811
1812 def normalise_asm( str ):
1813 lines = str.split('\n')
1814 # Only keep instructions and labels not starting with a dot.
1815 metadata = re.compile('^[ \t]*\\..*$')
1816 out = []
1817 for line in lines:
1818 # Drop metadata directives (e.g. ".type")
1819 if not metadata.match(line):
1820 line = re.sub('@plt', '', line)
1821 instr = line.lstrip().split()
1822 # Drop empty lines.
1823 if not instr:
1824 continue
1825 # Drop operands, except for call instructions.
1826 elif instr[0] == 'call':
1827 out.append(instr[0] + ' ' + instr[1])
1828 else:
1829 out.append(instr[0])
1830 out = '\n'.join(out)
1831 return out
1832
1833 def if_verbose( n, str ):
1834 if config.verbose >= n:
1835 print str
1836
1837 def if_verbose_dump( n, f ):
1838 if config.verbose >= n:
1839 try:
1840 print open(f).read()
1841 except:
1842 print ''
1843
1844 def rawSystem(cmd_and_args):
1845 # We prefer subprocess.call to os.spawnv as the latter
1846 # seems to send its arguments through a shell or something
1847 # with the Windows (non-cygwin) python. An argument "a b c"
1848 # turns into three arguments ["a", "b", "c"].
1849
1850 # However, subprocess is new in python 2.4, so fall back to
1851 # using spawnv if we don't have it
1852
1853 if have_subprocess:
1854 return subprocess.call(cmd_and_args)
1855 else:
1856 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1857
1858 # Note that this doesn't handle the timeout itself; it is just used for
1859 # commands that have timeout handling built-in.
1860 def rawSystemWithTimeout(cmd_and_args):
1861 r = rawSystem(cmd_and_args)
1862 if r == 98:
1863 # The python timeout program uses 98 to signal that ^C was pressed
1864 stopNow()
1865 return r
1866
1867 # cmd is a complex command in Bourne-shell syntax
1868 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1869 # Hence it must ultimately be run by a Bourne shell
1870 #
1871 # Mostly it invokes the command wrapped in 'timeout' thus
1872 # timeout 300 'cd . && ...blah blah'
1873 # so it's timeout's job to invoke the Bourne shell
1874 #
1875 # But watch out for the case when there is no timeout program!
1876 # Then, when using the native Python, os.system will invoke the cmd shell
1877
1878 def runCmd( cmd ):
1879 if_verbose( 1, cmd )
1880 r = 0
1881 if config.os == 'mingw32':
1882 # On MinGW, we will always have timeout
1883 assert config.timeout_prog!=''
1884
1885 if config.timeout_prog != '':
1886 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1887 else:
1888 r = os.system(cmd)
1889 return r << 8
1890
1891 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1892 if_verbose( 1, cmd )
1893 r = 0
1894 if config.os == 'mingw32':
1895 # On MinGW, we will always have timeout
1896 assert config.timeout_prog!=''
1897 timeout = int(ceil(config.timeout * timeout_multiplier))
1898
1899 if config.timeout_prog != '':
1900 if config.check_files_written:
1901 fn = name + ".strace"
1902 r = rawSystemWithTimeout(
1903 ["strace", "-o", fn, "-fF",
1904 "-e", "creat,open,chdir,clone,vfork",
1905 config.timeout_prog, str(timeout), cmd])
1906 addTestFilesWritten(name, fn)
1907 rm_no_fail(fn)
1908 else:
1909 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1910 else:
1911 r = os.system(cmd)
1912 return r << 8
1913
1914 def runCmdExitCode( cmd ):
1915 return (runCmd(cmd) >> 8);
1916
1917
1918 # -----------------------------------------------------------------------------
1919 # checking for files being written to by multiple tests
1920
1921 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1922 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1923 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1924 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1925 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1926 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1927 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1928 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1929 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1930 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1931 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1932 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1933 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1934 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1935 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1936 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1937 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1938
1939 # Files that are read or written but shouldn't be:
1940 # * ghci_history shouldn't be read or written by tests
1941 # * things under package.conf.d shouldn't be written by tests
1942 bad_file_usages = {}
1943
1944 # Mapping from tests to the list of files that they write
1945 files_written = {}
1946
1947 # Mapping from tests to the list of files that they write but don't clean
1948 files_written_not_removed = {}
1949
1950 def add_bad_file_usage(name, file):
1951 try:
1952 if not file in bad_file_usages[name]:
1953 bad_file_usages[name].append(file)
1954 except:
1955 bad_file_usages[name] = [file]
1956
1957 def mkPath(curdir, path):
1958 # Given the current full directory is 'curdir', what is the full
1959 # path to 'path'?
1960 return os.path.realpath(os.path.join(curdir, path))
1961
1962 def addTestFilesWritten(name, fn):
1963 if config.use_threads:
1964 with t.lockFilesWritten:
1965 addTestFilesWrittenHelper(name, fn)
1966 else:
1967 addTestFilesWrittenHelper(name, fn)
1968
1969 def addTestFilesWrittenHelper(name, fn):
1970 started = False
1971 working_directories = {}
1972
1973 with open(fn, 'r') as f:
1974 for line in f:
1975 m_pid = re_strace_pid.match(line)
1976 if m_pid:
1977 pid = m_pid.group(1)
1978 content = m_pid.group(2)
1979 elif re_strace_unavailable.match(line):
1980 next
1981 else:
1982 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1983
1984 m_open = re_strace_open.match(content)
1985 m_chdir = re_strace_chdir.match(content)
1986 m_clone = re_strace_clone.match(content)
1987 m_vfork = re_strace_vfork.match(content)
1988
1989 if not started:
1990 working_directories[pid] = os.getcwd()
1991 started = True
1992
1993 if m_open:
1994 file = m_open.group(1)
1995 file = mkPath(working_directories[pid], file)
1996 if file.endswith("ghci_history"):
1997 add_bad_file_usage(name, file)
1998 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1999 flags = m_open.group(2).split('|')
2000 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
2001 if re.match('package\.conf\.d', file):
2002 add_bad_file_usage(name, file)
2003 else:
2004 try:
2005 if not file in files_written[name]:
2006 files_written[name].append(file)
2007 except:
2008 files_written[name] = [file]
2009 elif 'O_RDONLY' in flags:
2010 pass
2011 else:
2012 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2013 elif m_chdir:
2014 # We optimistically assume that unfinished chdir's are going to succeed
2015 dir = m_chdir.group(1)
2016 working_directories[pid] = mkPath(working_directories[pid], dir)
2017 elif m_clone:
2018 working_directories[m_clone.group(2)] = working_directories[pid]
2019 elif m_vfork:
2020 working_directories[m_vfork.group(2)] = working_directories[pid]
2021 elif re_strace_open_resumed.match(content):
2022 pass
2023 elif re_strace_chdir_resumed.match(content):
2024 pass
2025 elif re_strace_vfork_unfinished.match(content):
2026 pass
2027 elif re_strace_clone_unfinished.match(content):
2028 pass
2029 elif re_strace_ignore_sigchild.match(content):
2030 pass
2031 elif re_strace_ignore_sigvtalarm.match(content):
2032 pass
2033 elif re_strace_ignore_sigint.match(content):
2034 pass
2035 elif re_strace_ignore_sigfpe.match(content):
2036 pass
2037 elif re_strace_ignore_sigsegv.match(content):
2038 pass
2039 elif re_strace_ignore_sigpipe.match(content):
2040 pass
2041 else:
2042 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2043
2044 def checkForFilesWrittenProblems(file):
2045 foundProblem = False
2046
2047 files_written_inverted = {}
2048 for t in files_written.keys():
2049 for f in files_written[t]:
2050 try:
2051 files_written_inverted[f].append(t)
2052 except:
2053 files_written_inverted[f] = [t]
2054
2055 for f in files_written_inverted.keys():
2056 if len(files_written_inverted[f]) > 1:
2057 if not foundProblem:
2058 foundProblem = True
2059 file.write("\n")
2060 file.write("\nSome files are written by multiple tests:\n")
2061 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2062 if foundProblem:
2063 file.write("\n")
2064
2065 # -----
2066
2067 if len(files_written_not_removed) > 0:
2068 file.write("\n")
2069 file.write("\nSome files written but not removed:\n")
2070 tests = files_written_not_removed.keys()
2071 tests.sort()
2072 for t in tests:
2073 for f in files_written_not_removed[t]:
2074 file.write(" " + t + ": " + f + "\n")
2075 file.write("\n")
2076
2077 # -----
2078
2079 if len(bad_file_usages) > 0:
2080 file.write("\n")
2081 file.write("\nSome bad file usages:\n")
2082 tests = bad_file_usages.keys()
2083 tests.sort()
2084 for t in tests:
2085 for f in bad_file_usages[t]:
2086 file.write(" " + t + ": " + f + "\n")
2087 file.write("\n")
2088
2089 # -----------------------------------------------------------------------------
2090 # checking if ghostscript is available for checking the output of hp2ps
2091
2092 def genGSCmd(psfile):
2093 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2094
2095 def gsNotWorking():
2096 global gs_working
2097 print "GhostScript not available for hp2ps tests"
2098
2099 global gs_working
2100 gs_working = 0
2101 if config.have_profiling:
2102 if config.gs != '':
2103 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2104 if resultGood == 0:
2105 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2106 if resultBad != 0:
2107 print "GhostScript available for hp2ps tests"
2108 gs_working = 1;
2109 else:
2110 gsNotWorking();
2111 else:
2112 gsNotWorking();
2113 else:
2114 gsNotWorking();
2115
2116 def rm_no_fail( file ):
2117 try:
2118 os.remove( file )
2119 finally:
2120 return
2121
2122 def add_suffix( name, suffix ):
2123 if suffix == '':
2124 return name
2125 else:
2126 return name + '.' + suffix
2127
2128 def add_hs_lhs_suffix(name):
2129 if getTestOpts().c_src:
2130 return add_suffix(name, 'c')
2131 elif getTestOpts().cmm_src:
2132 return add_suffix(name, 'cmm')
2133 elif getTestOpts().objc_src:
2134 return add_suffix(name, 'm')
2135 elif getTestOpts().objcpp_src:
2136 return add_suffix(name, 'mm')
2137 elif getTestOpts().literate:
2138 return add_suffix(name, 'lhs')
2139 else:
2140 return add_suffix(name, 'hs')
2141
2142 def replace_suffix( name, suffix ):
2143 base, suf = os.path.splitext(name)
2144 return base + '.' + suffix
2145
2146 def in_testdir( name ):
2147 return (getTestOpts().testdir + '/' + name)
2148
2149 def qualify( name, suff ):
2150 return in_testdir(add_suffix(name, suff))
2151
2152
2153 # Finding the sample output. The filename is of the form
2154 #
2155 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2156 #
2157 # and we pick the most specific version available. The <version> is
2158 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2159 # more fine-grained control use if_compiler_lt().
2160 #
2161 def platform_wordsize_qualify( name, suff ):
2162
2163 basepath = qualify(name, suff)
2164
2165 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2166 for (platformSpecific, plat) in [(1, '-' + config.platform),
2167 (1, '-' + config.os),
2168 (0, '')]
2169 for ws in ['-ws-' + config.wordsize, '']
2170 for comp in ['-' + config.compiler_type, '']
2171 for vers in ['-' + config.compiler_maj_version, '']]
2172
2173 dir = glob.glob(basepath + '*')
2174 dir = map (lambda d: normalise_slashes_(d), dir)
2175
2176 for (platformSpecific, f) in paths:
2177 if f in dir:
2178 return (platformSpecific,f)
2179
2180 return (0, basepath)
2181
2182 # Clean up prior to the test, so that we can't spuriously conclude
2183 # that it passed on the basis of old run outputs.
2184 def pretest_cleanup(name):
2185 if getTestOpts().outputdir != None:
2186 odir = in_testdir(getTestOpts().outputdir)
2187 try:
2188 shutil.rmtree(odir)
2189 except:
2190 pass
2191 os.mkdir(odir)
2192
2193 rm_no_fail(qualify(name,'comp.stderr'))
2194 rm_no_fail(qualify(name,'run.stderr'))
2195 rm_no_fail(qualify(name,'run.stdout'))
2196 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2197 # simple_build zaps the following:
2198 # rm_nofail(qualify("o"))
2199 # rm_nofail(qualify(""))
2200 # not interested in the return code
2201
2202 # -----------------------------------------------------------------------------
2203 # Return a list of all the files ending in '.T' below the directory dir.
2204
2205 def findTFiles(roots):
2206 return concat(map(findTFiles_,roots))
2207
2208 def findTFiles_(path):
2209 if os.path.isdir(path):
2210 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2211 return findTFiles(paths)
2212 elif path[-2:] == '.T':
2213 return [path]
2214 else:
2215 return []
2216
2217 # -----------------------------------------------------------------------------
2218 # Output a test summary to the specified file object
2219
2220 def summary(t, file):
2221
2222 file.write('\n')
2223 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2224 file.write('OVERALL SUMMARY for test run started at '
2225 + t.start_time + '\n'
2226 + string.rjust(`t.total_tests`, 8)
2227 + ' total tests, which gave rise to\n'
2228 + string.rjust(`t.total_test_cases`, 8)
2229 + ' test cases, of which\n'
2230 + string.rjust(`t.n_tests_skipped`, 8)
2231 + ' were skipped\n'
2232 + '\n'
2233 + string.rjust(`t.n_missing_libs`, 8)
2234 + ' had missing libraries\n'
2235 + string.rjust(`t.n_expected_passes`, 8)
2236 + ' expected passes\n'
2237 + string.rjust(`t.n_expected_failures`, 8)
2238 + ' expected failures\n'
2239 + '\n'
2240 + string.rjust(`t.n_framework_failures`, 8)
2241 + ' caused framework failures\n'
2242 + string.rjust(`t.n_unexpected_passes`, 8)
2243 + ' unexpected passes\n'
2244 + string.rjust(`t.n_unexpected_failures`, 8)
2245 + ' unexpected failures\n'
2246 + '\n')
2247
2248 if t.n_unexpected_passes > 0:
2249 file.write('Unexpected passes:\n')
2250 printPassingTestInfosSummary(file, t.unexpected_passes)
2251
2252 if t.n_unexpected_failures > 0:
2253 file.write('Unexpected failures:\n')
2254 printFailingTestInfosSummary(file, t.unexpected_failures)
2255
2256 if config.check_files_written:
2257 checkForFilesWrittenProblems(file)
2258
2259 if stopping():
2260 file.write('WARNING: Testsuite run was terminated early\n')
2261
2262 def printUnexpectedTests(file, testInfoss):
2263 unexpected = []
2264 for testInfos in testInfoss:
2265 directories = testInfos.keys()
2266 for directory in directories:
2267 tests = testInfos[directory].keys()
2268 unexpected += tests
2269 if unexpected != []:
2270 file.write('Unexpected results from:\n')
2271 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2272 file.write('\n')
2273
2274 def printPassingTestInfosSummary(file, testInfos):
2275 directories = testInfos.keys()
2276 directories.sort()
2277 maxDirLen = max(map ((lambda x : len(x)), directories))
2278 for directory in directories:
2279 tests = testInfos[directory].keys()
2280 tests.sort()
2281 for test in tests:
2282 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2283 ' (' + join(testInfos[directory][test],',') + ')\n')
2284 file.write('\n')
2285
2286 def printFailingTestInfosSummary(file, testInfos):
2287 directories = testInfos.keys()
2288 directories.sort()
2289 maxDirLen = max(map ((lambda x : len(x)), directories))
2290 for directory in directories:
2291 tests = testInfos[directory].keys()
2292 tests.sort()
2293 for test in tests:
2294 reasons = testInfos[directory][test].keys()
2295 for reason in reasons:
2296 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2297 ' [' + reason + ']' + \
2298 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2299 file.write('\n')
2300
2301 def getStdout(cmd):
2302 if have_subprocess:
2303 p = subprocess.Popen(cmd,
2304 stdout=subprocess.PIPE,
2305 stderr=subprocess.PIPE)
2306 (stdout, stderr) = p.communicate()
2307 r = p.wait()
2308 if r != 0:
2309 raise Exception("Command failed: " + str(cmd))
2310 if stderr != '':
2311 raise Exception("stderr from command: " + str(cmd))
2312 return stdout
2313 else:
2314 raise Exception("Need subprocess to get stdout, but don't have it")