Merge branch 'master' of http://darcs.haskell.org/testsuite
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 opts.expect = 'fail';
138
139 def ignore_output( name, opts ):
140 opts.ignore_output = 1
141
142 def no_stdin( name, opts ):
143 opts.no_stdin = 1
144
145 def combined_output( name, opts ):
146 opts.combined_output = True
147
148 # -----
149
150 def expect_fail_for( ways ):
151 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
152
153 def _expect_fail_for( name, opts, ways ):
154 opts.expect_fail_for = ways
155
156 def expect_broken_for( bug, ways ):
157 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
158
159 def _expect_broken_for( name, opts, bug, ways ):
160 opts.expect_fail_for = ways
161
162 # -----
163
164 def omit_ways( ways ):
165 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
166
167 def _omit_ways( name, opts, ways ):
168 opts.omit_ways = ways
169
170 # -----
171
172 def only_ways( ways ):
173 return lambda name, opts, w=ways: _only_ways( name, opts, w )
174
175 def _only_ways( name, opts, ways ):
176 opts.only_ways = ways
177
178 # -----
179
180 def extra_ways( ways ):
181 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
182
183 def _extra_ways( name, opts, ways ):
184 opts.extra_ways = ways
185
186 # -----
187
188 def omit_compiler_types( compiler_types ):
189 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
190
191 def _omit_compiler_types( name, opts, compiler_types ):
192 if config.compiler_type in compiler_types:
193 opts.skip = 1
194
195 # -----
196
197 def only_compiler_types( compiler_types ):
198 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
199
200 def _only_compiler_types( name, opts, compiler_types ):
201 if config.compiler_type not in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def set_stdin( file ):
207 return lambda name, opts, f=file: _set_stdin(name, opts, f);
208
209 def _set_stdin( name, opts, f ):
210 opts.stdin = f
211
212 # -----
213
214 def exit_code( val ):
215 return lambda name, opts, v=val: _exit_code(name, opts, v);
216
217 def _exit_code( name, opts, v ):
218 opts.exit_code = v
219
220 # -----
221
222 def timeout_multiplier( val ):
223 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
224
225 def _timeout_multiplier( name, opts, v ):
226 opts.timeout_multiplier = v
227
228 # -----
229
230 def extra_run_opts( val ):
231 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
232
233 def _extra_run_opts( name, opts, v ):
234 opts.extra_run_opts = v
235
236 # -----
237
238 def extra_hc_opts( val ):
239 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
240
241 def _extra_hc_opts( name, opts, v ):
242 opts.extra_hc_opts = v
243
244 # -----
245
246 def extra_clean( files ):
247 return lambda name, opts, v=files: _extra_clean(name, opts, v);
248
249 def _extra_clean( name, opts, v ):
250 opts.clean_files = v
251
252 # -----
253
254 def stats_num_field( field, expecteds ):
255 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
256
257 def _stats_num_field( name, opts, field, expecteds ):
258 if field in opts.stats_range_fields:
259 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
260
261 if type(expecteds) is types.ListType:
262 for (b, expected, dev) in expecteds:
263 if b:
264 opts.stats_range_fields[field] = (expected, dev)
265 return
266 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
267
268 else:
269 (expected, dev) = expecteds
270 opts.stats_range_fields[field] = (expected, dev)
271
272 def compiler_stats_num_field( field, expecteds ):
273 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
274
275 def _compiler_stats_num_field( name, opts, field, expecteds ):
276 if field in opts.compiler_stats_range_fields:
277 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
278
279 for (b, expected, dev) in expecteds:
280 if b:
281 opts.compiler_stats_range_fields[field] = (expected, dev)
282 return
283
284 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
285
286 # -----
287
288 def skip_if_no_ghci(name, opts):
289 if not ('ghci' in config.run_ways):
290 opts.skip = 1
291
292 # ----
293
294 def skip_if_fast(name, opts):
295 if config.fast:
296 opts.skip = 1
297
298 # -----
299
300 def when(b, f):
301 if b:
302 return f
303 else:
304 return normal
305
306 def unless(b, f):
307 return when(not b, f)
308
309 def platform( plat ):
310 return config.platform == plat
311
312 def if_os( os, f ):
313 if config.os == os:
314 return f
315 else:
316 return normal
317
318 def unless_os( os, f ):
319 if config.os == os:
320 return normal
321 else:
322 return f
323
324 def if_arch( arch, f ):
325 if config.arch == arch:
326 return f
327 else:
328 return normal
329
330 def unless_arch( arch, f ):
331 if config.arch == arch:
332 return normal
333 else:
334 return f
335
336 def wordsize( ws ):
337 return config.wordsize == str(ws)
338
339 def if_unregisterised( f ):
340 if config.unregisterised:
341 return f
342 else:
343 return normal
344
345 def unless_unregisterised( f ):
346 if config.unregisterised:
347 return normal
348 else:
349 return f
350
351 def if_msys( f ):
352 if config.msys:
353 return f
354 else:
355 return normal
356
357 def if_cygwin( f ):
358 if config.cygwin:
359 return f
360 else:
361 return normal
362
363 def when_have_vanilla( f ):
364 if config.have_vanilla:
365 return f
366 else:
367 return normal
368
369 def unless_have_vanilla( f ):
370 if config.have_vanilla:
371 return normal
372 else:
373 return f
374
375 def when_have_dynamic( f ):
376 if config.have_dynamic:
377 return f
378 else:
379 return normal
380
381 def unless_have_dynamic( f ):
382 if config.have_dynamic:
383 return normal
384 else:
385 return f
386
387 def when_have_profiling( f ):
388 if config.have_profiling:
389 return f
390 else:
391 return normal
392
393 def unless_have_profiling( f ):
394 if config.have_profiling:
395 return normal
396 else:
397 return f
398
399 # ---
400
401 def if_ghci_dynamic( f ):
402 if config.ghc_dynamic_by_default:
403 return f
404 else:
405 return normal
406
407 def if_in_tree_compiler( f ):
408 if config.in_tree_compiler:
409 return f
410 else:
411 return normal
412
413 def unless_in_tree_compiler( f ):
414 if config.in_tree_compiler:
415 return normal
416 else:
417 return f
418
419 def if_compiler_type( compiler, f ):
420 if config.compiler_type == compiler:
421 return f
422 else:
423 return normal
424
425 def if_compiler_profiled( f ):
426 if config.compiler_profiled:
427 return f
428 else:
429 return normal
430
431 def unless_compiler_profiled( f ):
432 if config.compiler_profiled:
433 return normal
434 else:
435 return f
436
437 def if_compiler_lt( compiler, version, f ):
438 if config.compiler_type == compiler and \
439 version_lt(config.compiler_version, version):
440 return f
441 else:
442 return normal
443
444 def if_compiler_le( compiler, version, f ):
445 if config.compiler_type == compiler and \
446 version_le(config.compiler_version, version):
447 return f
448 else:
449 return normal
450
451 def if_compiler_gt( compiler, version, f ):
452 if config.compiler_type == compiler and \
453 version_gt(config.compiler_version, version):
454 return f
455 else:
456 return normal
457
458 def if_compiler_ge( compiler, version, f ):
459 if config.compiler_type == compiler and \
460 version_ge(config.compiler_version, version):
461 return f
462 else:
463 return normal
464
465 def if_compiler_debugged( f ):
466 if config.compiler_debugged:
467 return f
468 else:
469 return normal
470
471 def namebase( nb ):
472 return lambda opts, nb=nb: _namebase(opts, nb)
473
474 def _namebase( opts, nb ):
475 opts.with_namebase = nb
476
477 # ---
478
479 def if_tag( tag, f ):
480 if tag in config.compiler_tags:
481 return f
482 else:
483 return normal
484
485 def unless_tag( tag, f ):
486 if not (tag in config.compiler_tags):
487 return f
488 else:
489 return normal
490
491 # ---
492 def high_memory_usage(name, opts):
493 opts.alone = True
494
495 # ---
496 def literate( name, opts ):
497 opts.literate = 1;
498
499 def c_src( name, opts ):
500 opts.c_src = 1;
501
502 def objc_src( name, opts ):
503 opts.objc_src = 1;
504
505 def objcpp_src( name, opts ):
506 opts.objcpp_src = 1;
507
508 def cmm_src( name, opts ):
509 opts.cmm_src = 1;
510
511 def outputdir( odir ):
512 return lambda name, opts, d=odir: _outputdir(name, opts, d)
513
514 def _outputdir( name, opts, odir ):
515 opts.outputdir = odir;
516
517 # ----
518
519 def pre_cmd( cmd ):
520 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
521
522 def _pre_cmd( name, opts, cmd ):
523 opts.pre_cmd = cmd
524
525 # ----
526
527 def clean_cmd( cmd ):
528 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
529
530 def _clean_cmd( name, opts, cmd ):
531 opts.clean_cmd = cmd
532
533 # ----
534
535 def cmd_prefix( prefix ):
536 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
537
538 def _cmd_prefix( name, opts, prefix ):
539 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
540
541 # ----
542
543 def cmd_wrapper( fun ):
544 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
545
546 def _cmd_wrapper( name, opts, fun ):
547 opts.cmd_wrapper = fun
548
549 # ----
550
551 def compile_cmd_prefix( prefix ):
552 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
553
554 def _compile_cmd_prefix( name, opts, prefix ):
555 opts.compile_cmd_prefix = prefix
556
557 # ----
558
559 def normalise_slashes( name, opts ):
560 opts.extra_normaliser = normalise_slashes_
561
562 def normalise_exe( name, opts ):
563 opts.extra_normaliser = normalise_exe_
564
565 def normalise_fun( fun ):
566 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
567
568 def _normalise_fun( name, opts, f ):
569 opts.extra_normaliser = f
570
571 def normalise_errmsg_fun( fun ):
572 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
573
574 def _normalise_errmsg_fun( name, opts, f ):
575 opts.extra_errmsg_normaliser = f
576
577 def two_normalisers(f, g):
578 return lambda x, f=f, g=g: f(g(x))
579
580 # ----
581 # Function for composing two opt-fns together
582
583 def composes( fs ):
584 return reduce(lambda f, g: compose(f, g), fs)
585
586 def compose( f, g ):
587 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
588
589 def _compose( name, opts, f, g ):
590 f(name, opts)
591 g(name, opts)
592
593 # -----------------------------------------------------------------------------
594 # The current directory of tests
595
596 def newTestDir( dir ):
597 global thisdir_settings
598 # reset the options for this test directory
599 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
600
601 def _newTestDir( name, opts, dir ):
602 opts.testdir = dir
603 opts.compiler_always_flags = config.compiler_always_flags
604
605 # -----------------------------------------------------------------------------
606 # Actually doing tests
607
608 parallelTests = []
609 aloneTests = []
610 allTestNames = set([])
611
612 def runTest (opts, name, func, args):
613 ok = 0
614
615 if config.use_threads:
616 t.thread_pool.acquire()
617 try:
618 while config.threads<(t.running_threads+1):
619 t.thread_pool.wait()
620 t.running_threads = t.running_threads+1
621 ok=1
622 t.thread_pool.release()
623 thread.start_new_thread(test_common_thread, (name, opts, func, args))
624 except:
625 if not ok:
626 t.thread_pool.release()
627 else:
628 test_common_work (name, opts, func, args)
629
630 # name :: String
631 # setup :: TestOpts -> IO ()
632 def test (name, setup, func, args):
633 global aloneTests
634 global parallelTests
635 global allTestNames
636 global thisdir_settings
637 if name in allTestNames:
638 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
639 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
640 framework_fail(name, 'bad_name', 'This test has an invalid name')
641
642 # Make a deep copy of the default_testopts, as we need our own copy
643 # of any dictionaries etc inside it. Otherwise, if one test modifies
644 # them, all tests will see the modified version!
645 myTestOpts = copy.deepcopy(default_testopts)
646
647 if type(setup) is types.ListType:
648 setup = composes(setup)
649
650 setup = compose(thisdir_settings, setup)
651 setup(name, myTestOpts)
652
653 thisTest = lambda : runTest(myTestOpts, name, func, args)
654 if myTestOpts.alone:
655 aloneTests.append(thisTest)
656 else:
657 parallelTests.append(thisTest)
658 allTestNames.add(name)
659
660 if config.use_threads:
661 def test_common_thread(name, opts, func, args):
662 t.lock.acquire()
663 try:
664 test_common_work(name,opts,func,args)
665 finally:
666 t.lock.release()
667 t.thread_pool.acquire()
668 t.running_threads = t.running_threads - 1
669 t.thread_pool.notify()
670 t.thread_pool.release()
671
672 def get_package_cache_timestamp():
673 if config.package_conf_cache_file == '':
674 return 0.0
675 else:
676 try:
677 return os.stat(config.package_conf_cache_file).st_mtime
678 except:
679 return 0.0
680
681
682 def test_common_work (name, opts, func, args):
683 try:
684 t.total_tests = t.total_tests+1
685 setLocalTestOpts(opts)
686
687 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
688
689 # All the ways we might run this test
690 if func == compile or func == multimod_compile:
691 all_ways = config.compile_ways
692 elif func == compile_and_run or func == multimod_compile_and_run:
693 all_ways = config.run_ways
694 elif func == ghci_script:
695 if 'ghci' in config.run_ways:
696 all_ways = ['ghci']
697 else:
698 all_ways = []
699 else:
700 all_ways = ['normal']
701
702 # A test itself can request extra ways by setting opts.extra_ways
703 all_ways = all_ways + filter(lambda way: way not in all_ways,
704 opts.extra_ways)
705
706 t.total_test_cases = t.total_test_cases + len(all_ways)
707
708 ok_way = lambda way: \
709 not getTestOpts().skip \
710 and (config.only == [] or name in config.only) \
711 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
712 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
713 and way not in getTestOpts().omit_ways
714
715 # Which ways we are asked to skip
716 do_ways = filter (ok_way,all_ways)
717
718 # In fast mode, we skip all but one way
719 if config.fast and len(do_ways) > 0:
720 do_ways = [do_ways[0]]
721
722 if not config.clean_only:
723 # Run the required tests...
724 for way in do_ways:
725 if stopping():
726 break
727 do_test (name, way, func, args)
728
729 for way in all_ways:
730 if way not in do_ways:
731 skiptest (name,way)
732
733 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
734 clean(map (lambda suff: name + suff,
735 ['', '.exe', '.exe.manifest', '.genscript',
736 '.stderr.normalised', '.stdout.normalised',
737 '.run.stderr', '.run.stdout',
738 '.run.stderr.normalised', '.run.stdout.normalised',
739 '.comp.stderr', '.comp.stdout',
740 '.comp.stderr.normalised', '.comp.stdout.normalised',
741 '.interp.stderr', '.interp.stdout',
742 '.interp.stderr.normalised', '.interp.stdout.normalised',
743 '.stats', '.comp.stats',
744 '.hi', '.o', '.prof', '.exe.prof', '.hc',
745 '_stub.h', '_stub.c', '_stub.o',
746 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
747
748 if func == multi_compile or func == multi_compile_fail:
749 extra_mods = args[1]
750 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
751 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
752
753 clean(getTestOpts().clean_files)
754
755 if getTestOpts().outputdir != None:
756 odir = in_testdir(getTestOpts().outputdir)
757 try:
758 shutil.rmtree(odir)
759 except:
760 pass
761
762 try:
763 shutil.rmtree(in_testdir('.hpc.' + name))
764 except:
765 pass
766
767 try:
768 cleanCmd = getTestOpts().clean_cmd
769 if cleanCmd != None:
770 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
771 if result != 0:
772 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
773 except e:
774 framework_fail(name, 'cleaning', 'clean-command exception')
775
776 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
777
778 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
779 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
780
781 try:
782 for f in files_written[name]:
783 if os.path.exists(f):
784 try:
785 if not f in files_written_not_removed[name]:
786 files_written_not_removed[name].append(f)
787 except:
788 files_written_not_removed[name] = [f]
789 except:
790 pass
791 except Exception, e:
792 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
793
794 def clean(strs):
795 for str in strs:
796 for name in glob.glob(in_testdir(str)):
797 clean_full_path(name)
798
799 def clean_full_path(name):
800 try:
801 # Remove files...
802 os.remove(name)
803 except OSError, e1:
804 try:
805 # ... and empty directories
806 os.rmdir(name)
807 except OSError, e2:
808 # We don't want to fail here, but we do want to know
809 # what went wrong, so print out the exceptions.
810 # ENOENT isn't a problem, though, as we clean files
811 # that don't necessarily exist.
812 if e1.errno != errno.ENOENT:
813 print e1
814 if e2.errno != errno.ENOENT:
815 print e2
816
817 def do_test(name, way, func, args):
818 full_name = name + '(' + way + ')'
819
820 try:
821 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
822 str([t.n_unexpected_passes, \
823 t.n_unexpected_failures, \
824 t.n_framework_failures])
825
826 if config.use_threads:
827 t.lock.release()
828
829 try:
830 preCmd = getTestOpts().pre_cmd
831 if preCmd != None:
832 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
833 if result != 0:
834 framework_fail(name, way, 'pre-command failed: ' + str(result))
835 except e:
836 framework_fail(name, way, 'pre-command exception')
837
838 try:
839 result = apply(func, [name,way] + args)
840 finally:
841 if config.use_threads:
842 t.lock.acquire()
843
844 if getTestOpts().expect != 'pass' and \
845 getTestOpts().expect != 'fail' and \
846 getTestOpts().expect != 'missing-lib':
847 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
848
849 try:
850 passFail = result['passFail']
851 except:
852 passFail = 'No passFail found'
853
854 if passFail == 'pass':
855 if getTestOpts().expect == 'pass' \
856 and way not in getTestOpts().expect_fail_for:
857 t.n_expected_passes = t.n_expected_passes + 1
858 if name in t.expected_passes:
859 t.expected_passes[name].append(way)
860 else:
861 t.expected_passes[name] = [way]
862 else:
863 print '*** unexpected pass for', full_name
864 t.n_unexpected_passes = t.n_unexpected_passes + 1
865 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
866 elif passFail == 'fail':
867 if getTestOpts().expect == 'pass' \
868 and way not in getTestOpts().expect_fail_for:
869 print '*** unexpected failure for', full_name
870 t.n_unexpected_failures = t.n_unexpected_failures + 1
871 reason = result['reason']
872 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
873 else:
874 if getTestOpts().expect == 'missing-lib':
875 t.n_missing_libs = t.n_missing_libs + 1
876 if name in t.missing_libs:
877 t.missing_libs[name].append(way)
878 else:
879 t.missing_libs[name] = [way]
880 else:
881 t.n_expected_failures = t.n_expected_failures + 1
882 if name in t.expected_failures:
883 t.expected_failures[name].append(way)
884 else:
885 t.expected_failures[name] = [way]
886 else:
887 framework_fail(name, way, 'bad result ' + passFail)
888 except KeyboardInterrupt:
889 stopNow()
890 except:
891 framework_fail(name, way, 'do_test exception')
892 traceback.print_exc()
893
894 def addPassingTestInfo (testInfos, directory, name, way):
895 directory = re.sub('^\\.[/\\\\]', '', directory)
896
897 if not directory in testInfos:
898 testInfos[directory] = {}
899
900 if not name in testInfos[directory]:
901 testInfos[directory][name] = []
902
903 testInfos[directory][name].append(way)
904
905 def addFailingTestInfo (testInfos, directory, name, reason, way):
906 directory = re.sub('^\\.[/\\\\]', '', directory)
907
908 if not directory in testInfos:
909 testInfos[directory] = {}
910
911 if not name in testInfos[directory]:
912 testInfos[directory][name] = {}
913
914 if not reason in testInfos[directory][name]:
915 testInfos[directory][name][reason] = []
916
917 testInfos[directory][name][reason].append(way)
918
919 def skiptest (name, way):
920 # print 'Skipping test \"', name, '\"'
921 t.n_tests_skipped = t.n_tests_skipped + 1
922 if name in t.tests_skipped:
923 t.tests_skipped[name].append(way)
924 else:
925 t.tests_skipped[name] = [way]
926
927 def framework_fail( name, way, reason ):
928 full_name = name + '(' + way + ')'
929 print '*** framework failure for', full_name, reason
930 t.n_framework_failures = t.n_framework_failures + 1
931 if name in t.framework_failures:
932 t.framework_failures[name].append(way)
933 else:
934 t.framework_failures[name] = [way]
935
936 def badResult(result):
937 try:
938 if result['passFail'] == 'pass':
939 return False
940 return True
941 except:
942 return True
943
944 def passed():
945 return {'passFail': 'pass'}
946
947 def failBecause(reason):
948 return {'passFail': 'fail', 'reason': reason}
949
950 # -----------------------------------------------------------------------------
951 # Generic command tests
952
953 # A generic command test is expected to run and exit successfully.
954 #
955 # The expected exit code can be changed via exit_code() as normal, and
956 # the expected stdout/stderr are stored in <testname>.stdout and
957 # <testname>.stderr. The output of the command can be ignored
958 # altogether by using run_command_ignore_output instead of
959 # run_command.
960
961 def run_command( name, way, cmd ):
962 return simple_run( name, '', cmd, '' )
963
964 # -----------------------------------------------------------------------------
965 # GHCi tests
966
967 def ghci_script( name, way, script ):
968 # filter out -fforce-recomp from compiler_always_flags, because we're
969 # actually testing the recompilation behaviour in the GHCi tests.
970 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
971 flags.append(getTestOpts().extra_hc_opts)
972 if getTestOpts().outputdir != None:
973 flags.extend(["-outputdir", getTestOpts().outputdir])
974
975 # We pass HC and HC_OPTS as environment variables, so that the
976 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
977 cmd = "HC='" + config.compiler + "' " + \
978 "HC_OPTS='" + join(flags,' ') + "' " + \
979 "'" + config.compiler + "'" + \
980 ' --interactive -v0 -ignore-dot-ghci ' + \
981 join(flags,' ')
982
983 getTestOpts().stdin = script
984 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
985
986 # -----------------------------------------------------------------------------
987 # Compile-only tests
988
989 def compile( name, way, extra_hc_opts ):
990 return do_compile( name, way, 0, '', [], extra_hc_opts )
991
992 def compile_fail( name, way, extra_hc_opts ):
993 return do_compile( name, way, 1, '', [], extra_hc_opts )
994
995 def multimod_compile( name, way, top_mod, extra_hc_opts ):
996 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
997
998 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
999 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1000
1001 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1002 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1003
1004 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1005 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1006
1007 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1008 # print 'Compile only, extra args = ', extra_hc_opts
1009 pretest_cleanup(name)
1010
1011 result = extras_build( way, extra_mods, extra_hc_opts )
1012 if badResult(result):
1013 return result
1014 extra_hc_opts = result['hc_opts']
1015
1016 force = 0
1017 if extra_mods:
1018 force = 1
1019 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1020
1021 if badResult(result):
1022 return result
1023
1024 # the actual stderr should always match the expected, regardless
1025 # of whether we expected the compilation to fail or not (successful
1026 # compilations may generate warnings).
1027
1028 if getTestOpts().with_namebase == None:
1029 namebase = name
1030 else:
1031 namebase = getTestOpts().with_namebase
1032
1033 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1034 actual_stderr_file = qualify(name, 'comp.stderr')
1035
1036 if not compare_outputs('stderr', \
1037 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1038 expected_stderr_file, actual_stderr_file):
1039 return failBecause('stderr mismatch')
1040
1041 # no problems found, this test passed
1042 return passed()
1043
1044 def compile_cmp_asm( name, way, extra_hc_opts ):
1045 print 'Compile only, extra args = ', extra_hc_opts
1046 pretest_cleanup(name)
1047 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1048
1049 if badResult(result):
1050 return result
1051
1052 # the actual stderr should always match the expected, regardless
1053 # of whether we expected the compilation to fail or not (successful
1054 # compilations may generate warnings).
1055
1056 if getTestOpts().with_namebase == None:
1057 namebase = name
1058 else:
1059 namebase = getTestOpts().with_namebase
1060
1061 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1062 actual_asm_file = qualify(name, 's')
1063
1064 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1065 expected_asm_file, actual_asm_file):
1066 return failBecause('asm mismatch')
1067
1068 # no problems found, this test passed
1069 return passed()
1070
1071 # -----------------------------------------------------------------------------
1072 # Compile-and-run tests
1073
1074 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1075 # print 'Compile and run, extra args = ', extra_hc_opts
1076 pretest_cleanup(name)
1077
1078 result = extras_build( way, extra_mods, extra_hc_opts )
1079 if badResult(result):
1080 return result
1081 extra_hc_opts = result['hc_opts']
1082
1083 if way == 'ghci': # interpreted...
1084 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1085 elif way == 'extcore' or way == 'optextcore' :
1086 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1087 else: # compiled...
1088 force = 0
1089 if extra_mods:
1090 force = 1
1091
1092 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1093 if badResult(result):
1094 return result
1095
1096 cmd = './' + name;
1097
1098 # we don't check the compiler's stderr for a compile-and-run test
1099 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1100
1101 def compile_and_run( name, way, extra_hc_opts ):
1102 return compile_and_run__( name, way, '', [], extra_hc_opts)
1103
1104 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1105 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1106
1107 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1108 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1109
1110 def stats( name, way, stats_file ):
1111 opts = getTestOpts()
1112 return checkStats(stats_file, opts.stats_range_fields)
1113
1114 # -----------------------------------------------------------------------------
1115 # Check -t stats info
1116
1117 def checkStats(stats_file, range_fields):
1118 result = passed()
1119 if len(range_fields) > 0:
1120 f = open(in_testdir(stats_file))
1121 contents = f.read()
1122 f.close()
1123
1124 for (field, (expected, dev)) in range_fields.items():
1125 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1126 if m == None:
1127 print 'Failed to find field: ', field
1128 result = failBecause('no such stats field')
1129 val = int(m.group(1))
1130
1131 lowerBound = trunc( expected * ((100 - float(dev))/100));
1132 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1133
1134 if val < lowerBound:
1135 print field, 'value is too low:'
1136 print '(If this is because you have improved GHC, please'
1137 print 'update the test so that GHC doesn\'t regress again)'
1138 result = failBecause('stat too good')
1139 if val > upperBound:
1140 print field, 'value is too high:'
1141 result = failBecause('stat not good enough')
1142
1143 if val < lowerBound or val > upperBound:
1144 valStr = str(val)
1145 valLen = len(valStr)
1146 expectedStr = str(expected)
1147 expectedLen = len(expectedStr)
1148 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1149 def display(descr, val, extra):
1150 print descr, string.rjust(str(val), length), extra
1151 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1152 display(' Lower bound ' + field + ':', lowerBound, '')
1153 display(' Upper bound ' + field + ':', upperBound, '')
1154 display(' Actual ' + field + ':', val, '')
1155
1156 return result
1157
1158 # -----------------------------------------------------------------------------
1159 # Build a single-module program
1160
1161 def extras_build( way, extra_mods, extra_hc_opts ):
1162 for modopts in extra_mods:
1163 mod, opts = modopts
1164 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1165 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1166 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1167 if badResult(result):
1168 return result
1169
1170 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1171
1172
1173 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1174 opts = getTestOpts()
1175 errname = add_suffix(name, 'comp.stderr')
1176 rm_no_fail( qualify(errname, '') )
1177
1178 if top_mod != '':
1179 srcname = top_mod
1180 rm_no_fail( qualify(name, '') )
1181 base, suf = os.path.splitext(top_mod)
1182 rm_no_fail( qualify(base, '') )
1183 rm_no_fail( qualify(base, 'exe') )
1184 elif addsuf:
1185 srcname = add_hs_lhs_suffix(name)
1186 rm_no_fail( qualify(name, '') )
1187 else:
1188 srcname = name
1189 rm_no_fail( qualify(name, 'o') )
1190
1191 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1192
1193 to_do = ''
1194 if top_mod != '':
1195 to_do = '--make '
1196 if link:
1197 to_do = to_do + '-o ' + name
1198 elif link:
1199 to_do = '-o ' + name
1200 elif opts.compile_to_hc:
1201 to_do = '-C'
1202 else:
1203 to_do = '-c' # just compile
1204
1205 stats_file = name + '.comp.stats'
1206 if len(opts.compiler_stats_range_fields) > 0:
1207 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1208
1209 # Required by GHC 7.3+, harmless for earlier versions:
1210 if (getTestOpts().c_src or
1211 getTestOpts().objc_src or
1212 getTestOpts().objcpp_src or
1213 getTestOpts().cmm_src):
1214 extra_hc_opts += ' -no-hs-main '
1215
1216 if getTestOpts().compile_cmd_prefix == '':
1217 cmd_prefix = ''
1218 else:
1219 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1220
1221 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1222 if noforce:
1223 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1224 if getTestOpts().outputdir != None:
1225 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1226
1227 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1228 + config.compiler + "' " \
1229 + join(comp_flags,' ') + ' ' \
1230 + to_do + ' ' + srcname + ' ' \
1231 + join(config.way_flags(name)[way],' ') + ' ' \
1232 + extra_hc_opts + ' ' \
1233 + opts.extra_hc_opts + ' ' \
1234 + '>' + errname + ' 2>&1'
1235
1236 result = runCmdFor(name, cmd)
1237
1238 if result != 0 and not should_fail:
1239 actual_stderr = qualify(name, 'comp.stderr')
1240 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1241 if_verbose_dump(1,actual_stderr)
1242
1243 # ToDo: if the sub-shell was killed by ^C, then exit
1244
1245 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1246
1247 if badResult(statsResult):
1248 return statsResult
1249
1250 if should_fail:
1251 if result == 0:
1252 return failBecause('exit code 0')
1253 else:
1254 if result != 0:
1255 return failBecause('exit code non-0')
1256
1257 return passed()
1258
1259 # -----------------------------------------------------------------------------
1260 # Run a program and check its output
1261 #
1262 # If testname.stdin exists, route input from that, else
1263 # from /dev/null. Route output to testname.run.stdout and
1264 # testname.run.stderr. Returns the exit code of the run.
1265
1266 def simple_run( name, way, prog, args ):
1267 opts = getTestOpts()
1268
1269 # figure out what to use for stdin
1270 if opts.stdin != '':
1271 use_stdin = opts.stdin
1272 else:
1273 stdin_file = add_suffix(name, 'stdin')
1274 if os.path.exists(in_testdir(stdin_file)):
1275 use_stdin = stdin_file
1276 else:
1277 use_stdin = '/dev/null'
1278
1279 run_stdout = add_suffix(name,'run.stdout')
1280 run_stderr = add_suffix(name,'run.stderr')
1281
1282 rm_no_fail(qualify(name,'run.stdout'))
1283 rm_no_fail(qualify(name,'run.stderr'))
1284 rm_no_fail(qualify(name, 'hp'))
1285 rm_no_fail(qualify(name,'ps'))
1286 rm_no_fail(qualify(name, 'prof'))
1287
1288 my_rts_flags = rts_flags(way)
1289
1290 stats_file = name + '.stats'
1291 if len(opts.stats_range_fields) > 0:
1292 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1293
1294 if opts.no_stdin:
1295 stdin_comes_from = ''
1296 else:
1297 stdin_comes_from = ' <' + use_stdin
1298
1299 if opts.combined_output:
1300 redirection = ' >' + run_stdout \
1301 + ' 2>&1'
1302 else:
1303 redirection = ' >' + run_stdout \
1304 + ' 2>' + run_stderr
1305
1306 cmd = prog + ' ' + args + ' ' \
1307 + my_rts_flags + ' ' \
1308 + stdin_comes_from \
1309 + redirection
1310
1311 if opts.cmd_wrapper != None:
1312 cmd = opts.cmd_wrapper(cmd);
1313
1314 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1315
1316 # run the command
1317 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1318
1319 exit_code = result >> 8
1320 signal = result & 0xff
1321
1322 # check the exit code
1323 if exit_code != opts.exit_code:
1324 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1325 dump_stdout(name)
1326 dump_stderr(name)
1327 return failBecause('bad exit code')
1328
1329 check_hp = my_rts_flags.find("-h") != -1
1330 check_prof = my_rts_flags.find("-p") != -1
1331
1332 if not opts.ignore_output:
1333 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1334 bad_stdout = not check_stdout_ok(name)
1335 if bad_stderr:
1336 return failBecause('bad stderr')
1337 if bad_stdout:
1338 return failBecause('bad stdout')
1339 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1340 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1341 return failBecause('bad heap profile')
1342 if check_prof and not check_prof_ok(name):
1343 return failBecause('bad profile')
1344
1345 return checkStats(stats_file, opts.stats_range_fields)
1346
1347 def rts_flags(way):
1348 if (way == ''):
1349 return ''
1350 else:
1351 args = config.way_rts_flags[way]
1352
1353 if args == []:
1354 return ''
1355 else:
1356 return '+RTS ' + join(args,' ') + ' -RTS'
1357
1358 # -----------------------------------------------------------------------------
1359 # Run a program in the interpreter and check its output
1360
1361 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1362 outname = add_suffix(name, 'interp.stdout')
1363 errname = add_suffix(name, 'interp.stderr')
1364 rm_no_fail(outname)
1365 rm_no_fail(errname)
1366 rm_no_fail(name)
1367
1368 if (top_mod == ''):
1369 srcname = add_hs_lhs_suffix(name)
1370 else:
1371 srcname = top_mod
1372
1373 scriptname = add_suffix(name, 'genscript')
1374 qscriptname = in_testdir(scriptname)
1375 rm_no_fail(qscriptname)
1376
1377 delimiter = '===== program output begins here\n'
1378
1379 script = open(qscriptname, 'w')
1380 if not compile_only:
1381 # set the prog name and command-line args to match the compiled
1382 # environment.
1383 script.write(':set prog ' + name + '\n')
1384 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1385 # Add marker lines to the stdout and stderr output files, so we
1386 # can separate GHCi's output from the program's.
1387 script.write(':! echo ' + delimiter)
1388 script.write(':! echo 1>&2 ' + delimiter)
1389 # Set stdout to be line-buffered to match the compiled environment.
1390 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1391 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1392 # in the event of an exception as for the compiled program.
1393 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1394 script.close()
1395
1396 # figure out what to use for stdin
1397 if getTestOpts().stdin != '':
1398 stdin_file = in_testdir(getTestOpts().stdin)
1399 else:
1400 stdin_file = qualify(name, 'stdin')
1401
1402 if os.path.exists(stdin_file):
1403 stdin = open(stdin_file, 'r')
1404 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1405
1406 script.close()
1407
1408 flags = copy.copy(getTestOpts().compiler_always_flags)
1409 if getTestOpts().outputdir != None:
1410 flags.extend(["-outputdir", getTestOpts().outputdir])
1411
1412 cmd = "'" + config.compiler + "' " \
1413 + join(flags,' ') + ' ' \
1414 + srcname + ' ' \
1415 + join(config.way_flags(name)[way],' ') + ' ' \
1416 + extra_hc_opts + ' ' \
1417 + getTestOpts().extra_hc_opts + ' ' \
1418 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1419
1420 if getTestOpts().cmd_wrapper != None:
1421 cmd = getTestOpts().cmd_wrapper(cmd);
1422
1423 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1424
1425 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1426
1427 exit_code = result >> 8
1428 signal = result & 0xff
1429
1430 # split the stdout into compilation/program output
1431 split_file(in_testdir(outname), delimiter,
1432 qualify(name, 'comp.stdout'),
1433 qualify(name, 'run.stdout'))
1434 split_file(in_testdir(errname), delimiter,
1435 qualify(name, 'comp.stderr'),
1436 qualify(name, 'run.stderr'))
1437
1438 # check the exit code
1439 if exit_code != getTestOpts().exit_code:
1440 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1441 dump_stdout(name)
1442 dump_stderr(name)
1443 return failBecause('bad exit code')
1444
1445 # ToDo: if the sub-shell was killed by ^C, then exit
1446
1447 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1448 check_stdout_ok(name)):
1449 return passed()
1450 else:
1451 return failBecause('bad stdout or stderr')
1452
1453
1454 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1455 infile = open(in_fn)
1456 out1 = open(out1_fn, 'w')
1457 out2 = open(out2_fn, 'w')
1458
1459 line = infile.readline()
1460 line = re.sub('\r', '', line) # ignore Windows EOL
1461 while (re.sub('^\s*','',line) != delimiter and line != ''):
1462 out1.write(line)
1463 line = infile.readline()
1464 line = re.sub('\r', '', line)
1465 out1.close()
1466
1467 line = infile.readline()
1468 while (line != ''):
1469 out2.write(line)
1470 line = infile.readline()
1471 out2.close()
1472
1473 # -----------------------------------------------------------------------------
1474 # Generate External Core for the given program, then compile the resulting Core
1475 # and compare its output to the expected output
1476
1477 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1478
1479 depsfilename = qualify(name, 'deps')
1480 errname = add_suffix(name, 'comp.stderr')
1481 qerrname = qualify(errname,'')
1482
1483 hcname = qualify(name, 'hc')
1484 oname = qualify(name, 'o')
1485
1486 rm_no_fail( qerrname )
1487 rm_no_fail( qualify(name, '') )
1488
1489 if (top_mod == ''):
1490 srcname = add_hs_lhs_suffix(name)
1491 else:
1492 srcname = top_mod
1493
1494 qcorefilename = qualify(name, 'hcr')
1495 corefilename = add_suffix(name, 'hcr')
1496 rm_no_fail(qcorefilename)
1497
1498 # Generate External Core
1499
1500 if (top_mod == ''):
1501 to_do = ' ' + srcname + ' '
1502 else:
1503 to_do = ' --make ' + top_mod + ' '
1504
1505 flags = copy.copy(getTestOpts().compiler_always_flags)
1506 if getTestOpts().outputdir != None:
1507 flags.extend(["-outputdir", getTestOpts().outputdir])
1508 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1509 + config.compiler + "' " \
1510 + join(flags,' ') + ' ' \
1511 + join(config.way_flags(name)[way],' ') + ' ' \
1512 + extra_hc_opts + ' ' \
1513 + getTestOpts().extra_hc_opts \
1514 + to_do \
1515 + '>' + errname + ' 2>&1'
1516 result = runCmdFor(name, cmd)
1517
1518 exit_code = result >> 8
1519
1520 if exit_code != 0:
1521 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1522 if_verbose_dump(1,qerrname)
1523 return failBecause('ext core exit code non-0')
1524
1525 # Compile the resulting files -- if there's more than one module, we need to read the output
1526 # of the previous compilation in order to find the dependencies
1527 if (top_mod == ''):
1528 to_compile = corefilename
1529 else:
1530 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1531 deps = open(depsfilename).read()
1532 deplist = string.replace(deps, '\n',' ');
1533 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1534 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1535
1536 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1537 if getTestOpts().outputdir != None:
1538 flags.extend(["-outputdir", getTestOpts().outputdir])
1539
1540 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1541 + config.compiler + "' " \
1542 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1543 + to_compile + ' ' \
1544 + extra_hc_opts + ' ' \
1545 + getTestOpts().extra_hc_opts + ' ' \
1546 + flags \
1547 + ' -fglasgow-exts -o ' + name \
1548 + '>' + errname + ' 2>&1'
1549
1550 result = runCmdFor(name, cmd)
1551 exit_code = result >> 8
1552
1553 if exit_code != 0:
1554 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1555 if_verbose_dump(1,qerrname)
1556 return failBecause('ext core exit code non-0')
1557
1558 # Clean up
1559 rm_no_fail ( oname )
1560 rm_no_fail ( hcname )
1561 rm_no_fail ( qcorefilename )
1562 rm_no_fail ( depsfilename )
1563
1564 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1565
1566 # -----------------------------------------------------------------------------
1567 # Utils
1568
1569 def check_stdout_ok( name ):
1570 if getTestOpts().with_namebase == None:
1571 namebase = name
1572 else:
1573 namebase = getTestOpts().with_namebase
1574
1575 actual_stdout_file = qualify(name, 'run.stdout')
1576 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1577
1578 def norm(str):
1579 if platform_specific:
1580 return str
1581 else:
1582 return normalise_output(str)
1583
1584 return compare_outputs('stdout', \
1585 two_normalisers(norm, getTestOpts().extra_normaliser), \
1586 expected_stdout_file, actual_stdout_file)
1587
1588 def dump_stdout( name ):
1589 print 'Stdout:'
1590 print read_no_crs(qualify(name, 'run.stdout'))
1591
1592 def check_stderr_ok( name ):
1593 if getTestOpts().with_namebase == None:
1594 namebase = name
1595 else:
1596 namebase = getTestOpts().with_namebase
1597
1598 actual_stderr_file = qualify(name, 'run.stderr')
1599 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1600
1601 def norm(str):
1602 if platform_specific:
1603 return str
1604 else:
1605 return normalise_errmsg(str)
1606
1607 return compare_outputs('stderr', \
1608 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1609 expected_stderr_file, actual_stderr_file)
1610
1611 def dump_stderr( name ):
1612 print "Stderr:"
1613 print read_no_crs(qualify(name, 'run.stderr'))
1614
1615 def read_no_crs(file):
1616 str = ''
1617 try:
1618 h = open(file)
1619 str = h.read()
1620 h.close
1621 except:
1622 # On Windows, if the program fails very early, it seems the
1623 # files stdout/stderr are redirected to may not get created
1624 pass
1625 return re.sub('\r', '', str)
1626
1627 def write_file(file, str):
1628 h = open(file, 'w')
1629 h.write(str)
1630 h.close
1631
1632 def check_hp_ok(name):
1633
1634 # do not qualify for hp2ps because we should be in the right directory
1635 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1636
1637 hp2psResult = runCmdExitCode(hp2psCmd)
1638
1639 actual_ps_file = qualify(name, 'ps')
1640
1641 if(hp2psResult == 0):
1642 if (os.path.exists(actual_ps_file)):
1643 if gs_working:
1644 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1645 if (gsResult == 0):
1646 return (True)
1647 else:
1648 print "hp2ps output for " + name + "is not valid PostScript"
1649 else: return (True) # assume postscript is valid without ghostscript
1650 else:
1651 print "hp2ps did not generate PostScript for " + name
1652 return (False)
1653 else:
1654 print "hp2ps error when processing heap profile for " + name
1655 return(False)
1656
1657 def check_prof_ok(name):
1658
1659 prof_file = qualify(name,'prof')
1660
1661 if not os.path.exists(prof_file):
1662 print prof_file + " does not exist"
1663 return(False)
1664
1665 if os.path.getsize(qualify(name,'prof')) == 0:
1666 print prof_file + " is empty"
1667 return(False)
1668
1669 if getTestOpts().with_namebase == None:
1670 namebase = name
1671 else:
1672 namebase = getTestOpts().with_namebase
1673
1674 (platform_specific, expected_prof_file) = \
1675 platform_wordsize_qualify(namebase, 'prof.sample')
1676
1677 # sample prof file is not required
1678 if not os.path.exists(expected_prof_file):
1679 return True
1680 else:
1681 return compare_outputs('prof', \
1682 two_normalisers(normalise_whitespace,normalise_prof), \
1683 expected_prof_file, prof_file)
1684
1685 # Compare expected output to actual output, and optionally accept the
1686 # new output. Returns true if output matched or was accepted, false
1687 # otherwise.
1688 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1689 if os.path.exists(expected_file):
1690 expected_raw = read_no_crs(expected_file)
1691 # print "norm:", normaliser(expected_raw)
1692 expected_str = normaliser(expected_raw)
1693 expected_file_for_diff = expected_file
1694 else:
1695 expected_str = ''
1696 expected_file_for_diff = '/dev/null'
1697
1698 actual_raw = read_no_crs(actual_file)
1699 actual_str = normaliser(actual_raw)
1700
1701 if expected_str == actual_str:
1702 return 1
1703 else:
1704 print 'Actual ' + kind + ' output differs from expected:'
1705
1706 if expected_file_for_diff == '/dev/null':
1707 expected_normalised_file = '/dev/null'
1708 else:
1709 expected_normalised_file = expected_file + ".normalised"
1710 write_file(expected_normalised_file, expected_str)
1711
1712 actual_normalised_file = actual_file + ".normalised"
1713 write_file(actual_normalised_file, actual_str)
1714
1715 # Ignore whitespace when diffing. We should only get to this
1716 # point if there are non-whitespace differences
1717 #
1718 # Note we are diffing the *actual* output, not the normalised
1719 # output. The normalised output may have whitespace squashed
1720 # (including newlines) so the diff would be hard to read.
1721 # This does mean that the diff might contain changes that
1722 # would be normalised away.
1723 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1724 ' ' + actual_file )
1725
1726 # If for some reason there were no non-whitespace differences,
1727 # then do a full diff
1728 if r == 0:
1729 r = os.system( 'diff -u ' + expected_file_for_diff + \
1730 ' ' + actual_file )
1731
1732 if config.accept:
1733 print 'Accepting new output.'
1734 write_file(expected_file, actual_raw)
1735 return 1
1736 else:
1737 return 0
1738
1739
1740 def normalise_whitespace( str ):
1741 # Merge contiguous whitespace characters into a single space.
1742 str = re.sub('[ \t\n]+', ' ', str)
1743 return str
1744
1745 def normalise_errmsg( str ):
1746 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1747 # the colon is there because it appears in error messages; this
1748 # hacky solution is used in place of more sophisticated filename
1749 # mangling
1750 str = re.sub('([^\\s])\\.exe', '\\1', str)
1751 # normalise slashes, minimise Windows/Unix filename differences
1752 str = re.sub('\\\\', '/', str)
1753 # The inplace ghc's are called ghc-stage[123] to avoid filename
1754 # collisions, so we need to normalise that to just "ghc"
1755 str = re.sub('ghc-stage[123]', 'ghc', str)
1756 # We sometimes see the name of the integer-gmp package on stderr,
1757 # but this can change (either the implementation name or the
1758 # version number), so we canonicalise it here
1759 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1760 return str
1761
1762 # normalise a .prof file, so that we can reasonably compare it against
1763 # a sample. This doesn't compare any of the actual profiling data,
1764 # only the shape of the profile and the number of entries.
1765 def normalise_prof (str):
1766 # strip everything up to the line beginning "COST CENTRE"
1767 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1768
1769 # strip results for CAFs, these tend to change unpredictably
1770 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1771
1772 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1773 # sometimes under MAIN.
1774 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1775
1776 # We have somthing like this:
1777
1778 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1779 # k Main 204 1 0.0 0.0 0.0 0.0
1780 # foo Main 205 1 0.0 0.0 0.0 0.0
1781 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1782
1783 # then we remove all the specific profiling data, leaving only the
1784 # cost centre name, module, and entries, to end up with this:
1785
1786 # MAIN MAIN 0
1787 # k Main 1
1788 # foo Main 1
1789 # foo.bar Main 1
1790
1791 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1792 return str
1793
1794 def normalise_slashes_( str ):
1795 str = re.sub('\\\\', '/', str)
1796 return str
1797
1798 def normalise_exe_( str ):
1799 str = re.sub('\.exe', '', str)
1800 return str
1801
1802 def normalise_output( str ):
1803 # Remove a .exe extension (for Windows)
1804 # This can occur in error messages generated by the program.
1805 str = re.sub('([^\\s])\\.exe', '\\1', str)
1806 return str
1807
1808 def normalise_asm( str ):
1809 lines = str.split('\n')
1810 # Only keep instructions and labels not starting with a dot.
1811 metadata = re.compile('^[ \t]*\\..*$')
1812 out = []
1813 for line in lines:
1814 # Drop metadata directives (e.g. ".type")
1815 if not metadata.match(line):
1816 line = re.sub('@plt', '', line)
1817 instr = line.lstrip().split()
1818 # Drop empty lines.
1819 if not instr:
1820 continue
1821 # Drop operands, except for call instructions.
1822 elif instr[0] == 'call':
1823 out.append(instr[0] + ' ' + instr[1])
1824 else:
1825 out.append(instr[0])
1826 out = '\n'.join(out)
1827 return out
1828
1829 def if_verbose( n, str ):
1830 if config.verbose >= n:
1831 print str
1832
1833 def if_verbose_dump( n, f ):
1834 if config.verbose >= n:
1835 try:
1836 print open(f).read()
1837 except:
1838 print ''
1839
1840 def rawSystem(cmd_and_args):
1841 # We prefer subprocess.call to os.spawnv as the latter
1842 # seems to send its arguments through a shell or something
1843 # with the Windows (non-cygwin) python. An argument "a b c"
1844 # turns into three arguments ["a", "b", "c"].
1845
1846 # However, subprocess is new in python 2.4, so fall back to
1847 # using spawnv if we don't have it
1848
1849 if have_subprocess:
1850 return subprocess.call(cmd_and_args)
1851 else:
1852 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1853
1854 # Note that this doesn't handle the timeout itself; it is just used for
1855 # commands that have timeout handling built-in.
1856 def rawSystemWithTimeout(cmd_and_args):
1857 r = rawSystem(cmd_and_args)
1858 if r == 98:
1859 # The python timeout program uses 98 to signal that ^C was pressed
1860 stopNow()
1861 return r
1862
1863 # cmd is a complex command in Bourne-shell syntax
1864 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1865 # Hence it must ultimately be run by a Bourne shell
1866 #
1867 # Mostly it invokes the command wrapped in 'timeout' thus
1868 # timeout 300 'cd . && ...blah blah'
1869 # so it's timeout's job to invoke the Bourne shell
1870 #
1871 # But watch out for the case when there is no timeout program!
1872 # Then, when using the native Python, os.system will invoke the cmd shell
1873
1874 def runCmd( cmd ):
1875 if_verbose( 1, cmd )
1876 r = 0
1877 if config.os == 'mingw32':
1878 # On MinGW, we will always have timeout
1879 assert config.timeout_prog!=''
1880
1881 if config.timeout_prog != '':
1882 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1883 else:
1884 r = os.system(cmd)
1885 return r << 8
1886
1887 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1888 if_verbose( 1, cmd )
1889 r = 0
1890 if config.os == 'mingw32':
1891 # On MinGW, we will always have timeout
1892 assert config.timeout_prog!=''
1893 timeout = int(ceil(config.timeout * timeout_multiplier))
1894
1895 if config.timeout_prog != '':
1896 if config.check_files_written:
1897 fn = name + ".strace"
1898 r = rawSystemWithTimeout(
1899 ["strace", "-o", fn, "-fF",
1900 "-e", "creat,open,chdir,clone,vfork",
1901 config.timeout_prog, str(timeout), cmd])
1902 addTestFilesWritten(name, fn)
1903 rm_no_fail(fn)
1904 else:
1905 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1906 else:
1907 r = os.system(cmd)
1908 return r << 8
1909
1910 def runCmdExitCode( cmd ):
1911 return (runCmd(cmd) >> 8);
1912
1913
1914 # -----------------------------------------------------------------------------
1915 # checking for files being written to by multiple tests
1916
1917 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1918 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1919 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1920 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1921 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1922 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1923 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1924 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1925 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1926 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1927 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1928 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1929 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1930 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1931 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1932 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1933 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1934
1935 # Files that are read or written but shouldn't be:
1936 # * ghci_history shouldn't be read or written by tests
1937 # * things under package.conf.d shouldn't be written by tests
1938 bad_file_usages = {}
1939
1940 # Mapping from tests to the list of files that they write
1941 files_written = {}
1942
1943 # Mapping from tests to the list of files that they write but don't clean
1944 files_written_not_removed = {}
1945
1946 def add_bad_file_usage(name, file):
1947 try:
1948 if not file in bad_file_usages[name]:
1949 bad_file_usages[name].append(file)
1950 except:
1951 bad_file_usages[name] = [file]
1952
1953 def mkPath(curdir, path):
1954 # Given the current full directory is 'curdir', what is the full
1955 # path to 'path'?
1956 return os.path.realpath(os.path.join(curdir, path))
1957
1958 def addTestFilesWritten(name, fn):
1959 if config.use_threads:
1960 with t.lockFilesWritten:
1961 addTestFilesWrittenHelper(name, fn)
1962 else:
1963 addTestFilesWrittenHelper(name, fn)
1964
1965 def addTestFilesWrittenHelper(name, fn):
1966 started = False
1967 working_directories = {}
1968
1969 with open(fn, 'r') as f:
1970 for line in f:
1971 m_pid = re_strace_pid.match(line)
1972 if m_pid:
1973 pid = m_pid.group(1)
1974 content = m_pid.group(2)
1975 elif re_strace_unavailable.match(line):
1976 next
1977 else:
1978 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1979
1980 m_open = re_strace_open.match(content)
1981 m_chdir = re_strace_chdir.match(content)
1982 m_clone = re_strace_clone.match(content)
1983 m_vfork = re_strace_vfork.match(content)
1984
1985 if not started:
1986 working_directories[pid] = os.getcwd()
1987 started = True
1988
1989 if m_open:
1990 file = m_open.group(1)
1991 file = mkPath(working_directories[pid], file)
1992 if file.endswith("ghci_history"):
1993 add_bad_file_usage(name, file)
1994 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1995 flags = m_open.group(2).split('|')
1996 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1997 if re.match('package\.conf\.d', file):
1998 add_bad_file_usage(name, file)
1999 else:
2000 try:
2001 if not file in files_written[name]:
2002 files_written[name].append(file)
2003 except:
2004 files_written[name] = [file]
2005 elif 'O_RDONLY' in flags:
2006 pass
2007 else:
2008 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2009 elif m_chdir:
2010 # We optimistically assume that unfinished chdir's are going to succeed
2011 dir = m_chdir.group(1)
2012 working_directories[pid] = mkPath(working_directories[pid], dir)
2013 elif m_clone:
2014 working_directories[m_clone.group(2)] = working_directories[pid]
2015 elif m_vfork:
2016 working_directories[m_vfork.group(2)] = working_directories[pid]
2017 elif re_strace_open_resumed.match(content):
2018 pass
2019 elif re_strace_chdir_resumed.match(content):
2020 pass
2021 elif re_strace_vfork_unfinished.match(content):
2022 pass
2023 elif re_strace_clone_unfinished.match(content):
2024 pass
2025 elif re_strace_ignore_sigchild.match(content):
2026 pass
2027 elif re_strace_ignore_sigvtalarm.match(content):
2028 pass
2029 elif re_strace_ignore_sigint.match(content):
2030 pass
2031 elif re_strace_ignore_sigfpe.match(content):
2032 pass
2033 elif re_strace_ignore_sigsegv.match(content):
2034 pass
2035 elif re_strace_ignore_sigpipe.match(content):
2036 pass
2037 else:
2038 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2039
2040 def checkForFilesWrittenProblems(file):
2041 foundProblem = False
2042
2043 files_written_inverted = {}
2044 for t in files_written.keys():
2045 for f in files_written[t]:
2046 try:
2047 files_written_inverted[f].append(t)
2048 except:
2049 files_written_inverted[f] = [t]
2050
2051 for f in files_written_inverted.keys():
2052 if len(files_written_inverted[f]) > 1:
2053 if not foundProblem:
2054 foundProblem = True
2055 file.write("\n")
2056 file.write("\nSome files are written by multiple tests:\n")
2057 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2058 if foundProblem:
2059 file.write("\n")
2060
2061 # -----
2062
2063 if len(files_written_not_removed) > 0:
2064 file.write("\n")
2065 file.write("\nSome files written but not removed:\n")
2066 tests = files_written_not_removed.keys()
2067 tests.sort()
2068 for t in tests:
2069 for f in files_written_not_removed[t]:
2070 file.write(" " + t + ": " + f + "\n")
2071 file.write("\n")
2072
2073 # -----
2074
2075 if len(bad_file_usages) > 0:
2076 file.write("\n")
2077 file.write("\nSome bad file usages:\n")
2078 tests = bad_file_usages.keys()
2079 tests.sort()
2080 for t in tests:
2081 for f in bad_file_usages[t]:
2082 file.write(" " + t + ": " + f + "\n")
2083 file.write("\n")
2084
2085 # -----------------------------------------------------------------------------
2086 # checking if ghostscript is available for checking the output of hp2ps
2087
2088 def genGSCmd(psfile):
2089 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2090
2091 def gsNotWorking():
2092 global gs_working
2093 print "GhostScript not available for hp2ps tests"
2094
2095 global gs_working
2096 gs_working = 0
2097 if config.have_profiling:
2098 if config.gs != '':
2099 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2100 if resultGood == 0:
2101 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2102 if resultBad != 0:
2103 print "GhostScript available for hp2ps tests"
2104 gs_working = 1;
2105 else:
2106 gsNotWorking();
2107 else:
2108 gsNotWorking();
2109 else:
2110 gsNotWorking();
2111
2112 def rm_no_fail( file ):
2113 try:
2114 os.remove( file )
2115 finally:
2116 return
2117
2118 def add_suffix( name, suffix ):
2119 if suffix == '':
2120 return name
2121 else:
2122 return name + '.' + suffix
2123
2124 def add_hs_lhs_suffix(name):
2125 if getTestOpts().c_src:
2126 return add_suffix(name, 'c')
2127 elif getTestOpts().cmm_src:
2128 return add_suffix(name, 'cmm')
2129 elif getTestOpts().objc_src:
2130 return add_suffix(name, 'm')
2131 elif getTestOpts().objcpp_src:
2132 return add_suffix(name, 'mm')
2133 elif getTestOpts().literate:
2134 return add_suffix(name, 'lhs')
2135 else:
2136 return add_suffix(name, 'hs')
2137
2138 def replace_suffix( name, suffix ):
2139 base, suf = os.path.splitext(name)
2140 return base + '.' + suffix
2141
2142 def in_testdir( name ):
2143 return (getTestOpts().testdir + '/' + name)
2144
2145 def qualify( name, suff ):
2146 return in_testdir(add_suffix(name, suff))
2147
2148
2149 # Finding the sample output. The filename is of the form
2150 #
2151 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2152 #
2153 # and we pick the most specific version available. The <version> is
2154 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2155 # more fine-grained control use if_compiler_lt().
2156 #
2157 def platform_wordsize_qualify( name, suff ):
2158
2159 basepath = qualify(name, suff)
2160
2161 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2162 for (platformSpecific, plat) in [(1, '-' + config.platform),
2163 (1, '-' + config.os),
2164 (0, '')]
2165 for ws in ['-ws-' + config.wordsize, '']
2166 for comp in ['-' + config.compiler_type, '']
2167 for vers in ['-' + config.compiler_maj_version, '']]
2168
2169 dir = glob.glob(basepath + '*')
2170 dir = map (lambda d: normalise_slashes_(d), dir)
2171
2172 for (platformSpecific, f) in paths:
2173 if f in dir:
2174 return (platformSpecific,f)
2175
2176 return (0, basepath)
2177
2178 # Clean up prior to the test, so that we can't spuriously conclude
2179 # that it passed on the basis of old run outputs.
2180 def pretest_cleanup(name):
2181 if getTestOpts().outputdir != None:
2182 odir = in_testdir(getTestOpts().outputdir)
2183 try:
2184 shutil.rmtree(odir)
2185 except:
2186 pass
2187 os.mkdir(odir)
2188
2189 rm_no_fail(qualify(name,'comp.stderr'))
2190 rm_no_fail(qualify(name,'run.stderr'))
2191 rm_no_fail(qualify(name,'run.stdout'))
2192 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2193 # simple_build zaps the following:
2194 # rm_nofail(qualify("o"))
2195 # rm_nofail(qualify(""))
2196 # not interested in the return code
2197
2198 # -----------------------------------------------------------------------------
2199 # Return a list of all the files ending in '.T' below the directory dir.
2200
2201 def findTFiles(roots):
2202 return concat(map(findTFiles_,roots))
2203
2204 def findTFiles_(path):
2205 if os.path.isdir(path):
2206 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2207 return findTFiles(paths)
2208 elif path[-2:] == '.T':
2209 return [path]
2210 else:
2211 return []
2212
2213 # -----------------------------------------------------------------------------
2214 # Output a test summary to the specified file object
2215
2216 def summary(t, file):
2217
2218 file.write('\n')
2219 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2220 file.write('OVERALL SUMMARY for test run started at ' \
2221 + t.start_time + '\n'\
2222 + string.rjust(`t.total_tests`, 8) \
2223 + ' total tests, which gave rise to\n' \
2224 + string.rjust(`t.total_test_cases`, 8) \
2225 + ' test cases, of which\n' \
2226 + string.rjust(`t.n_framework_failures`, 8) \
2227 + ' caused framework failures\n' \
2228 + string.rjust(`t.n_tests_skipped`, 8)
2229 + ' were skipped\n\n' \
2230 + string.rjust(`t.n_expected_passes`, 8)
2231 + ' expected passes\n' \
2232 + string.rjust(`t.n_missing_libs`, 8)
2233 + ' had missing libraries\n' \
2234 + string.rjust(`t.n_expected_failures`, 8) \
2235 + ' expected failures\n' \
2236 + string.rjust(`t.n_unexpected_passes`, 8) \
2237 + ' unexpected passes\n'
2238 + string.rjust(`t.n_unexpected_failures`, 8) \
2239 + ' unexpected failures\n'
2240 + '\n')
2241
2242 if t.n_unexpected_passes > 0:
2243 file.write('Unexpected passes:\n')
2244 printPassingTestInfosSummary(file, t.unexpected_passes)
2245
2246 if t.n_unexpected_failures > 0:
2247 file.write('Unexpected failures:\n')
2248 printFailingTestInfosSummary(file, t.unexpected_failures)
2249
2250 if config.check_files_written:
2251 checkForFilesWrittenProblems(file)
2252
2253 if stopping():
2254 file.write('WARNING: Testsuite run was terminated early\n')
2255
2256 def printUnexpectedTests(file, testInfoss):
2257 unexpected = []
2258 for testInfos in testInfoss:
2259 directories = testInfos.keys()
2260 for directory in directories:
2261 tests = testInfos[directory].keys()
2262 unexpected += tests
2263 if unexpected != []:
2264 file.write('Unexpected results from:\n')
2265 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2266 file.write('\n')
2267
2268 def printPassingTestInfosSummary(file, testInfos):
2269 directories = testInfos.keys()
2270 directories.sort()
2271 maxDirLen = max(map ((lambda x : len(x)), directories))
2272 for directory in directories:
2273 tests = testInfos[directory].keys()
2274 tests.sort()
2275 for test in tests:
2276 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2277 ' (' + join(testInfos[directory][test],',') + ')\n')
2278 file.write('\n')
2279
2280 def printFailingTestInfosSummary(file, testInfos):
2281 directories = testInfos.keys()
2282 directories.sort()
2283 maxDirLen = max(map ((lambda x : len(x)), directories))
2284 for directory in directories:
2285 tests = testInfos[directory].keys()
2286 tests.sort()
2287 for test in tests:
2288 reasons = testInfos[directory][test].keys()
2289 for reason in reasons:
2290 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2291 ' [' + reason + ']' + \
2292 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2293 file.write('\n')
2294
2295 def getStdout(cmd):
2296 if have_subprocess:
2297 p = subprocess.Popen(cmd,
2298 stdout=subprocess.PIPE,
2299 stderr=subprocess.PIPE)
2300 (stdout, stderr) = p.communicate()
2301 r = p.wait()
2302 if r != 0:
2303 raise Exception("Command failed: " + str(cmd))
2304 if stderr != '':
2305 raise Exception("stderr from command: " + str(cmd))
2306 return stdout
2307 else:
2308 raise Exception("Need subprocess to get stdout, but don't have it")