Pass the test name to the test options
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 opts.expect = 'fail';
138
139 def ignore_output( name, opts ):
140 opts.ignore_output = 1
141
142 def no_stdin( name, opts ):
143 opts.no_stdin = 1
144
145 def combined_output( name, opts ):
146 opts.combined_output = True
147
148 # -----
149
150 def expect_fail_for( ways ):
151 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
152
153 def _expect_fail_for( name, opts, ways ):
154 opts.expect_fail_for = ways
155
156 def expect_broken_for( bug, ways ):
157 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
158
159 def _expect_broken_for( name, opts, bug, ways ):
160 opts.expect_fail_for = ways
161
162 # -----
163
164 def omit_ways( ways ):
165 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
166
167 def _omit_ways( name, opts, ways ):
168 opts.omit_ways = ways
169
170 # -----
171
172 def only_ways( ways ):
173 return lambda name, opts, w=ways: _only_ways( name, opts, w )
174
175 def _only_ways( name, opts, ways ):
176 opts.only_ways = ways
177
178 # -----
179
180 def extra_ways( ways ):
181 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
182
183 def _extra_ways( name, opts, ways ):
184 opts.extra_ways = ways
185
186 # -----
187
188 def omit_compiler_types( compiler_types ):
189 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
190
191 def _omit_compiler_types( name, opts, compiler_types ):
192 if config.compiler_type in compiler_types:
193 opts.skip = 1
194
195 # -----
196
197 def only_compiler_types( compiler_types ):
198 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
199
200 def _only_compiler_types( name, opts, compiler_types ):
201 if config.compiler_type not in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def set_stdin( file ):
207 return lambda name, opts, f=file: _set_stdin(name, opts, f);
208
209 def _set_stdin( name, opts, f ):
210 opts.stdin = f
211
212 # -----
213
214 def exit_code( val ):
215 return lambda name, opts, v=val: _exit_code(name, opts, v);
216
217 def _exit_code( name, opts, v ):
218 opts.exit_code = v
219
220 # -----
221
222 def timeout_multiplier( val ):
223 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
224
225 def _timeout_multiplier( name, opts, v ):
226 opts.timeout_multiplier = v
227
228 # -----
229
230 def extra_run_opts( val ):
231 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
232
233 def _extra_run_opts( name, opts, v ):
234 opts.extra_run_opts = v
235
236 # -----
237
238 def extra_hc_opts( val ):
239 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
240
241 def _extra_hc_opts( name, opts, v ):
242 opts.extra_hc_opts = v
243
244 # -----
245
246 def extra_clean( files ):
247 return lambda name, opts, v=files: _extra_clean(name, opts, v);
248
249 def _extra_clean( name, opts, v ):
250 opts.clean_files = v
251
252 # -----
253
254 def stats_range_field( field, expected, dev ):
255 return lambda name, opts, f=field, x=expected, y=dev: _stats_range_field(name, opts, f, x, y);
256
257 def _stats_range_field( name, opts, f, x, y ):
258 opts.stats_range_fields[f] = (x, y)
259
260 def compiler_stats_range_field( field, expected, dev ):
261 return lambda name, opts, f=field, x=expected, y=dev: _compiler_stats_range_field(name, opts, f, x, y);
262
263 def _compiler_stats_range_field( name, opts, f, x, y ):
264 opts.compiler_stats_range_fields[f] = (x, y)
265
266 # -----
267
268 def skip_if_no_ghci(name, opts):
269 if not ('ghci' in config.run_ways):
270 opts.skip = 1
271
272 # ----
273
274 def skip_if_fast(name, opts):
275 if config.fast:
276 opts.skip = 1
277
278 # -----
279
280 def when(b, f):
281 if b:
282 return f
283 else:
284 return normal
285
286 def unless(b, f):
287 return when(not b, f)
288
289 def if_platform( plat, f ):
290 if config.platform == plat:
291 return f
292 else:
293 return normal
294
295 def unless_platform( plat, f ):
296 if config.platform != plat:
297 return f
298 else:
299 return normal
300
301 def if_os( os, f ):
302 if config.os == os:
303 return f
304 else:
305 return normal
306
307 def unless_os( os, f ):
308 if config.os == os:
309 return normal
310 else:
311 return f
312
313 def if_arch( arch, f ):
314 if config.arch == arch:
315 return f
316 else:
317 return normal
318
319 def unless_arch( arch, f ):
320 if config.arch == arch:
321 return normal
322 else:
323 return f
324
325 def wordsize( ws ):
326 return config.wordsize == str(ws)
327
328 def if_unregisterised( f ):
329 if config.unregisterised:
330 return f
331 else:
332 return normal
333
334 def unless_unregisterised( f ):
335 if config.unregisterised:
336 return normal
337 else:
338 return f
339
340 def if_msys( f ):
341 if config.msys:
342 return f
343 else:
344 return normal
345
346 def if_cygwin( f ):
347 if config.cygwin:
348 return f
349 else:
350 return normal
351
352 def when_have_vanilla( f ):
353 if config.have_vanilla:
354 return f
355 else:
356 return normal
357
358 def unless_have_vanilla( f ):
359 if config.have_vanilla:
360 return normal
361 else:
362 return f
363
364 def when_have_dynamic( f ):
365 if config.have_dynamic:
366 return f
367 else:
368 return normal
369
370 def unless_have_dynamic( f ):
371 if config.have_dynamic:
372 return normal
373 else:
374 return f
375
376 def when_have_profiling( f ):
377 if config.have_profiling:
378 return f
379 else:
380 return normal
381
382 def unless_have_profiling( f ):
383 if config.have_profiling:
384 return normal
385 else:
386 return f
387
388 # ---
389
390 def if_ghci_dynamic( f ):
391 if config.ghc_dynamic_by_default:
392 return f
393 else:
394 return normal
395
396 def if_in_tree_compiler( f ):
397 if config.in_tree_compiler:
398 return f
399 else:
400 return normal
401
402 def unless_in_tree_compiler( f ):
403 if config.in_tree_compiler:
404 return normal
405 else:
406 return f
407
408 def if_compiler_type( compiler, f ):
409 if config.compiler_type == compiler:
410 return f
411 else:
412 return normal
413
414 def if_compiler_profiled( f ):
415 if config.compiler_profiled:
416 return f
417 else:
418 return normal
419
420 def unless_compiler_profiled( f ):
421 if config.compiler_profiled:
422 return normal
423 else:
424 return f
425
426 def if_compiler_lt( compiler, version, f ):
427 if config.compiler_type == compiler and \
428 version_lt(config.compiler_version, version):
429 return f
430 else:
431 return normal
432
433 def if_compiler_le( compiler, version, f ):
434 if config.compiler_type == compiler and \
435 version_le(config.compiler_version, version):
436 return f
437 else:
438 return normal
439
440 def if_compiler_gt( compiler, version, f ):
441 if config.compiler_type == compiler and \
442 version_gt(config.compiler_version, version):
443 return f
444 else:
445 return normal
446
447 def if_compiler_ge( compiler, version, f ):
448 if config.compiler_type == compiler and \
449 version_ge(config.compiler_version, version):
450 return f
451 else:
452 return normal
453
454 def if_compiler_debugged( f ):
455 if config.compiler_debugged:
456 return f
457 else:
458 return normal
459
460 def namebase( nb ):
461 return lambda opts, nb=nb: _namebase(opts, nb)
462
463 def _namebase( opts, nb ):
464 opts.with_namebase = nb
465
466 # ---
467
468 def if_tag( tag, f ):
469 if tag in config.compiler_tags:
470 return f
471 else:
472 return normal
473
474 def unless_tag( tag, f ):
475 if not (tag in config.compiler_tags):
476 return f
477 else:
478 return normal
479
480 # ---
481 def high_memory_usage(name, opts):
482 opts.alone = True
483
484 # ---
485 def literate( name, opts ):
486 opts.literate = 1;
487
488 def c_src( name, opts ):
489 opts.c_src = 1;
490
491 def objc_src( name, opts ):
492 opts.objc_src = 1;
493
494 def objcpp_src( name, opts ):
495 opts.objcpp_src = 1;
496
497 def cmm_src( name, opts ):
498 opts.cmm_src = 1;
499
500 def outputdir( odir ):
501 return lambda name, opts, d=odir: _outputdir(name, opts, d)
502
503 def _outputdir( name, opts, odir ):
504 opts.outputdir = odir;
505
506 # ----
507
508 def pre_cmd( cmd ):
509 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
510
511 def _pre_cmd( name, opts, cmd ):
512 opts.pre_cmd = cmd
513
514 # ----
515
516 def clean_cmd( cmd ):
517 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
518
519 def _clean_cmd( name, opts, cmd ):
520 opts.clean_cmd = cmd
521
522 # ----
523
524 def cmd_prefix( prefix ):
525 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
526
527 def _cmd_prefix( name, opts, prefix ):
528 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
529
530 # ----
531
532 def cmd_wrapper( fun ):
533 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
534
535 def _cmd_wrapper( name, opts, fun ):
536 opts.cmd_wrapper = fun
537
538 # ----
539
540 def compile_cmd_prefix( prefix ):
541 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
542
543 def _compile_cmd_prefix( name, opts, prefix ):
544 opts.compile_cmd_prefix = prefix
545
546 # ----
547
548 def normalise_slashes( name, opts ):
549 opts.extra_normaliser = normalise_slashes_
550
551 def normalise_exe( name, opts ):
552 opts.extra_normaliser = normalise_exe_
553
554 def normalise_fun( fun ):
555 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
556
557 def _normalise_fun( name, opts, f ):
558 opts.extra_normaliser = f
559
560 def normalise_errmsg_fun( fun ):
561 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
562
563 def _normalise_errmsg_fun( name, opts, f ):
564 opts.extra_errmsg_normaliser = f
565
566 def two_normalisers(f, g):
567 return lambda x, f=f, g=g: f(g(x))
568
569 # ----
570 # Function for composing two opt-fns together
571
572 def composes( fs ):
573 return reduce(lambda f, g: compose(f, g), fs)
574
575 def compose( f, g ):
576 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
577
578 def _compose( name, opts, f, g ):
579 f(name, opts)
580 g(name, opts)
581
582 # -----------------------------------------------------------------------------
583 # The current directory of tests
584
585 def newTestDir( dir ):
586 global thisdir_settings
587 # reset the options for this test directory
588 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
589
590 def _newTestDir( name, opts, dir ):
591 opts.testdir = dir
592 opts.compiler_always_flags = config.compiler_always_flags
593
594 # -----------------------------------------------------------------------------
595 # Actually doing tests
596
597 parallelTests = []
598 aloneTests = []
599 allTestNames = set([])
600
601 def runTest (opts, name, func, args):
602 ok = 0
603
604 if config.use_threads:
605 t.thread_pool.acquire()
606 try:
607 while config.threads<(t.running_threads+1):
608 t.thread_pool.wait()
609 t.running_threads = t.running_threads+1
610 ok=1
611 t.thread_pool.release()
612 thread.start_new_thread(test_common_thread, (name, opts, func, args))
613 except:
614 if not ok:
615 t.thread_pool.release()
616 else:
617 test_common_work (name, opts, func, args)
618
619 # name :: String
620 # setup :: TestOpts -> IO ()
621 def test (name, setup, func, args):
622 global aloneTests
623 global parallelTests
624 global allTestNames
625 global thisdir_settings
626 if name in allTestNames:
627 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
628 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
629 framework_fail(name, 'bad_name', 'This test has an invalid name')
630
631 # Make a deep copy of the default_testopts, as we need our own copy
632 # of any dictionaries etc inside it. Otherwise, if one test modifies
633 # them, all tests will see the modified version!
634 myTestOpts = copy.deepcopy(default_testopts)
635
636 if type(setup) is types.ListType:
637 setup = composes(setup)
638
639 setup = compose(thisdir_settings, setup)
640 setup(name, myTestOpts)
641
642 thisTest = lambda : runTest(myTestOpts, name, func, args)
643 if myTestOpts.alone:
644 aloneTests.append(thisTest)
645 else:
646 parallelTests.append(thisTest)
647 allTestNames.add(name)
648
649 if config.use_threads:
650 def test_common_thread(name, opts, func, args):
651 t.lock.acquire()
652 try:
653 test_common_work(name,opts,func,args)
654 finally:
655 t.lock.release()
656 t.thread_pool.acquire()
657 t.running_threads = t.running_threads - 1
658 t.thread_pool.notify()
659 t.thread_pool.release()
660
661 def get_package_cache_timestamp():
662 if config.package_conf_cache_file == '':
663 return 0.0
664 else:
665 try:
666 return os.stat(config.package_conf_cache_file).st_mtime
667 except:
668 return 0.0
669
670
671 def test_common_work (name, opts, func, args):
672 try:
673 t.total_tests = t.total_tests+1
674 setLocalTestOpts(opts)
675
676 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
677
678 # All the ways we might run this test
679 if func == compile or func == multimod_compile:
680 all_ways = config.compile_ways
681 elif func == compile_and_run or func == multimod_compile_and_run:
682 all_ways = config.run_ways
683 elif func == ghci_script:
684 if 'ghci' in config.run_ways:
685 all_ways = ['ghci']
686 else:
687 all_ways = []
688 else:
689 all_ways = ['normal']
690
691 # A test itself can request extra ways by setting opts.extra_ways
692 all_ways = all_ways + filter(lambda way: way not in all_ways,
693 opts.extra_ways)
694
695 t.total_test_cases = t.total_test_cases + len(all_ways)
696
697 ok_way = lambda way: \
698 not getTestOpts().skip \
699 and (config.only == [] or name in config.only) \
700 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
701 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
702 and way not in getTestOpts().omit_ways
703
704 # Which ways we are asked to skip
705 do_ways = filter (ok_way,all_ways)
706
707 # In fast mode, we skip all but one way
708 if config.fast and len(do_ways) > 0:
709 do_ways = [do_ways[0]]
710
711 if not config.clean_only:
712 # Run the required tests...
713 for way in do_ways:
714 if stopping():
715 break
716 do_test (name, way, func, args)
717
718 for way in all_ways:
719 if way not in do_ways:
720 skiptest (name,way)
721
722 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
723 clean(map (lambda suff: name + suff,
724 ['', '.exe', '.exe.manifest', '.genscript',
725 '.stderr.normalised', '.stdout.normalised',
726 '.run.stderr', '.run.stdout',
727 '.run.stderr.normalised', '.run.stdout.normalised',
728 '.comp.stderr', '.comp.stdout',
729 '.comp.stderr.normalised', '.comp.stdout.normalised',
730 '.interp.stderr', '.interp.stdout',
731 '.interp.stderr.normalised', '.interp.stdout.normalised',
732 '.stats', '.comp.stats',
733 '.hi', '.o', '.prof', '.exe.prof', '.hc',
734 '_stub.h', '_stub.c', '_stub.o',
735 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
736
737 if func == multi_compile or func == multi_compile_fail:
738 extra_mods = args[1]
739 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
740 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
741
742 clean(getTestOpts().clean_files)
743
744 if getTestOpts().outputdir != None:
745 odir = in_testdir(getTestOpts().outputdir)
746 try:
747 shutil.rmtree(odir)
748 except:
749 pass
750
751 try:
752 shutil.rmtree(in_testdir('.hpc.' + name))
753 except:
754 pass
755
756 try:
757 cleanCmd = getTestOpts().clean_cmd
758 if cleanCmd != None:
759 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
760 if result != 0:
761 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
762 except e:
763 framework_fail(name, 'cleaning', 'clean-command exception')
764
765 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
766
767 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
768 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
769
770 try:
771 for f in files_written[name]:
772 if os.path.exists(f):
773 try:
774 if not f in files_written_not_removed[name]:
775 files_written_not_removed[name].append(f)
776 except:
777 files_written_not_removed[name] = [f]
778 except:
779 pass
780 except Exception, e:
781 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
782
783 def clean(strs):
784 for str in strs:
785 for name in glob.glob(in_testdir(str)):
786 clean_full_path(name)
787
788 def clean_full_path(name):
789 try:
790 # Remove files...
791 os.remove(name)
792 except OSError, e1:
793 try:
794 # ... and empty directories
795 os.rmdir(name)
796 except OSError, e2:
797 # We don't want to fail here, but we do want to know
798 # what went wrong, so print out the exceptions.
799 # ENOENT isn't a problem, though, as we clean files
800 # that don't necessarily exist.
801 if e1.errno != errno.ENOENT:
802 print e1
803 if e2.errno != errno.ENOENT:
804 print e2
805
806 def do_test(name, way, func, args):
807 full_name = name + '(' + way + ')'
808
809 try:
810 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
811 str([t.n_unexpected_passes, \
812 t.n_unexpected_failures, \
813 t.n_framework_failures])
814
815 if config.use_threads:
816 t.lock.release()
817
818 try:
819 preCmd = getTestOpts().pre_cmd
820 if preCmd != None:
821 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
822 if result != 0:
823 framework_fail(name, way, 'pre-command failed: ' + str(result))
824 except e:
825 framework_fail(name, way, 'pre-command exception')
826
827 try:
828 result = apply(func, [name,way] + args)
829 finally:
830 if config.use_threads:
831 t.lock.acquire()
832
833 if getTestOpts().expect != 'pass' and \
834 getTestOpts().expect != 'fail' and \
835 getTestOpts().expect != 'missing-lib':
836 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
837
838 try:
839 passFail = result['passFail']
840 except:
841 passFail = 'No passFail found'
842
843 if passFail == 'pass':
844 if getTestOpts().expect == 'pass' \
845 and way not in getTestOpts().expect_fail_for:
846 t.n_expected_passes = t.n_expected_passes + 1
847 if name in t.expected_passes:
848 t.expected_passes[name].append(way)
849 else:
850 t.expected_passes[name] = [way]
851 else:
852 print '*** unexpected pass for', full_name
853 t.n_unexpected_passes = t.n_unexpected_passes + 1
854 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
855 elif passFail == 'fail':
856 if getTestOpts().expect == 'pass' \
857 and way not in getTestOpts().expect_fail_for:
858 print '*** unexpected failure for', full_name
859 t.n_unexpected_failures = t.n_unexpected_failures + 1
860 reason = result['reason']
861 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
862 else:
863 if getTestOpts().expect == 'missing-lib':
864 t.n_missing_libs = t.n_missing_libs + 1
865 if name in t.missing_libs:
866 t.missing_libs[name].append(way)
867 else:
868 t.missing_libs[name] = [way]
869 else:
870 t.n_expected_failures = t.n_expected_failures + 1
871 if name in t.expected_failures:
872 t.expected_failures[name].append(way)
873 else:
874 t.expected_failures[name] = [way]
875 else:
876 framework_fail(name, way, 'bad result ' + passFail)
877 except KeyboardInterrupt:
878 stopNow()
879 except:
880 framework_fail(name, way, 'do_test exception')
881 traceback.print_exc()
882
883 def addPassingTestInfo (testInfos, directory, name, way):
884 directory = re.sub('^\\.[/\\\\]', '', directory)
885
886 if not directory in testInfos:
887 testInfos[directory] = {}
888
889 if not name in testInfos[directory]:
890 testInfos[directory][name] = []
891
892 testInfos[directory][name].append(way)
893
894 def addFailingTestInfo (testInfos, directory, name, reason, way):
895 directory = re.sub('^\\.[/\\\\]', '', directory)
896
897 if not directory in testInfos:
898 testInfos[directory] = {}
899
900 if not name in testInfos[directory]:
901 testInfos[directory][name] = {}
902
903 if not reason in testInfos[directory][name]:
904 testInfos[directory][name][reason] = []
905
906 testInfos[directory][name][reason].append(way)
907
908 def skiptest (name, way):
909 # print 'Skipping test \"', name, '\"'
910 t.n_tests_skipped = t.n_tests_skipped + 1
911 if name in t.tests_skipped:
912 t.tests_skipped[name].append(way)
913 else:
914 t.tests_skipped[name] = [way]
915
916 def framework_fail( name, way, reason ):
917 full_name = name + '(' + way + ')'
918 print '*** framework failure for', full_name, reason
919 t.n_framework_failures = t.n_framework_failures + 1
920 if name in t.framework_failures:
921 t.framework_failures[name].append(way)
922 else:
923 t.framework_failures[name] = [way]
924
925 def badResult(result):
926 try:
927 if result['passFail'] == 'pass':
928 return False
929 return True
930 except:
931 return True
932
933 def passed():
934 return {'passFail': 'pass'}
935
936 def failBecause(reason):
937 return {'passFail': 'fail', 'reason': reason}
938
939 # -----------------------------------------------------------------------------
940 # Generic command tests
941
942 # A generic command test is expected to run and exit successfully.
943 #
944 # The expected exit code can be changed via exit_code() as normal, and
945 # the expected stdout/stderr are stored in <testname>.stdout and
946 # <testname>.stderr. The output of the command can be ignored
947 # altogether by using run_command_ignore_output instead of
948 # run_command.
949
950 def run_command( name, way, cmd ):
951 return simple_run( name, '', cmd, '' )
952
953 # -----------------------------------------------------------------------------
954 # GHCi tests
955
956 def ghci_script( name, way, script ):
957 # filter out -fforce-recomp from compiler_always_flags, because we're
958 # actually testing the recompilation behaviour in the GHCi tests.
959 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
960 flags.append(getTestOpts().extra_hc_opts)
961 if getTestOpts().outputdir != None:
962 flags.extend(["-outputdir", getTestOpts().outputdir])
963
964 # We pass HC and HC_OPTS as environment variables, so that the
965 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
966 cmd = "HC='" + config.compiler + "' " + \
967 "HC_OPTS='" + join(flags,' ') + "' " + \
968 "'" + config.compiler + "'" + \
969 ' --interactive -v0 -ignore-dot-ghci ' + \
970 join(flags,' ')
971
972 getTestOpts().stdin = script
973 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
974
975 # -----------------------------------------------------------------------------
976 # Compile-only tests
977
978 def compile( name, way, extra_hc_opts ):
979 return do_compile( name, way, 0, '', [], extra_hc_opts )
980
981 def compile_fail( name, way, extra_hc_opts ):
982 return do_compile( name, way, 1, '', [], extra_hc_opts )
983
984 def multimod_compile( name, way, top_mod, extra_hc_opts ):
985 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
986
987 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
988 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
989
990 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
991 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
992
993 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
994 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
995
996 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
997 # print 'Compile only, extra args = ', extra_hc_opts
998 pretest_cleanup(name)
999
1000 result = extras_build( way, extra_mods, extra_hc_opts )
1001 if badResult(result):
1002 return result
1003 extra_hc_opts = result['hc_opts']
1004
1005 force = 0
1006 if extra_mods:
1007 force = 1
1008 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1009
1010 if badResult(result):
1011 return result
1012
1013 # the actual stderr should always match the expected, regardless
1014 # of whether we expected the compilation to fail or not (successful
1015 # compilations may generate warnings).
1016
1017 if getTestOpts().with_namebase == None:
1018 namebase = name
1019 else:
1020 namebase = getTestOpts().with_namebase
1021
1022 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1023 actual_stderr_file = qualify(name, 'comp.stderr')
1024
1025 if not compare_outputs('stderr', \
1026 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1027 expected_stderr_file, actual_stderr_file):
1028 return failBecause('stderr mismatch')
1029
1030 # no problems found, this test passed
1031 return passed()
1032
1033 def compile_cmp_asm( name, way, extra_hc_opts ):
1034 print 'Compile only, extra args = ', extra_hc_opts
1035 pretest_cleanup(name)
1036 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1037
1038 if badResult(result):
1039 return result
1040
1041 # the actual stderr should always match the expected, regardless
1042 # of whether we expected the compilation to fail or not (successful
1043 # compilations may generate warnings).
1044
1045 if getTestOpts().with_namebase == None:
1046 namebase = name
1047 else:
1048 namebase = getTestOpts().with_namebase
1049
1050 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1051 actual_asm_file = qualify(name, 's')
1052
1053 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1054 expected_asm_file, actual_asm_file):
1055 return failBecause('asm mismatch')
1056
1057 # no problems found, this test passed
1058 return passed()
1059
1060 # -----------------------------------------------------------------------------
1061 # Compile-and-run tests
1062
1063 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1064 # print 'Compile and run, extra args = ', extra_hc_opts
1065 pretest_cleanup(name)
1066
1067 result = extras_build( way, extra_mods, extra_hc_opts )
1068 if badResult(result):
1069 return result
1070 extra_hc_opts = result['hc_opts']
1071
1072 if way == 'ghci': # interpreted...
1073 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1074 elif way == 'extcore' or way == 'optextcore' :
1075 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1076 else: # compiled...
1077 force = 0
1078 if extra_mods:
1079 force = 1
1080
1081 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1082 if badResult(result):
1083 return result
1084
1085 cmd = './' + name;
1086
1087 # we don't check the compiler's stderr for a compile-and-run test
1088 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1089
1090 def compile_and_run( name, way, extra_hc_opts ):
1091 return compile_and_run__( name, way, '', [], extra_hc_opts)
1092
1093 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1094 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1095
1096 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1097 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1098
1099 def stats( name, way, stats_file ):
1100 opts = getTestOpts()
1101 return checkStats(stats_file, opts.stats_range_fields)
1102
1103 # -----------------------------------------------------------------------------
1104 # Check -t stats info
1105
1106 def checkStats(stats_file, range_fields):
1107 result = passed()
1108 if len(range_fields) > 0:
1109 f = open(in_testdir(stats_file))
1110 contents = f.read()
1111 f.close()
1112
1113 for (field, (expected, dev)) in range_fields.items():
1114 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1115 if m == None:
1116 print 'Failed to find field: ', field
1117 result = failBecause('no such stats field')
1118 val = int(m.group(1))
1119
1120 lowerBound = trunc( expected * ((100 - float(dev))/100));
1121 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1122
1123 if val < lowerBound:
1124 print field, 'value is too low:'
1125 print '(If this is because you have improved GHC, please'
1126 print 'update the test so that GHC doesn\'t regress again)'
1127 result = failBecause('stat too good')
1128 if val > upperBound:
1129 print field, 'value is too high:'
1130 result = failBecause('stat not good enough')
1131
1132 if val < lowerBound or val > upperBound:
1133 valStr = str(val)
1134 valLen = len(valStr)
1135 expectedStr = str(expected)
1136 expectedLen = len(expectedStr)
1137 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1138 def display(descr, val, extra):
1139 print descr, string.rjust(str(val), length), extra
1140 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1141 display(' Lower bound ' + field + ':', lowerBound, '')
1142 display(' Upper bound ' + field + ':', upperBound, '')
1143 display(' Actual ' + field + ':', val, '')
1144
1145 return result
1146
1147 # -----------------------------------------------------------------------------
1148 # Build a single-module program
1149
1150 def extras_build( way, extra_mods, extra_hc_opts ):
1151 for modopts in extra_mods:
1152 mod, opts = modopts
1153 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1154 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1155 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1156 if badResult(result):
1157 return result
1158
1159 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1160
1161
1162 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1163 opts = getTestOpts()
1164 errname = add_suffix(name, 'comp.stderr')
1165 rm_no_fail( qualify(errname, '') )
1166
1167 if top_mod != '':
1168 srcname = top_mod
1169 rm_no_fail( qualify(name, '') )
1170 base, suf = os.path.splitext(top_mod)
1171 rm_no_fail( qualify(base, '') )
1172 rm_no_fail( qualify(base, 'exe') )
1173 elif addsuf:
1174 srcname = add_hs_lhs_suffix(name)
1175 rm_no_fail( qualify(name, '') )
1176 else:
1177 srcname = name
1178 rm_no_fail( qualify(name, 'o') )
1179
1180 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1181
1182 to_do = ''
1183 if top_mod != '':
1184 to_do = '--make '
1185 if link:
1186 to_do = to_do + '-o ' + name
1187 elif link:
1188 to_do = '-o ' + name
1189 elif opts.compile_to_hc:
1190 to_do = '-C'
1191 else:
1192 to_do = '-c' # just compile
1193
1194 stats_file = name + '.comp.stats'
1195 if len(opts.compiler_stats_range_fields) > 0:
1196 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1197
1198 # Required by GHC 7.3+, harmless for earlier versions:
1199 if (getTestOpts().c_src or
1200 getTestOpts().objc_src or
1201 getTestOpts().objcpp_src or
1202 getTestOpts().cmm_src):
1203 extra_hc_opts += ' -no-hs-main '
1204
1205 if getTestOpts().compile_cmd_prefix == '':
1206 cmd_prefix = ''
1207 else:
1208 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1209
1210 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1211 if noforce:
1212 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1213 if getTestOpts().outputdir != None:
1214 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1215
1216 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1217 + config.compiler + "' " \
1218 + join(comp_flags,' ') + ' ' \
1219 + to_do + ' ' + srcname + ' ' \
1220 + join(config.way_flags(name)[way],' ') + ' ' \
1221 + extra_hc_opts + ' ' \
1222 + opts.extra_hc_opts + ' ' \
1223 + '>' + errname + ' 2>&1'
1224
1225 result = runCmdFor(name, cmd)
1226
1227 if result != 0 and not should_fail:
1228 actual_stderr = qualify(name, 'comp.stderr')
1229 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1230 if_verbose_dump(1,actual_stderr)
1231
1232 # ToDo: if the sub-shell was killed by ^C, then exit
1233
1234 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1235
1236 if badResult(statsResult):
1237 return statsResult
1238
1239 if should_fail:
1240 if result == 0:
1241 return failBecause('exit code 0')
1242 else:
1243 if result != 0:
1244 return failBecause('exit code non-0')
1245
1246 return passed()
1247
1248 # -----------------------------------------------------------------------------
1249 # Run a program and check its output
1250 #
1251 # If testname.stdin exists, route input from that, else
1252 # from /dev/null. Route output to testname.run.stdout and
1253 # testname.run.stderr. Returns the exit code of the run.
1254
1255 def simple_run( name, way, prog, args ):
1256 opts = getTestOpts()
1257
1258 # figure out what to use for stdin
1259 if opts.stdin != '':
1260 use_stdin = opts.stdin
1261 else:
1262 stdin_file = add_suffix(name, 'stdin')
1263 if os.path.exists(in_testdir(stdin_file)):
1264 use_stdin = stdin_file
1265 else:
1266 use_stdin = '/dev/null'
1267
1268 run_stdout = add_suffix(name,'run.stdout')
1269 run_stderr = add_suffix(name,'run.stderr')
1270
1271 rm_no_fail(qualify(name,'run.stdout'))
1272 rm_no_fail(qualify(name,'run.stderr'))
1273 rm_no_fail(qualify(name, 'hp'))
1274 rm_no_fail(qualify(name,'ps'))
1275 rm_no_fail(qualify(name, 'prof'))
1276
1277 my_rts_flags = rts_flags(way)
1278
1279 stats_file = name + '.stats'
1280 if len(opts.stats_range_fields) > 0:
1281 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1282
1283 if opts.no_stdin:
1284 stdin_comes_from = ''
1285 else:
1286 stdin_comes_from = ' <' + use_stdin
1287
1288 if opts.combined_output:
1289 redirection = ' >' + run_stdout \
1290 + ' 2>&1'
1291 else:
1292 redirection = ' >' + run_stdout \
1293 + ' 2>' + run_stderr
1294
1295 cmd = prog + ' ' + args + ' ' \
1296 + my_rts_flags + ' ' \
1297 + stdin_comes_from \
1298 + redirection
1299
1300 if opts.cmd_wrapper != None:
1301 cmd = opts.cmd_wrapper(cmd);
1302
1303 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1304
1305 # run the command
1306 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1307
1308 exit_code = result >> 8
1309 signal = result & 0xff
1310
1311 # check the exit code
1312 if exit_code != opts.exit_code:
1313 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1314 dump_stdout(name)
1315 dump_stderr(name)
1316 return failBecause('bad exit code')
1317
1318 check_hp = my_rts_flags.find("-h") != -1
1319 check_prof = my_rts_flags.find("-p") != -1
1320
1321 if not opts.ignore_output:
1322 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1323 bad_stdout = not check_stdout_ok(name)
1324 if bad_stderr:
1325 return failBecause('bad stderr')
1326 if bad_stdout:
1327 return failBecause('bad stdout')
1328 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1329 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1330 return failBecause('bad heap profile')
1331 if check_prof and not check_prof_ok(name):
1332 return failBecause('bad profile')
1333
1334 return checkStats(stats_file, opts.stats_range_fields)
1335
1336 def rts_flags(way):
1337 if (way == ''):
1338 return ''
1339 else:
1340 args = config.way_rts_flags[way]
1341
1342 if args == []:
1343 return ''
1344 else:
1345 return '+RTS ' + join(args,' ') + ' -RTS'
1346
1347 # -----------------------------------------------------------------------------
1348 # Run a program in the interpreter and check its output
1349
1350 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1351 outname = add_suffix(name, 'interp.stdout')
1352 errname = add_suffix(name, 'interp.stderr')
1353 rm_no_fail(outname)
1354 rm_no_fail(errname)
1355 rm_no_fail(name)
1356
1357 if (top_mod == ''):
1358 srcname = add_hs_lhs_suffix(name)
1359 else:
1360 srcname = top_mod
1361
1362 scriptname = add_suffix(name, 'genscript')
1363 qscriptname = in_testdir(scriptname)
1364 rm_no_fail(qscriptname)
1365
1366 delimiter = '===== program output begins here\n'
1367
1368 script = open(qscriptname, 'w')
1369 if not compile_only:
1370 # set the prog name and command-line args to match the compiled
1371 # environment.
1372 script.write(':set prog ' + name + '\n')
1373 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1374 # Add marker lines to the stdout and stderr output files, so we
1375 # can separate GHCi's output from the program's.
1376 script.write(':! echo ' + delimiter)
1377 script.write(':! echo 1>&2 ' + delimiter)
1378 # Set stdout to be line-buffered to match the compiled environment.
1379 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1380 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1381 # in the event of an exception as for the compiled program.
1382 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1383 script.close()
1384
1385 # figure out what to use for stdin
1386 if getTestOpts().stdin != '':
1387 stdin_file = in_testdir(getTestOpts().stdin)
1388 else:
1389 stdin_file = qualify(name, 'stdin')
1390
1391 if os.path.exists(stdin_file):
1392 stdin = open(stdin_file, 'r')
1393 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1394
1395 script.close()
1396
1397 flags = copy.copy(getTestOpts().compiler_always_flags)
1398 if getTestOpts().outputdir != None:
1399 flags.extend(["-outputdir", getTestOpts().outputdir])
1400
1401 cmd = "'" + config.compiler + "' " \
1402 + join(flags,' ') + ' ' \
1403 + srcname + ' ' \
1404 + join(config.way_flags(name)[way],' ') + ' ' \
1405 + extra_hc_opts + ' ' \
1406 + getTestOpts().extra_hc_opts + ' ' \
1407 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1408
1409 if getTestOpts().cmd_wrapper != None:
1410 cmd = getTestOpts().cmd_wrapper(cmd);
1411
1412 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1413
1414 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1415
1416 exit_code = result >> 8
1417 signal = result & 0xff
1418
1419 # split the stdout into compilation/program output
1420 split_file(in_testdir(outname), delimiter,
1421 qualify(name, 'comp.stdout'),
1422 qualify(name, 'run.stdout'))
1423 split_file(in_testdir(errname), delimiter,
1424 qualify(name, 'comp.stderr'),
1425 qualify(name, 'run.stderr'))
1426
1427 # check the exit code
1428 if exit_code != getTestOpts().exit_code:
1429 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1430 dump_stdout(name)
1431 dump_stderr(name)
1432 return failBecause('bad exit code')
1433
1434 # ToDo: if the sub-shell was killed by ^C, then exit
1435
1436 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1437 check_stdout_ok(name)):
1438 return passed()
1439 else:
1440 return failBecause('bad stdout or stderr')
1441
1442
1443 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1444 infile = open(in_fn)
1445 out1 = open(out1_fn, 'w')
1446 out2 = open(out2_fn, 'w')
1447
1448 line = infile.readline()
1449 line = re.sub('\r', '', line) # ignore Windows EOL
1450 while (re.sub('^\s*','',line) != delimiter and line != ''):
1451 out1.write(line)
1452 line = infile.readline()
1453 line = re.sub('\r', '', line)
1454 out1.close()
1455
1456 line = infile.readline()
1457 while (line != ''):
1458 out2.write(line)
1459 line = infile.readline()
1460 out2.close()
1461
1462 # -----------------------------------------------------------------------------
1463 # Generate External Core for the given program, then compile the resulting Core
1464 # and compare its output to the expected output
1465
1466 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1467
1468 depsfilename = qualify(name, 'deps')
1469 errname = add_suffix(name, 'comp.stderr')
1470 qerrname = qualify(errname,'')
1471
1472 hcname = qualify(name, 'hc')
1473 oname = qualify(name, 'o')
1474
1475 rm_no_fail( qerrname )
1476 rm_no_fail( qualify(name, '') )
1477
1478 if (top_mod == ''):
1479 srcname = add_hs_lhs_suffix(name)
1480 else:
1481 srcname = top_mod
1482
1483 qcorefilename = qualify(name, 'hcr')
1484 corefilename = add_suffix(name, 'hcr')
1485 rm_no_fail(qcorefilename)
1486
1487 # Generate External Core
1488
1489 if (top_mod == ''):
1490 to_do = ' ' + srcname + ' '
1491 else:
1492 to_do = ' --make ' + top_mod + ' '
1493
1494 flags = copy.copy(getTestOpts().compiler_always_flags)
1495 if getTestOpts().outputdir != None:
1496 flags.extend(["-outputdir", getTestOpts().outputdir])
1497 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1498 + config.compiler + "' " \
1499 + join(flags,' ') + ' ' \
1500 + join(config.way_flags(name)[way],' ') + ' ' \
1501 + extra_hc_opts + ' ' \
1502 + getTestOpts().extra_hc_opts \
1503 + to_do \
1504 + '>' + errname + ' 2>&1'
1505 result = runCmdFor(name, cmd)
1506
1507 exit_code = result >> 8
1508
1509 if exit_code != 0:
1510 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1511 if_verbose_dump(1,qerrname)
1512 return failBecause('ext core exit code non-0')
1513
1514 # Compile the resulting files -- if there's more than one module, we need to read the output
1515 # of the previous compilation in order to find the dependencies
1516 if (top_mod == ''):
1517 to_compile = corefilename
1518 else:
1519 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1520 deps = open(depsfilename).read()
1521 deplist = string.replace(deps, '\n',' ');
1522 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1523 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1524
1525 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1526 if getTestOpts().outputdir != None:
1527 flags.extend(["-outputdir", getTestOpts().outputdir])
1528
1529 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1530 + config.compiler + "' " \
1531 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1532 + to_compile + ' ' \
1533 + extra_hc_opts + ' ' \
1534 + getTestOpts().extra_hc_opts + ' ' \
1535 + flags \
1536 + ' -fglasgow-exts -o ' + name \
1537 + '>' + errname + ' 2>&1'
1538
1539 result = runCmdFor(name, cmd)
1540 exit_code = result >> 8
1541
1542 if exit_code != 0:
1543 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1544 if_verbose_dump(1,qerrname)
1545 return failBecause('ext core exit code non-0')
1546
1547 # Clean up
1548 rm_no_fail ( oname )
1549 rm_no_fail ( hcname )
1550 rm_no_fail ( qcorefilename )
1551 rm_no_fail ( depsfilename )
1552
1553 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1554
1555 # -----------------------------------------------------------------------------
1556 # Utils
1557
1558 def check_stdout_ok( name ):
1559 if getTestOpts().with_namebase == None:
1560 namebase = name
1561 else:
1562 namebase = getTestOpts().with_namebase
1563
1564 actual_stdout_file = qualify(name, 'run.stdout')
1565 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1566
1567 def norm(str):
1568 if platform_specific:
1569 return str
1570 else:
1571 return normalise_output(str)
1572
1573 return compare_outputs('stdout', \
1574 two_normalisers(norm, getTestOpts().extra_normaliser), \
1575 expected_stdout_file, actual_stdout_file)
1576
1577 def dump_stdout( name ):
1578 print 'Stdout:'
1579 print read_no_crs(qualify(name, 'run.stdout'))
1580
1581 def check_stderr_ok( name ):
1582 if getTestOpts().with_namebase == None:
1583 namebase = name
1584 else:
1585 namebase = getTestOpts().with_namebase
1586
1587 actual_stderr_file = qualify(name, 'run.stderr')
1588 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1589
1590 def norm(str):
1591 if platform_specific:
1592 return str
1593 else:
1594 return normalise_errmsg(str)
1595
1596 return compare_outputs('stderr', \
1597 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1598 expected_stderr_file, actual_stderr_file)
1599
1600 def dump_stderr( name ):
1601 print "Stderr:"
1602 print read_no_crs(qualify(name, 'run.stderr'))
1603
1604 def read_no_crs(file):
1605 str = ''
1606 try:
1607 h = open(file)
1608 str = h.read()
1609 h.close
1610 except:
1611 # On Windows, if the program fails very early, it seems the
1612 # files stdout/stderr are redirected to may not get created
1613 pass
1614 return re.sub('\r', '', str)
1615
1616 def write_file(file, str):
1617 h = open(file, 'w')
1618 h.write(str)
1619 h.close
1620
1621 def check_hp_ok(name):
1622
1623 # do not qualify for hp2ps because we should be in the right directory
1624 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1625
1626 hp2psResult = runCmdExitCode(hp2psCmd)
1627
1628 actual_ps_file = qualify(name, 'ps')
1629
1630 if(hp2psResult == 0):
1631 if (os.path.exists(actual_ps_file)):
1632 if gs_working:
1633 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1634 if (gsResult == 0):
1635 return (True)
1636 else:
1637 print "hp2ps output for " + name + "is not valid PostScript"
1638 else: return (True) # assume postscript is valid without ghostscript
1639 else:
1640 print "hp2ps did not generate PostScript for " + name
1641 return (False)
1642 else:
1643 print "hp2ps error when processing heap profile for " + name
1644 return(False)
1645
1646 def check_prof_ok(name):
1647
1648 prof_file = qualify(name,'prof')
1649
1650 if not os.path.exists(prof_file):
1651 print prof_file + " does not exist"
1652 return(False)
1653
1654 if os.path.getsize(qualify(name,'prof')) == 0:
1655 print prof_file + " is empty"
1656 return(False)
1657
1658 if getTestOpts().with_namebase == None:
1659 namebase = name
1660 else:
1661 namebase = getTestOpts().with_namebase
1662
1663 (platform_specific, expected_prof_file) = \
1664 platform_wordsize_qualify(namebase, 'prof.sample')
1665
1666 # sample prof file is not required
1667 if not os.path.exists(expected_prof_file):
1668 return True
1669 else:
1670 return compare_outputs('prof', \
1671 two_normalisers(normalise_whitespace,normalise_prof), \
1672 expected_prof_file, prof_file)
1673
1674 # Compare expected output to actual output, and optionally accept the
1675 # new output. Returns true if output matched or was accepted, false
1676 # otherwise.
1677 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1678 if os.path.exists(expected_file):
1679 expected_raw = read_no_crs(expected_file)
1680 # print "norm:", normaliser(expected_raw)
1681 expected_str = normaliser(expected_raw)
1682 expected_file_for_diff = expected_file
1683 else:
1684 expected_str = ''
1685 expected_file_for_diff = '/dev/null'
1686
1687 actual_raw = read_no_crs(actual_file)
1688 actual_str = normaliser(actual_raw)
1689
1690 if expected_str == actual_str:
1691 return 1
1692 else:
1693 print 'Actual ' + kind + ' output differs from expected:'
1694
1695 if expected_file_for_diff == '/dev/null':
1696 expected_normalised_file = '/dev/null'
1697 else:
1698 expected_normalised_file = expected_file + ".normalised"
1699 write_file(expected_normalised_file, expected_str)
1700
1701 actual_normalised_file = actual_file + ".normalised"
1702 write_file(actual_normalised_file, actual_str)
1703
1704 # Ignore whitespace when diffing. We should only get to this
1705 # point if there are non-whitespace differences
1706 #
1707 # Note we are diffing the *actual* output, not the normalised
1708 # output. The normalised output may have whitespace squashed
1709 # (including newlines) so the diff would be hard to read.
1710 # This does mean that the diff might contain changes that
1711 # would be normalised away.
1712 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1713 ' ' + actual_file )
1714
1715 # If for some reason there were no non-whitespace differences,
1716 # then do a full diff
1717 if r == 0:
1718 r = os.system( 'diff -u ' + expected_file_for_diff + \
1719 ' ' + actual_file )
1720
1721 if config.accept:
1722 print 'Accepting new output.'
1723 write_file(expected_file, actual_raw)
1724 return 1
1725 else:
1726 return 0
1727
1728
1729 def normalise_whitespace( str ):
1730 # Merge contiguous whitespace characters into a single space.
1731 str = re.sub('[ \t\n]+', ' ', str)
1732 return str
1733
1734 def normalise_errmsg( str ):
1735 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1736 # the colon is there because it appears in error messages; this
1737 # hacky solution is used in place of more sophisticated filename
1738 # mangling
1739 str = re.sub('([^\\s])\\.exe', '\\1', str)
1740 # normalise slashes, minimise Windows/Unix filename differences
1741 str = re.sub('\\\\', '/', str)
1742 # The inplace ghc's are called ghc-stage[123] to avoid filename
1743 # collisions, so we need to normalise that to just "ghc"
1744 str = re.sub('ghc-stage[123]', 'ghc', str)
1745 # We sometimes see the name of the integer-gmp package on stderr,
1746 # but this can change (either the implementation name or the
1747 # version number), so we canonicalise it here
1748 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1749 return str
1750
1751 # normalise a .prof file, so that we can reasonably compare it against
1752 # a sample. This doesn't compare any of the actual profiling data,
1753 # only the shape of the profile and the number of entries.
1754 def normalise_prof (str):
1755 # strip everything up to the line beginning "COST CENTRE"
1756 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1757
1758 # strip results for CAFs, these tend to change unpredictably
1759 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1760
1761 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1762 # sometimes under MAIN.
1763 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1764
1765 # We have somthing like this:
1766
1767 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1768 # k Main 204 1 0.0 0.0 0.0 0.0
1769 # foo Main 205 1 0.0 0.0 0.0 0.0
1770 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1771
1772 # then we remove all the specific profiling data, leaving only the
1773 # cost centre name, module, and entries, to end up with this:
1774
1775 # MAIN MAIN 0
1776 # k Main 1
1777 # foo Main 1
1778 # foo.bar Main 1
1779
1780 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1781 return str
1782
1783 def normalise_slashes_( str ):
1784 str = re.sub('\\\\', '/', str)
1785 return str
1786
1787 def normalise_exe_( str ):
1788 str = re.sub('\.exe', '', str)
1789 return str
1790
1791 def normalise_output( str ):
1792 # Remove a .exe extension (for Windows)
1793 # This can occur in error messages generated by the program.
1794 str = re.sub('([^\\s])\\.exe', '\\1', str)
1795 return str
1796
1797 def normalise_asm( str ):
1798 lines = str.split('\n')
1799 # Only keep instructions and labels not starting with a dot.
1800 metadata = re.compile('^[ \t]*\\..*$')
1801 out = []
1802 for line in lines:
1803 # Drop metadata directives (e.g. ".type")
1804 if not metadata.match(line):
1805 line = re.sub('@plt', '', line)
1806 instr = line.lstrip().split()
1807 # Drop empty lines.
1808 if not instr:
1809 continue
1810 # Drop operands, except for call instructions.
1811 elif instr[0] == 'call':
1812 out.append(instr[0] + ' ' + instr[1])
1813 else:
1814 out.append(instr[0])
1815 out = '\n'.join(out)
1816 return out
1817
1818 def if_verbose( n, str ):
1819 if config.verbose >= n:
1820 print str
1821
1822 def if_verbose_dump( n, f ):
1823 if config.verbose >= n:
1824 try:
1825 print open(f).read()
1826 except:
1827 print ''
1828
1829 def rawSystem(cmd_and_args):
1830 # We prefer subprocess.call to os.spawnv as the latter
1831 # seems to send its arguments through a shell or something
1832 # with the Windows (non-cygwin) python. An argument "a b c"
1833 # turns into three arguments ["a", "b", "c"].
1834
1835 # However, subprocess is new in python 2.4, so fall back to
1836 # using spawnv if we don't have it
1837
1838 if have_subprocess:
1839 return subprocess.call(cmd_and_args)
1840 else:
1841 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1842
1843 # Note that this doesn't handle the timeout itself; it is just used for
1844 # commands that have timeout handling built-in.
1845 def rawSystemWithTimeout(cmd_and_args):
1846 r = rawSystem(cmd_and_args)
1847 if r == 98:
1848 # The python timeout program uses 98 to signal that ^C was pressed
1849 stopNow()
1850 return r
1851
1852 # cmd is a complex command in Bourne-shell syntax
1853 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1854 # Hence it must ultimately be run by a Bourne shell
1855 #
1856 # Mostly it invokes the command wrapped in 'timeout' thus
1857 # timeout 300 'cd . && ...blah blah'
1858 # so it's timeout's job to invoke the Bourne shell
1859 #
1860 # But watch out for the case when there is no timeout program!
1861 # Then, when using the native Python, os.system will invoke the cmd shell
1862
1863 def runCmd( cmd ):
1864 if_verbose( 1, cmd )
1865 r = 0
1866 if config.os == 'mingw32':
1867 # On MinGW, we will always have timeout
1868 assert config.timeout_prog!=''
1869
1870 if config.timeout_prog != '':
1871 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1872 else:
1873 r = os.system(cmd)
1874 return r << 8
1875
1876 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1877 if_verbose( 1, cmd )
1878 r = 0
1879 if config.os == 'mingw32':
1880 # On MinGW, we will always have timeout
1881 assert config.timeout_prog!=''
1882 timeout = int(ceil(config.timeout * timeout_multiplier))
1883
1884 if config.timeout_prog != '':
1885 if config.check_files_written:
1886 fn = name + ".strace"
1887 r = rawSystemWithTimeout(
1888 ["strace", "-o", fn, "-fF",
1889 "-e", "creat,open,chdir,clone,vfork",
1890 config.timeout_prog, str(timeout), cmd])
1891 addTestFilesWritten(name, fn)
1892 rm_no_fail(fn)
1893 else:
1894 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1895 else:
1896 r = os.system(cmd)
1897 return r << 8
1898
1899 def runCmdExitCode( cmd ):
1900 return (runCmd(cmd) >> 8);
1901
1902
1903 # -----------------------------------------------------------------------------
1904 # checking for files being written to by multiple tests
1905
1906 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1907 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1908 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1909 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1910 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1911 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1912 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1913 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1914 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1915 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1916 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1917 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1918 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1919 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1920 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1921 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1922 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1923
1924 # Files that are read or written but shouldn't be:
1925 # * ghci_history shouldn't be read or written by tests
1926 # * things under package.conf.d shouldn't be written by tests
1927 bad_file_usages = {}
1928
1929 # Mapping from tests to the list of files that they write
1930 files_written = {}
1931
1932 # Mapping from tests to the list of files that they write but don't clean
1933 files_written_not_removed = {}
1934
1935 def add_bad_file_usage(name, file):
1936 try:
1937 if not file in bad_file_usages[name]:
1938 bad_file_usages[name].append(file)
1939 except:
1940 bad_file_usages[name] = [file]
1941
1942 def mkPath(curdir, path):
1943 # Given the current full directory is 'curdir', what is the full
1944 # path to 'path'?
1945 return os.path.realpath(os.path.join(curdir, path))
1946
1947 def addTestFilesWritten(name, fn):
1948 if config.use_threads:
1949 with t.lockFilesWritten:
1950 addTestFilesWrittenHelper(name, fn)
1951 else:
1952 addTestFilesWrittenHelper(name, fn)
1953
1954 def addTestFilesWrittenHelper(name, fn):
1955 started = False
1956 working_directories = {}
1957
1958 with open(fn, 'r') as f:
1959 for line in f:
1960 m_pid = re_strace_pid.match(line)
1961 if m_pid:
1962 pid = m_pid.group(1)
1963 content = m_pid.group(2)
1964 elif re_strace_unavailable.match(line):
1965 next
1966 else:
1967 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1968
1969 m_open = re_strace_open.match(content)
1970 m_chdir = re_strace_chdir.match(content)
1971 m_clone = re_strace_clone.match(content)
1972 m_vfork = re_strace_vfork.match(content)
1973
1974 if not started:
1975 working_directories[pid] = os.getcwd()
1976 started = True
1977
1978 if m_open:
1979 file = m_open.group(1)
1980 file = mkPath(working_directories[pid], file)
1981 if file.endswith("ghci_history"):
1982 add_bad_file_usage(name, file)
1983 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1984 flags = m_open.group(2).split('|')
1985 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1986 if re.match('package\.conf\.d', file):
1987 add_bad_file_usage(name, file)
1988 else:
1989 try:
1990 if not file in files_written[name]:
1991 files_written[name].append(file)
1992 except:
1993 files_written[name] = [file]
1994 elif 'O_RDONLY' in flags:
1995 pass
1996 else:
1997 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1998 elif m_chdir:
1999 # We optimistically assume that unfinished chdir's are going to succeed
2000 dir = m_chdir.group(1)
2001 working_directories[pid] = mkPath(working_directories[pid], dir)
2002 elif m_clone:
2003 working_directories[m_clone.group(2)] = working_directories[pid]
2004 elif m_vfork:
2005 working_directories[m_vfork.group(2)] = working_directories[pid]
2006 elif re_strace_open_resumed.match(content):
2007 pass
2008 elif re_strace_chdir_resumed.match(content):
2009 pass
2010 elif re_strace_vfork_unfinished.match(content):
2011 pass
2012 elif re_strace_clone_unfinished.match(content):
2013 pass
2014 elif re_strace_ignore_sigchild.match(content):
2015 pass
2016 elif re_strace_ignore_sigvtalarm.match(content):
2017 pass
2018 elif re_strace_ignore_sigint.match(content):
2019 pass
2020 elif re_strace_ignore_sigfpe.match(content):
2021 pass
2022 elif re_strace_ignore_sigsegv.match(content):
2023 pass
2024 elif re_strace_ignore_sigpipe.match(content):
2025 pass
2026 else:
2027 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2028
2029 def checkForFilesWrittenProblems(file):
2030 foundProblem = False
2031
2032 files_written_inverted = {}
2033 for t in files_written.keys():
2034 for f in files_written[t]:
2035 try:
2036 files_written_inverted[f].append(t)
2037 except:
2038 files_written_inverted[f] = [t]
2039
2040 for f in files_written_inverted.keys():
2041 if len(files_written_inverted[f]) > 1:
2042 if not foundProblem:
2043 foundProblem = True
2044 file.write("\n")
2045 file.write("\nSome files are written by multiple tests:\n")
2046 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2047 if foundProblem:
2048 file.write("\n")
2049
2050 # -----
2051
2052 if len(files_written_not_removed) > 0:
2053 file.write("\n")
2054 file.write("\nSome files written but not removed:\n")
2055 tests = files_written_not_removed.keys()
2056 tests.sort()
2057 for t in tests:
2058 for f in files_written_not_removed[t]:
2059 file.write(" " + t + ": " + f + "\n")
2060 file.write("\n")
2061
2062 # -----
2063
2064 if len(bad_file_usages) > 0:
2065 file.write("\n")
2066 file.write("\nSome bad file usages:\n")
2067 tests = bad_file_usages.keys()
2068 tests.sort()
2069 for t in tests:
2070 for f in bad_file_usages[t]:
2071 file.write(" " + t + ": " + f + "\n")
2072 file.write("\n")
2073
2074 # -----------------------------------------------------------------------------
2075 # checking if ghostscript is available for checking the output of hp2ps
2076
2077 def genGSCmd(psfile):
2078 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2079
2080 def gsNotWorking():
2081 global gs_working
2082 print "GhostScript not available for hp2ps tests"
2083
2084 global gs_working
2085 gs_working = 0
2086 if config.have_profiling:
2087 if config.gs != '':
2088 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2089 if resultGood == 0:
2090 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2091 if resultBad != 0:
2092 print "GhostScript available for hp2ps tests"
2093 gs_working = 1;
2094 else:
2095 gsNotWorking();
2096 else:
2097 gsNotWorking();
2098 else:
2099 gsNotWorking();
2100
2101 def rm_no_fail( file ):
2102 try:
2103 os.remove( file )
2104 finally:
2105 return
2106
2107 def add_suffix( name, suffix ):
2108 if suffix == '':
2109 return name
2110 else:
2111 return name + '.' + suffix
2112
2113 def add_hs_lhs_suffix(name):
2114 if getTestOpts().c_src:
2115 return add_suffix(name, 'c')
2116 elif getTestOpts().cmm_src:
2117 return add_suffix(name, 'cmm')
2118 elif getTestOpts().objc_src:
2119 return add_suffix(name, 'm')
2120 elif getTestOpts().objcpp_src:
2121 return add_suffix(name, 'mm')
2122 elif getTestOpts().literate:
2123 return add_suffix(name, 'lhs')
2124 else:
2125 return add_suffix(name, 'hs')
2126
2127 def replace_suffix( name, suffix ):
2128 base, suf = os.path.splitext(name)
2129 return base + '.' + suffix
2130
2131 def in_testdir( name ):
2132 return (getTestOpts().testdir + '/' + name)
2133
2134 def qualify( name, suff ):
2135 return in_testdir(add_suffix(name, suff))
2136
2137
2138 # Finding the sample output. The filename is of the form
2139 #
2140 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2141 #
2142 # and we pick the most specific version available. The <version> is
2143 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2144 # more fine-grained control use if_compiler_lt().
2145 #
2146 def platform_wordsize_qualify( name, suff ):
2147
2148 basepath = qualify(name, suff)
2149
2150 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2151 for (platformSpecific, plat) in [(1, '-' + config.platform),
2152 (1, '-' + config.os),
2153 (0, '')]
2154 for ws in ['-ws-' + config.wordsize, '']
2155 for comp in ['-' + config.compiler_type, '']
2156 for vers in ['-' + config.compiler_maj_version, '']]
2157
2158 dir = glob.glob(basepath + '*')
2159 dir = map (lambda d: normalise_slashes_(d), dir)
2160
2161 for (platformSpecific, f) in paths:
2162 if f in dir:
2163 return (platformSpecific,f)
2164
2165 return (0, basepath)
2166
2167 # Clean up prior to the test, so that we can't spuriously conclude
2168 # that it passed on the basis of old run outputs.
2169 def pretest_cleanup(name):
2170 if getTestOpts().outputdir != None:
2171 odir = in_testdir(getTestOpts().outputdir)
2172 try:
2173 shutil.rmtree(odir)
2174 except:
2175 pass
2176 os.mkdir(odir)
2177
2178 rm_no_fail(qualify(name,'comp.stderr'))
2179 rm_no_fail(qualify(name,'run.stderr'))
2180 rm_no_fail(qualify(name,'run.stdout'))
2181 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2182 # simple_build zaps the following:
2183 # rm_nofail(qualify("o"))
2184 # rm_nofail(qualify(""))
2185 # not interested in the return code
2186
2187 # -----------------------------------------------------------------------------
2188 # Return a list of all the files ending in '.T' below the directory dir.
2189
2190 def findTFiles(roots):
2191 return concat(map(findTFiles_,roots))
2192
2193 def findTFiles_(path):
2194 if os.path.isdir(path):
2195 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2196 return findTFiles(paths)
2197 elif path[-2:] == '.T':
2198 return [path]
2199 else:
2200 return []
2201
2202 # -----------------------------------------------------------------------------
2203 # Output a test summary to the specified file object
2204
2205 def summary(t, file):
2206
2207 file.write('\n')
2208 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2209 file.write('OVERALL SUMMARY for test run started at ' \
2210 + t.start_time + '\n'\
2211 + string.rjust(`t.total_tests`, 8) \
2212 + ' total tests, which gave rise to\n' \
2213 + string.rjust(`t.total_test_cases`, 8) \
2214 + ' test cases, of which\n' \
2215 + string.rjust(`t.n_framework_failures`, 8) \
2216 + ' caused framework failures\n' \
2217 + string.rjust(`t.n_tests_skipped`, 8)
2218 + ' were skipped\n\n' \
2219 + string.rjust(`t.n_expected_passes`, 8)
2220 + ' expected passes\n' \
2221 + string.rjust(`t.n_missing_libs`, 8)
2222 + ' had missing libraries\n' \
2223 + string.rjust(`t.n_expected_failures`, 8) \
2224 + ' expected failures\n' \
2225 + string.rjust(`t.n_unexpected_passes`, 8) \
2226 + ' unexpected passes\n'
2227 + string.rjust(`t.n_unexpected_failures`, 8) \
2228 + ' unexpected failures\n'
2229 + '\n')
2230
2231 if t.n_unexpected_passes > 0:
2232 file.write('Unexpected passes:\n')
2233 printPassingTestInfosSummary(file, t.unexpected_passes)
2234
2235 if t.n_unexpected_failures > 0:
2236 file.write('Unexpected failures:\n')
2237 printFailingTestInfosSummary(file, t.unexpected_failures)
2238
2239 if config.check_files_written:
2240 checkForFilesWrittenProblems(file)
2241
2242 if stopping():
2243 file.write('WARNING: Testsuite run was terminated early\n')
2244
2245 def printUnexpectedTests(file, testInfoss):
2246 unexpected = []
2247 for testInfos in testInfoss:
2248 directories = testInfos.keys()
2249 for directory in directories:
2250 tests = testInfos[directory].keys()
2251 unexpected += tests
2252 if unexpected != []:
2253 file.write('Unexpected results from:\n')
2254 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2255 file.write('\n')
2256
2257 def printPassingTestInfosSummary(file, testInfos):
2258 directories = testInfos.keys()
2259 directories.sort()
2260 maxDirLen = max(map ((lambda x : len(x)), directories))
2261 for directory in directories:
2262 tests = testInfos[directory].keys()
2263 tests.sort()
2264 for test in tests:
2265 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2266 ' (' + join(testInfos[directory][test],',') + ')\n')
2267 file.write('\n')
2268
2269 def printFailingTestInfosSummary(file, testInfos):
2270 directories = testInfos.keys()
2271 directories.sort()
2272 maxDirLen = max(map ((lambda x : len(x)), directories))
2273 for directory in directories:
2274 tests = testInfos[directory].keys()
2275 tests.sort()
2276 for test in tests:
2277 reasons = testInfos[directory][test].keys()
2278 for reason in reasons:
2279 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2280 ' [' + reason + ']' + \
2281 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2282 file.write('\n')
2283
2284 def getStdout(cmd):
2285 if have_subprocess:
2286 p = subprocess.Popen(cmd,
2287 stdout=subprocess.PIPE,
2288 stderr=subprocess.PIPE)
2289 (stdout, stderr) = p.communicate()
2290 r = p.wait()
2291 if r != 0:
2292 raise Exception("Command failed: " + str(cmd))
2293 if stderr != '':
2294 raise Exception("stderr from command: " + str(cmd))
2295 return stdout
2296 else:
2297 raise Exception("Need subprocess to get stdout, but don't have it")