Merge branch 'master' of http://darcs.haskell.org/testsuite
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 record_broken(name, opts, bug)
155 opts.expect = 'fail';
156
157 def expect_broken_for( bug, ways ):
158 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
159
160 def _expect_broken_for( name, opts, bug, ways ):
161 record_broken(name, opts, bug)
162 opts.expect_fail_for = ways
163
164 def record_broken(name, opts, bug):
165 global brokens
166 me = (bug, opts.testdir, name)
167 if not me in brokens:
168 brokens.append(me)
169
170 # -----
171
172 def omit_ways( ways ):
173 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174
175 def _omit_ways( name, opts, ways ):
176 opts.omit_ways = ways
177
178 # -----
179
180 def only_ways( ways ):
181 return lambda name, opts, w=ways: _only_ways( name, opts, w )
182
183 def _only_ways( name, opts, ways ):
184 opts.only_ways = ways
185
186 # -----
187
188 def extra_ways( ways ):
189 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190
191 def _extra_ways( name, opts, ways ):
192 opts.extra_ways = ways
193
194 # -----
195
196 def omit_compiler_types( compiler_types ):
197 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
198
199 def _omit_compiler_types( name, opts, compiler_types ):
200 if config.compiler_type in compiler_types:
201 opts.skip = 1
202
203 # -----
204
205 def only_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
207
208 def _only_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type not in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def set_stdin( file ):
215 return lambda name, opts, f=file: _set_stdin(name, opts, f);
216
217 def _set_stdin( name, opts, f ):
218 opts.stdin = f
219
220 # -----
221
222 def exit_code( val ):
223 return lambda name, opts, v=val: _exit_code(name, opts, v);
224
225 def _exit_code( name, opts, v ):
226 opts.exit_code = v
227
228 # -----
229
230 def timeout_multiplier( val ):
231 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232
233 def _timeout_multiplier( name, opts, v ):
234 opts.timeout_multiplier = v
235
236 # -----
237
238 def extra_run_opts( val ):
239 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240
241 def _extra_run_opts( name, opts, v ):
242 opts.extra_run_opts = v
243
244 # -----
245
246 def extra_hc_opts( val ):
247 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
248
249 def _extra_hc_opts( name, opts, v ):
250 opts.extra_hc_opts = v
251
252 # -----
253
254 def extra_clean( files ):
255 return lambda name, opts, v=files: _extra_clean(name, opts, v);
256
257 def _extra_clean( name, opts, v ):
258 opts.clean_files = v
259
260 # -----
261
262 def stats_num_field( field, expecteds ):
263 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
264
265 def _stats_num_field( name, opts, field, expecteds ):
266 if field in opts.stats_range_fields:
267 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
268
269 if type(expecteds) is types.ListType:
270 for (b, expected, dev) in expecteds:
271 if b:
272 opts.stats_range_fields[field] = (expected, dev)
273 return
274 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275
276 else:
277 (expected, dev) = expecteds
278 opts.stats_range_fields[field] = (expected, dev)
279
280 def compiler_stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282
283 def _compiler_stats_num_field( name, opts, field, expecteds ):
284 if field in opts.compiler_stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 for (b, expected, dev) in expecteds:
288 if b:
289 opts.compiler_stats_range_fields[field] = (expected, dev)
290 return
291
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 # -----
295
296 def when(b, f):
297 # When list_brokens is on, we want to see all expect_broken calls,
298 # so we always do f
299 if b or config.list_broken:
300 return f
301 else:
302 return normal
303
304 def unless(b, f):
305 return when(not b, f)
306
307 def doing_ghci():
308 return 'ghci' in config.run_ways
309
310 def ghci_dynamic( ):
311 return config.ghc_dynamic_by_default
312
313 def fast():
314 return config.fast
315
316 def platform( plat ):
317 return config.platform == plat
318
319 def opsys( os ):
320 return config.os == os
321
322 def arch( arch ):
323 return config.arch == arch
324
325 def wordsize( ws ):
326 return config.wordsize == str(ws)
327
328 def msys( ):
329 return config.msys
330
331 def cygwin( ):
332 return config.cygwin
333
334 def have_vanilla( ):
335 return config.have_vanilla
336
337 def have_dynamic( ):
338 return config.have_dynamic
339
340 def have_profiling( ):
341 return config.have_profiling
342
343 def in_tree_compiler( ):
344 return config.in_tree_compiler
345
346 def compiler_type( compiler ):
347 return config.compiler_type == compiler
348
349 def compiler_lt( compiler, version ):
350 return config.compiler_type == compiler and \
351 version_lt(config.compiler_version, version)
352
353 def compiler_le( compiler, version ):
354 return config.compiler_type == compiler and \
355 version_le(config.compiler_version, version)
356
357 def compiler_gt( compiler, version ):
358 return config.compiler_type == compiler and \
359 version_gt(config.compiler_version, version)
360
361 def compiler_ge( compiler, version ):
362 return config.compiler_type == compiler and \
363 version_ge(config.compiler_version, version)
364
365 def unregisterised( ):
366 return config.unregisterised
367
368 def compiler_profiled( ):
369 return config.compiler_profiled
370
371 def compiler_debugged( ):
372 return config.compiler_debugged
373
374 def tag( t ):
375 return t in config.compiler_tags
376
377 # ---
378
379 def namebase( nb ):
380 return lambda opts, nb=nb: _namebase(opts, nb)
381
382 def _namebase( opts, nb ):
383 opts.with_namebase = nb
384
385 # ---
386
387 def high_memory_usage(name, opts):
388 opts.alone = True
389
390 # ---
391 def literate( name, opts ):
392 opts.literate = 1;
393
394 def c_src( name, opts ):
395 opts.c_src = 1;
396
397 def objc_src( name, opts ):
398 opts.objc_src = 1;
399
400 def objcpp_src( name, opts ):
401 opts.objcpp_src = 1;
402
403 def cmm_src( name, opts ):
404 opts.cmm_src = 1;
405
406 def outputdir( odir ):
407 return lambda name, opts, d=odir: _outputdir(name, opts, d)
408
409 def _outputdir( name, opts, odir ):
410 opts.outputdir = odir;
411
412 # ----
413
414 def pre_cmd( cmd ):
415 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
416
417 def _pre_cmd( name, opts, cmd ):
418 opts.pre_cmd = cmd
419
420 # ----
421
422 def clean_cmd( cmd ):
423 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
424
425 def _clean_cmd( name, opts, cmd ):
426 opts.clean_cmd = cmd
427
428 # ----
429
430 def cmd_prefix( prefix ):
431 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
432
433 def _cmd_prefix( name, opts, prefix ):
434 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
435
436 # ----
437
438 def cmd_wrapper( fun ):
439 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
440
441 def _cmd_wrapper( name, opts, fun ):
442 opts.cmd_wrapper = fun
443
444 # ----
445
446 def compile_cmd_prefix( prefix ):
447 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
448
449 def _compile_cmd_prefix( name, opts, prefix ):
450 opts.compile_cmd_prefix = prefix
451
452 # ----
453
454 def normalise_slashes( name, opts ):
455 opts.extra_normaliser = normalise_slashes_
456
457 def normalise_exe( name, opts ):
458 opts.extra_normaliser = normalise_exe_
459
460 def normalise_fun( fun ):
461 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
462
463 def _normalise_fun( name, opts, f ):
464 opts.extra_normaliser = f
465
466 def normalise_errmsg_fun( fun ):
467 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
468
469 def _normalise_errmsg_fun( name, opts, f ):
470 opts.extra_errmsg_normaliser = f
471
472 def two_normalisers(f, g):
473 return lambda x, f=f, g=g: f(g(x))
474
475 # ----
476 # Function for composing two opt-fns together
477
478 def composes( fs ):
479 return reduce(lambda f, g: compose(f, g), fs)
480
481 def compose( f, g ):
482 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
483
484 def _compose( name, opts, f, g ):
485 f(name, opts)
486 g(name, opts)
487
488 # -----------------------------------------------------------------------------
489 # The current directory of tests
490
491 def newTestDir( dir ):
492 global thisdir_settings
493 # reset the options for this test directory
494 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
495
496 def _newTestDir( name, opts, dir ):
497 opts.testdir = dir
498 opts.compiler_always_flags = config.compiler_always_flags
499
500 # -----------------------------------------------------------------------------
501 # Actually doing tests
502
503 parallelTests = []
504 aloneTests = []
505 allTestNames = set([])
506
507 def runTest (opts, name, func, args):
508 ok = 0
509
510 if config.use_threads:
511 t.thread_pool.acquire()
512 try:
513 while config.threads<(t.running_threads+1):
514 t.thread_pool.wait()
515 t.running_threads = t.running_threads+1
516 ok=1
517 t.thread_pool.release()
518 thread.start_new_thread(test_common_thread, (name, opts, func, args))
519 except:
520 if not ok:
521 t.thread_pool.release()
522 else:
523 test_common_work (name, opts, func, args)
524
525 # name :: String
526 # setup :: TestOpts -> IO ()
527 def test (name, setup, func, args):
528 global aloneTests
529 global parallelTests
530 global allTestNames
531 global thisdir_settings
532 if name in allTestNames:
533 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
534 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
535 framework_fail(name, 'bad_name', 'This test has an invalid name')
536
537 # Make a deep copy of the default_testopts, as we need our own copy
538 # of any dictionaries etc inside it. Otherwise, if one test modifies
539 # them, all tests will see the modified version!
540 myTestOpts = copy.deepcopy(default_testopts)
541
542 if type(setup) is types.ListType:
543 setup = composes(setup)
544
545 setup = compose(thisdir_settings, setup)
546 setup(name, myTestOpts)
547
548 thisTest = lambda : runTest(myTestOpts, name, func, args)
549 if myTestOpts.alone:
550 aloneTests.append(thisTest)
551 else:
552 parallelTests.append(thisTest)
553 allTestNames.add(name)
554
555 if config.use_threads:
556 def test_common_thread(name, opts, func, args):
557 t.lock.acquire()
558 try:
559 test_common_work(name,opts,func,args)
560 finally:
561 t.lock.release()
562 t.thread_pool.acquire()
563 t.running_threads = t.running_threads - 1
564 t.thread_pool.notify()
565 t.thread_pool.release()
566
567 def get_package_cache_timestamp():
568 if config.package_conf_cache_file == '':
569 return 0.0
570 else:
571 try:
572 return os.stat(config.package_conf_cache_file).st_mtime
573 except:
574 return 0.0
575
576
577 def test_common_work (name, opts, func, args):
578 try:
579 t.total_tests = t.total_tests+1
580 setLocalTestOpts(opts)
581
582 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
583
584 # All the ways we might run this test
585 if func == compile or func == multimod_compile:
586 all_ways = config.compile_ways
587 elif func == compile_and_run or func == multimod_compile_and_run:
588 all_ways = config.run_ways
589 elif func == ghci_script:
590 if 'ghci' in config.run_ways:
591 all_ways = ['ghci']
592 else:
593 all_ways = []
594 else:
595 all_ways = ['normal']
596
597 # A test itself can request extra ways by setting opts.extra_ways
598 all_ways = all_ways + filter(lambda way: way not in all_ways,
599 opts.extra_ways)
600
601 t.total_test_cases = t.total_test_cases + len(all_ways)
602
603 ok_way = lambda way: \
604 not getTestOpts().skip \
605 and (config.only == [] or name in config.only) \
606 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
607 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
608 and way not in getTestOpts().omit_ways
609
610 # Which ways we are asked to skip
611 do_ways = filter (ok_way,all_ways)
612
613 # In fast mode, we skip all but one way
614 if config.fast and len(do_ways) > 0:
615 do_ways = [do_ways[0]]
616
617 if not config.clean_only:
618 # Run the required tests...
619 for way in do_ways:
620 if stopping():
621 break
622 do_test (name, way, func, args)
623
624 for way in all_ways:
625 if way not in do_ways:
626 skiptest (name,way)
627
628 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
629 clean(map (lambda suff: name + suff,
630 ['', '.exe', '.exe.manifest', '.genscript',
631 '.stderr.normalised', '.stdout.normalised',
632 '.run.stderr', '.run.stdout',
633 '.run.stderr.normalised', '.run.stdout.normalised',
634 '.comp.stderr', '.comp.stdout',
635 '.comp.stderr.normalised', '.comp.stdout.normalised',
636 '.interp.stderr', '.interp.stdout',
637 '.interp.stderr.normalised', '.interp.stdout.normalised',
638 '.stats', '.comp.stats',
639 '.hi', '.o', '.prof', '.exe.prof', '.hc',
640 '_stub.h', '_stub.c', '_stub.o',
641 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
642
643 if func == multi_compile or func == multi_compile_fail:
644 extra_mods = args[1]
645 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
646 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
647
648 clean(getTestOpts().clean_files)
649
650 if getTestOpts().outputdir != None:
651 odir = in_testdir(getTestOpts().outputdir)
652 try:
653 shutil.rmtree(odir)
654 except:
655 pass
656
657 try:
658 shutil.rmtree(in_testdir('.hpc.' + name))
659 except:
660 pass
661
662 try:
663 cleanCmd = getTestOpts().clean_cmd
664 if cleanCmd != None:
665 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
666 if result != 0:
667 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
668 except e:
669 framework_fail(name, 'cleaning', 'clean-command exception')
670
671 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
672
673 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
674 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
675
676 try:
677 for f in files_written[name]:
678 if os.path.exists(f):
679 try:
680 if not f in files_written_not_removed[name]:
681 files_written_not_removed[name].append(f)
682 except:
683 files_written_not_removed[name] = [f]
684 except:
685 pass
686 except Exception, e:
687 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
688
689 def clean(strs):
690 for str in strs:
691 for name in glob.glob(in_testdir(str)):
692 clean_full_path(name)
693
694 def clean_full_path(name):
695 try:
696 # Remove files...
697 os.remove(name)
698 except OSError, e1:
699 try:
700 # ... and empty directories
701 os.rmdir(name)
702 except OSError, e2:
703 # We don't want to fail here, but we do want to know
704 # what went wrong, so print out the exceptions.
705 # ENOENT isn't a problem, though, as we clean files
706 # that don't necessarily exist.
707 if e1.errno != errno.ENOENT:
708 print e1
709 if e2.errno != errno.ENOENT:
710 print e2
711
712 def do_test(name, way, func, args):
713 full_name = name + '(' + way + ')'
714
715 try:
716 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
717 str([t.n_unexpected_passes, \
718 t.n_unexpected_failures, \
719 t.n_framework_failures])
720
721 if config.use_threads:
722 t.lock.release()
723
724 try:
725 preCmd = getTestOpts().pre_cmd
726 if preCmd != None:
727 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
728 if result != 0:
729 framework_fail(name, way, 'pre-command failed: ' + str(result))
730 except e:
731 framework_fail(name, way, 'pre-command exception')
732
733 try:
734 result = apply(func, [name,way] + args)
735 finally:
736 if config.use_threads:
737 t.lock.acquire()
738
739 if getTestOpts().expect != 'pass' and \
740 getTestOpts().expect != 'fail' and \
741 getTestOpts().expect != 'missing-lib':
742 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
743
744 try:
745 passFail = result['passFail']
746 except:
747 passFail = 'No passFail found'
748
749 if passFail == 'pass':
750 if getTestOpts().expect == 'pass' \
751 and way not in getTestOpts().expect_fail_for:
752 t.n_expected_passes = t.n_expected_passes + 1
753 if name in t.expected_passes:
754 t.expected_passes[name].append(way)
755 else:
756 t.expected_passes[name] = [way]
757 else:
758 print '*** unexpected pass for', full_name
759 t.n_unexpected_passes = t.n_unexpected_passes + 1
760 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
761 elif passFail == 'fail':
762 if getTestOpts().expect == 'pass' \
763 and way not in getTestOpts().expect_fail_for:
764 print '*** unexpected failure for', full_name
765 t.n_unexpected_failures = t.n_unexpected_failures + 1
766 reason = result['reason']
767 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
768 else:
769 if getTestOpts().expect == 'missing-lib':
770 t.n_missing_libs = t.n_missing_libs + 1
771 if name in t.missing_libs:
772 t.missing_libs[name].append(way)
773 else:
774 t.missing_libs[name] = [way]
775 else:
776 t.n_expected_failures = t.n_expected_failures + 1
777 if name in t.expected_failures:
778 t.expected_failures[name].append(way)
779 else:
780 t.expected_failures[name] = [way]
781 else:
782 framework_fail(name, way, 'bad result ' + passFail)
783 except KeyboardInterrupt:
784 stopNow()
785 except:
786 framework_fail(name, way, 'do_test exception')
787 traceback.print_exc()
788
789 def addPassingTestInfo (testInfos, directory, name, way):
790 directory = re.sub('^\\.[/\\\\]', '', directory)
791
792 if not directory in testInfos:
793 testInfos[directory] = {}
794
795 if not name in testInfos[directory]:
796 testInfos[directory][name] = []
797
798 testInfos[directory][name].append(way)
799
800 def addFailingTestInfo (testInfos, directory, name, reason, way):
801 directory = re.sub('^\\.[/\\\\]', '', directory)
802
803 if not directory in testInfos:
804 testInfos[directory] = {}
805
806 if not name in testInfos[directory]:
807 testInfos[directory][name] = {}
808
809 if not reason in testInfos[directory][name]:
810 testInfos[directory][name][reason] = []
811
812 testInfos[directory][name][reason].append(way)
813
814 def skiptest (name, way):
815 # print 'Skipping test \"', name, '\"'
816 t.n_tests_skipped = t.n_tests_skipped + 1
817 if name in t.tests_skipped:
818 t.tests_skipped[name].append(way)
819 else:
820 t.tests_skipped[name] = [way]
821
822 def framework_fail( name, way, reason ):
823 full_name = name + '(' + way + ')'
824 print '*** framework failure for', full_name, reason
825 t.n_framework_failures = t.n_framework_failures + 1
826 if name in t.framework_failures:
827 t.framework_failures[name].append(way)
828 else:
829 t.framework_failures[name] = [way]
830
831 def badResult(result):
832 try:
833 if result['passFail'] == 'pass':
834 return False
835 return True
836 except:
837 return True
838
839 def passed():
840 return {'passFail': 'pass'}
841
842 def failBecause(reason):
843 return {'passFail': 'fail', 'reason': reason}
844
845 # -----------------------------------------------------------------------------
846 # Generic command tests
847
848 # A generic command test is expected to run and exit successfully.
849 #
850 # The expected exit code can be changed via exit_code() as normal, and
851 # the expected stdout/stderr are stored in <testname>.stdout and
852 # <testname>.stderr. The output of the command can be ignored
853 # altogether by using run_command_ignore_output instead of
854 # run_command.
855
856 def run_command( name, way, cmd ):
857 return simple_run( name, '', cmd, '' )
858
859 # -----------------------------------------------------------------------------
860 # GHCi tests
861
862 def ghci_script( name, way, script ):
863 # filter out -fforce-recomp from compiler_always_flags, because we're
864 # actually testing the recompilation behaviour in the GHCi tests.
865 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
866 flags.append(getTestOpts().extra_hc_opts)
867 if getTestOpts().outputdir != None:
868 flags.extend(["-outputdir", getTestOpts().outputdir])
869
870 # We pass HC and HC_OPTS as environment variables, so that the
871 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
872 cmd = "HC='" + config.compiler + "' " + \
873 "HC_OPTS='" + join(flags,' ') + "' " + \
874 "'" + config.compiler + "'" + \
875 ' --interactive -v0 -ignore-dot-ghci ' + \
876 join(flags,' ')
877
878 getTestOpts().stdin = script
879 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
880
881 # -----------------------------------------------------------------------------
882 # Compile-only tests
883
884 def compile( name, way, extra_hc_opts ):
885 return do_compile( name, way, 0, '', [], extra_hc_opts )
886
887 def compile_fail( name, way, extra_hc_opts ):
888 return do_compile( name, way, 1, '', [], extra_hc_opts )
889
890 def multimod_compile( name, way, top_mod, extra_hc_opts ):
891 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
892
893 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
894 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
895
896 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
897 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
898
899 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
900 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
901
902 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
903 # print 'Compile only, extra args = ', extra_hc_opts
904 pretest_cleanup(name)
905
906 result = extras_build( way, extra_mods, extra_hc_opts )
907 if badResult(result):
908 return result
909 extra_hc_opts = result['hc_opts']
910
911 force = 0
912 if extra_mods:
913 force = 1
914 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
915
916 if badResult(result):
917 return result
918
919 # the actual stderr should always match the expected, regardless
920 # of whether we expected the compilation to fail or not (successful
921 # compilations may generate warnings).
922
923 if getTestOpts().with_namebase == None:
924 namebase = name
925 else:
926 namebase = getTestOpts().with_namebase
927
928 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
929 actual_stderr_file = qualify(name, 'comp.stderr')
930
931 if not compare_outputs('stderr', \
932 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
933 expected_stderr_file, actual_stderr_file):
934 return failBecause('stderr mismatch')
935
936 # no problems found, this test passed
937 return passed()
938
939 def compile_cmp_asm( name, way, extra_hc_opts ):
940 print 'Compile only, extra args = ', extra_hc_opts
941 pretest_cleanup(name)
942 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
943
944 if badResult(result):
945 return result
946
947 # the actual stderr should always match the expected, regardless
948 # of whether we expected the compilation to fail or not (successful
949 # compilations may generate warnings).
950
951 if getTestOpts().with_namebase == None:
952 namebase = name
953 else:
954 namebase = getTestOpts().with_namebase
955
956 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
957 actual_asm_file = qualify(name, 's')
958
959 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
960 expected_asm_file, actual_asm_file):
961 return failBecause('asm mismatch')
962
963 # no problems found, this test passed
964 return passed()
965
966 # -----------------------------------------------------------------------------
967 # Compile-and-run tests
968
969 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
970 # print 'Compile and run, extra args = ', extra_hc_opts
971 pretest_cleanup(name)
972
973 result = extras_build( way, extra_mods, extra_hc_opts )
974 if badResult(result):
975 return result
976 extra_hc_opts = result['hc_opts']
977
978 if way == 'ghci': # interpreted...
979 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
980 elif way == 'extcore' or way == 'optextcore' :
981 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
982 else: # compiled...
983 force = 0
984 if extra_mods:
985 force = 1
986
987 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
988 if badResult(result):
989 return result
990
991 cmd = './' + name;
992
993 # we don't check the compiler's stderr for a compile-and-run test
994 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
995
996 def compile_and_run( name, way, extra_hc_opts ):
997 return compile_and_run__( name, way, '', [], extra_hc_opts)
998
999 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1000 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1001
1002 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1003 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1004
1005 def stats( name, way, stats_file ):
1006 opts = getTestOpts()
1007 return checkStats(stats_file, opts.stats_range_fields)
1008
1009 # -----------------------------------------------------------------------------
1010 # Check -t stats info
1011
1012 def checkStats(stats_file, range_fields):
1013 result = passed()
1014 if len(range_fields) > 0:
1015 f = open(in_testdir(stats_file))
1016 contents = f.read()
1017 f.close()
1018
1019 for (field, (expected, dev)) in range_fields.items():
1020 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1021 if m == None:
1022 print 'Failed to find field: ', field
1023 result = failBecause('no such stats field')
1024 val = int(m.group(1))
1025
1026 lowerBound = trunc( expected * ((100 - float(dev))/100));
1027 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1028
1029 if val < lowerBound:
1030 print field, 'value is too low:'
1031 print '(If this is because you have improved GHC, please'
1032 print 'update the test so that GHC doesn\'t regress again)'
1033 result = failBecause('stat too good')
1034 if val > upperBound:
1035 print field, 'value is too high:'
1036 result = failBecause('stat not good enough')
1037
1038 if val < lowerBound or val > upperBound:
1039 valStr = str(val)
1040 valLen = len(valStr)
1041 expectedStr = str(expected)
1042 expectedLen = len(expectedStr)
1043 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1044 def display(descr, val, extra):
1045 print descr, string.rjust(str(val), length), extra
1046 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1047 display(' Lower bound ' + field + ':', lowerBound, '')
1048 display(' Upper bound ' + field + ':', upperBound, '')
1049 display(' Actual ' + field + ':', val, '')
1050
1051 return result
1052
1053 # -----------------------------------------------------------------------------
1054 # Build a single-module program
1055
1056 def extras_build( way, extra_mods, extra_hc_opts ):
1057 for modopts in extra_mods:
1058 mod, opts = modopts
1059 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1060 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1061 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1062 if badResult(result):
1063 return result
1064
1065 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1066
1067
1068 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1069 opts = getTestOpts()
1070 errname = add_suffix(name, 'comp.stderr')
1071 rm_no_fail( qualify(errname, '') )
1072
1073 if top_mod != '':
1074 srcname = top_mod
1075 rm_no_fail( qualify(name, '') )
1076 base, suf = os.path.splitext(top_mod)
1077 rm_no_fail( qualify(base, '') )
1078 rm_no_fail( qualify(base, 'exe') )
1079 elif addsuf:
1080 srcname = add_hs_lhs_suffix(name)
1081 rm_no_fail( qualify(name, '') )
1082 else:
1083 srcname = name
1084 rm_no_fail( qualify(name, 'o') )
1085
1086 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1087
1088 to_do = ''
1089 if top_mod != '':
1090 to_do = '--make '
1091 if link:
1092 to_do = to_do + '-o ' + name
1093 elif link:
1094 to_do = '-o ' + name
1095 elif opts.compile_to_hc:
1096 to_do = '-C'
1097 else:
1098 to_do = '-c' # just compile
1099
1100 stats_file = name + '.comp.stats'
1101 if len(opts.compiler_stats_range_fields) > 0:
1102 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1103
1104 # Required by GHC 7.3+, harmless for earlier versions:
1105 if (getTestOpts().c_src or
1106 getTestOpts().objc_src or
1107 getTestOpts().objcpp_src or
1108 getTestOpts().cmm_src):
1109 extra_hc_opts += ' -no-hs-main '
1110
1111 if getTestOpts().compile_cmd_prefix == '':
1112 cmd_prefix = ''
1113 else:
1114 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1115
1116 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1117 if noforce:
1118 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1119 if getTestOpts().outputdir != None:
1120 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1121
1122 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1123 + config.compiler + "' " \
1124 + join(comp_flags,' ') + ' ' \
1125 + to_do + ' ' + srcname + ' ' \
1126 + join(config.way_flags(name)[way],' ') + ' ' \
1127 + extra_hc_opts + ' ' \
1128 + opts.extra_hc_opts + ' ' \
1129 + '>' + errname + ' 2>&1'
1130
1131 result = runCmdFor(name, cmd)
1132
1133 if result != 0 and not should_fail:
1134 actual_stderr = qualify(name, 'comp.stderr')
1135 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1136 if_verbose_dump(1,actual_stderr)
1137
1138 # ToDo: if the sub-shell was killed by ^C, then exit
1139
1140 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1141
1142 if badResult(statsResult):
1143 return statsResult
1144
1145 if should_fail:
1146 if result == 0:
1147 return failBecause('exit code 0')
1148 else:
1149 if result != 0:
1150 return failBecause('exit code non-0')
1151
1152 return passed()
1153
1154 # -----------------------------------------------------------------------------
1155 # Run a program and check its output
1156 #
1157 # If testname.stdin exists, route input from that, else
1158 # from /dev/null. Route output to testname.run.stdout and
1159 # testname.run.stderr. Returns the exit code of the run.
1160
1161 def simple_run( name, way, prog, args ):
1162 opts = getTestOpts()
1163
1164 # figure out what to use for stdin
1165 if opts.stdin != '':
1166 use_stdin = opts.stdin
1167 else:
1168 stdin_file = add_suffix(name, 'stdin')
1169 if os.path.exists(in_testdir(stdin_file)):
1170 use_stdin = stdin_file
1171 else:
1172 use_stdin = '/dev/null'
1173
1174 run_stdout = add_suffix(name,'run.stdout')
1175 run_stderr = add_suffix(name,'run.stderr')
1176
1177 rm_no_fail(qualify(name,'run.stdout'))
1178 rm_no_fail(qualify(name,'run.stderr'))
1179 rm_no_fail(qualify(name, 'hp'))
1180 rm_no_fail(qualify(name,'ps'))
1181 rm_no_fail(qualify(name, 'prof'))
1182
1183 my_rts_flags = rts_flags(way)
1184
1185 stats_file = name + '.stats'
1186 if len(opts.stats_range_fields) > 0:
1187 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1188
1189 if opts.no_stdin:
1190 stdin_comes_from = ''
1191 else:
1192 stdin_comes_from = ' <' + use_stdin
1193
1194 if opts.combined_output:
1195 redirection = ' >' + run_stdout \
1196 + ' 2>&1'
1197 else:
1198 redirection = ' >' + run_stdout \
1199 + ' 2>' + run_stderr
1200
1201 cmd = prog + ' ' + args + ' ' \
1202 + my_rts_flags + ' ' \
1203 + stdin_comes_from \
1204 + redirection
1205
1206 if opts.cmd_wrapper != None:
1207 cmd = opts.cmd_wrapper(cmd);
1208
1209 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1210
1211 # run the command
1212 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1213
1214 exit_code = result >> 8
1215 signal = result & 0xff
1216
1217 # check the exit code
1218 if exit_code != opts.exit_code:
1219 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1220 dump_stdout(name)
1221 dump_stderr(name)
1222 return failBecause('bad exit code')
1223
1224 check_hp = my_rts_flags.find("-h") != -1
1225 check_prof = my_rts_flags.find("-p") != -1
1226
1227 if not opts.ignore_output:
1228 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1229 bad_stdout = not check_stdout_ok(name)
1230 if bad_stderr:
1231 return failBecause('bad stderr')
1232 if bad_stdout:
1233 return failBecause('bad stdout')
1234 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1235 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1236 return failBecause('bad heap profile')
1237 if check_prof and not check_prof_ok(name):
1238 return failBecause('bad profile')
1239
1240 return checkStats(stats_file, opts.stats_range_fields)
1241
1242 def rts_flags(way):
1243 if (way == ''):
1244 return ''
1245 else:
1246 args = config.way_rts_flags[way]
1247
1248 if args == []:
1249 return ''
1250 else:
1251 return '+RTS ' + join(args,' ') + ' -RTS'
1252
1253 # -----------------------------------------------------------------------------
1254 # Run a program in the interpreter and check its output
1255
1256 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1257 outname = add_suffix(name, 'interp.stdout')
1258 errname = add_suffix(name, 'interp.stderr')
1259 rm_no_fail(outname)
1260 rm_no_fail(errname)
1261 rm_no_fail(name)
1262
1263 if (top_mod == ''):
1264 srcname = add_hs_lhs_suffix(name)
1265 else:
1266 srcname = top_mod
1267
1268 scriptname = add_suffix(name, 'genscript')
1269 qscriptname = in_testdir(scriptname)
1270 rm_no_fail(qscriptname)
1271
1272 delimiter = '===== program output begins here\n'
1273
1274 script = open(qscriptname, 'w')
1275 if not compile_only:
1276 # set the prog name and command-line args to match the compiled
1277 # environment.
1278 script.write(':set prog ' + name + '\n')
1279 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1280 # Add marker lines to the stdout and stderr output files, so we
1281 # can separate GHCi's output from the program's.
1282 script.write(':! echo ' + delimiter)
1283 script.write(':! echo 1>&2 ' + delimiter)
1284 # Set stdout to be line-buffered to match the compiled environment.
1285 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1286 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1287 # in the event of an exception as for the compiled program.
1288 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1289 script.close()
1290
1291 # figure out what to use for stdin
1292 if getTestOpts().stdin != '':
1293 stdin_file = in_testdir(getTestOpts().stdin)
1294 else:
1295 stdin_file = qualify(name, 'stdin')
1296
1297 if os.path.exists(stdin_file):
1298 stdin = open(stdin_file, 'r')
1299 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1300
1301 script.close()
1302
1303 flags = copy.copy(getTestOpts().compiler_always_flags)
1304 if getTestOpts().outputdir != None:
1305 flags.extend(["-outputdir", getTestOpts().outputdir])
1306
1307 cmd = "'" + config.compiler + "' " \
1308 + join(flags,' ') + ' ' \
1309 + srcname + ' ' \
1310 + join(config.way_flags(name)[way],' ') + ' ' \
1311 + extra_hc_opts + ' ' \
1312 + getTestOpts().extra_hc_opts + ' ' \
1313 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1314
1315 if getTestOpts().cmd_wrapper != None:
1316 cmd = getTestOpts().cmd_wrapper(cmd);
1317
1318 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1319
1320 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1321
1322 exit_code = result >> 8
1323 signal = result & 0xff
1324
1325 # split the stdout into compilation/program output
1326 split_file(in_testdir(outname), delimiter,
1327 qualify(name, 'comp.stdout'),
1328 qualify(name, 'run.stdout'))
1329 split_file(in_testdir(errname), delimiter,
1330 qualify(name, 'comp.stderr'),
1331 qualify(name, 'run.stderr'))
1332
1333 # check the exit code
1334 if exit_code != getTestOpts().exit_code:
1335 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1336 dump_stdout(name)
1337 dump_stderr(name)
1338 return failBecause('bad exit code')
1339
1340 # ToDo: if the sub-shell was killed by ^C, then exit
1341
1342 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1343 check_stdout_ok(name)):
1344 return passed()
1345 else:
1346 return failBecause('bad stdout or stderr')
1347
1348
1349 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1350 infile = open(in_fn)
1351 out1 = open(out1_fn, 'w')
1352 out2 = open(out2_fn, 'w')
1353
1354 line = infile.readline()
1355 line = re.sub('\r', '', line) # ignore Windows EOL
1356 while (re.sub('^\s*','',line) != delimiter and line != ''):
1357 out1.write(line)
1358 line = infile.readline()
1359 line = re.sub('\r', '', line)
1360 out1.close()
1361
1362 line = infile.readline()
1363 while (line != ''):
1364 out2.write(line)
1365 line = infile.readline()
1366 out2.close()
1367
1368 # -----------------------------------------------------------------------------
1369 # Generate External Core for the given program, then compile the resulting Core
1370 # and compare its output to the expected output
1371
1372 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1373
1374 depsfilename = qualify(name, 'deps')
1375 errname = add_suffix(name, 'comp.stderr')
1376 qerrname = qualify(errname,'')
1377
1378 hcname = qualify(name, 'hc')
1379 oname = qualify(name, 'o')
1380
1381 rm_no_fail( qerrname )
1382 rm_no_fail( qualify(name, '') )
1383
1384 if (top_mod == ''):
1385 srcname = add_hs_lhs_suffix(name)
1386 else:
1387 srcname = top_mod
1388
1389 qcorefilename = qualify(name, 'hcr')
1390 corefilename = add_suffix(name, 'hcr')
1391 rm_no_fail(qcorefilename)
1392
1393 # Generate External Core
1394
1395 if (top_mod == ''):
1396 to_do = ' ' + srcname + ' '
1397 else:
1398 to_do = ' --make ' + top_mod + ' '
1399
1400 flags = copy.copy(getTestOpts().compiler_always_flags)
1401 if getTestOpts().outputdir != None:
1402 flags.extend(["-outputdir", getTestOpts().outputdir])
1403 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1404 + config.compiler + "' " \
1405 + join(flags,' ') + ' ' \
1406 + join(config.way_flags(name)[way],' ') + ' ' \
1407 + extra_hc_opts + ' ' \
1408 + getTestOpts().extra_hc_opts \
1409 + to_do \
1410 + '>' + errname + ' 2>&1'
1411 result = runCmdFor(name, cmd)
1412
1413 exit_code = result >> 8
1414
1415 if exit_code != 0:
1416 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1417 if_verbose_dump(1,qerrname)
1418 return failBecause('ext core exit code non-0')
1419
1420 # Compile the resulting files -- if there's more than one module, we need to read the output
1421 # of the previous compilation in order to find the dependencies
1422 if (top_mod == ''):
1423 to_compile = corefilename
1424 else:
1425 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1426 deps = open(depsfilename).read()
1427 deplist = string.replace(deps, '\n',' ');
1428 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1429 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1430
1431 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1432 if getTestOpts().outputdir != None:
1433 flags.extend(["-outputdir", getTestOpts().outputdir])
1434
1435 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1436 + config.compiler + "' " \
1437 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1438 + to_compile + ' ' \
1439 + extra_hc_opts + ' ' \
1440 + getTestOpts().extra_hc_opts + ' ' \
1441 + flags \
1442 + ' -fglasgow-exts -o ' + name \
1443 + '>' + errname + ' 2>&1'
1444
1445 result = runCmdFor(name, cmd)
1446 exit_code = result >> 8
1447
1448 if exit_code != 0:
1449 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1450 if_verbose_dump(1,qerrname)
1451 return failBecause('ext core exit code non-0')
1452
1453 # Clean up
1454 rm_no_fail ( oname )
1455 rm_no_fail ( hcname )
1456 rm_no_fail ( qcorefilename )
1457 rm_no_fail ( depsfilename )
1458
1459 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1460
1461 # -----------------------------------------------------------------------------
1462 # Utils
1463
1464 def check_stdout_ok( name ):
1465 if getTestOpts().with_namebase == None:
1466 namebase = name
1467 else:
1468 namebase = getTestOpts().with_namebase
1469
1470 actual_stdout_file = qualify(name, 'run.stdout')
1471 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1472
1473 def norm(str):
1474 if platform_specific:
1475 return str
1476 else:
1477 return normalise_output(str)
1478
1479 return compare_outputs('stdout', \
1480 two_normalisers(norm, getTestOpts().extra_normaliser), \
1481 expected_stdout_file, actual_stdout_file)
1482
1483 def dump_stdout( name ):
1484 print 'Stdout:'
1485 print read_no_crs(qualify(name, 'run.stdout'))
1486
1487 def check_stderr_ok( name ):
1488 if getTestOpts().with_namebase == None:
1489 namebase = name
1490 else:
1491 namebase = getTestOpts().with_namebase
1492
1493 actual_stderr_file = qualify(name, 'run.stderr')
1494 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1495
1496 def norm(str):
1497 if platform_specific:
1498 return str
1499 else:
1500 return normalise_errmsg(str)
1501
1502 return compare_outputs('stderr', \
1503 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1504 expected_stderr_file, actual_stderr_file)
1505
1506 def dump_stderr( name ):
1507 print "Stderr:"
1508 print read_no_crs(qualify(name, 'run.stderr'))
1509
1510 def read_no_crs(file):
1511 str = ''
1512 try:
1513 h = open(file)
1514 str = h.read()
1515 h.close
1516 except:
1517 # On Windows, if the program fails very early, it seems the
1518 # files stdout/stderr are redirected to may not get created
1519 pass
1520 return re.sub('\r', '', str)
1521
1522 def write_file(file, str):
1523 h = open(file, 'w')
1524 h.write(str)
1525 h.close
1526
1527 def check_hp_ok(name):
1528
1529 # do not qualify for hp2ps because we should be in the right directory
1530 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1531
1532 hp2psResult = runCmdExitCode(hp2psCmd)
1533
1534 actual_ps_file = qualify(name, 'ps')
1535
1536 if(hp2psResult == 0):
1537 if (os.path.exists(actual_ps_file)):
1538 if gs_working:
1539 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1540 if (gsResult == 0):
1541 return (True)
1542 else:
1543 print "hp2ps output for " + name + "is not valid PostScript"
1544 else: return (True) # assume postscript is valid without ghostscript
1545 else:
1546 print "hp2ps did not generate PostScript for " + name
1547 return (False)
1548 else:
1549 print "hp2ps error when processing heap profile for " + name
1550 return(False)
1551
1552 def check_prof_ok(name):
1553
1554 prof_file = qualify(name,'prof')
1555
1556 if not os.path.exists(prof_file):
1557 print prof_file + " does not exist"
1558 return(False)
1559
1560 if os.path.getsize(qualify(name,'prof')) == 0:
1561 print prof_file + " is empty"
1562 return(False)
1563
1564 if getTestOpts().with_namebase == None:
1565 namebase = name
1566 else:
1567 namebase = getTestOpts().with_namebase
1568
1569 (platform_specific, expected_prof_file) = \
1570 platform_wordsize_qualify(namebase, 'prof.sample')
1571
1572 # sample prof file is not required
1573 if not os.path.exists(expected_prof_file):
1574 return True
1575 else:
1576 return compare_outputs('prof', \
1577 two_normalisers(normalise_whitespace,normalise_prof), \
1578 expected_prof_file, prof_file)
1579
1580 # Compare expected output to actual output, and optionally accept the
1581 # new output. Returns true if output matched or was accepted, false
1582 # otherwise.
1583 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1584 if os.path.exists(expected_file):
1585 expected_raw = read_no_crs(expected_file)
1586 # print "norm:", normaliser(expected_raw)
1587 expected_str = normaliser(expected_raw)
1588 expected_file_for_diff = expected_file
1589 else:
1590 expected_str = ''
1591 expected_file_for_diff = '/dev/null'
1592
1593 actual_raw = read_no_crs(actual_file)
1594 actual_str = normaliser(actual_raw)
1595
1596 if expected_str == actual_str:
1597 return 1
1598 else:
1599 print 'Actual ' + kind + ' output differs from expected:'
1600
1601 if expected_file_for_diff == '/dev/null':
1602 expected_normalised_file = '/dev/null'
1603 else:
1604 expected_normalised_file = expected_file + ".normalised"
1605 write_file(expected_normalised_file, expected_str)
1606
1607 actual_normalised_file = actual_file + ".normalised"
1608 write_file(actual_normalised_file, actual_str)
1609
1610 # Ignore whitespace when diffing. We should only get to this
1611 # point if there are non-whitespace differences
1612 #
1613 # Note we are diffing the *actual* output, not the normalised
1614 # output. The normalised output may have whitespace squashed
1615 # (including newlines) so the diff would be hard to read.
1616 # This does mean that the diff might contain changes that
1617 # would be normalised away.
1618 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1619 ' ' + actual_file )
1620
1621 # If for some reason there were no non-whitespace differences,
1622 # then do a full diff
1623 if r == 0:
1624 r = os.system( 'diff -u ' + expected_file_for_diff + \
1625 ' ' + actual_file )
1626
1627 if config.accept:
1628 print 'Accepting new output.'
1629 write_file(expected_file, actual_raw)
1630 return 1
1631 else:
1632 return 0
1633
1634
1635 def normalise_whitespace( str ):
1636 # Merge contiguous whitespace characters into a single space.
1637 str = re.sub('[ \t\n]+', ' ', str)
1638 return str
1639
1640 def normalise_errmsg( str ):
1641 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1642 # the colon is there because it appears in error messages; this
1643 # hacky solution is used in place of more sophisticated filename
1644 # mangling
1645 str = re.sub('([^\\s])\\.exe', '\\1', str)
1646 # normalise slashes, minimise Windows/Unix filename differences
1647 str = re.sub('\\\\', '/', str)
1648 # The inplace ghc's are called ghc-stage[123] to avoid filename
1649 # collisions, so we need to normalise that to just "ghc"
1650 str = re.sub('ghc-stage[123]', 'ghc', str)
1651 # We sometimes see the name of the integer-gmp package on stderr,
1652 # but this can change (either the implementation name or the
1653 # version number), so we canonicalise it here
1654 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1655 return str
1656
1657 # normalise a .prof file, so that we can reasonably compare it against
1658 # a sample. This doesn't compare any of the actual profiling data,
1659 # only the shape of the profile and the number of entries.
1660 def normalise_prof (str):
1661 # strip everything up to the line beginning "COST CENTRE"
1662 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1663
1664 # strip results for CAFs, these tend to change unpredictably
1665 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1666
1667 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1668 # sometimes under MAIN.
1669 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1670
1671 # We have somthing like this:
1672
1673 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1674 # k Main 204 1 0.0 0.0 0.0 0.0
1675 # foo Main 205 1 0.0 0.0 0.0 0.0
1676 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1677
1678 # then we remove all the specific profiling data, leaving only the
1679 # cost centre name, module, and entries, to end up with this:
1680
1681 # MAIN MAIN 0
1682 # k Main 1
1683 # foo Main 1
1684 # foo.bar Main 1
1685
1686 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1687 return str
1688
1689 def normalise_slashes_( str ):
1690 str = re.sub('\\\\', '/', str)
1691 return str
1692
1693 def normalise_exe_( str ):
1694 str = re.sub('\.exe', '', str)
1695 return str
1696
1697 def normalise_output( str ):
1698 # Remove a .exe extension (for Windows)
1699 # This can occur in error messages generated by the program.
1700 str = re.sub('([^\\s])\\.exe', '\\1', str)
1701 return str
1702
1703 def normalise_asm( str ):
1704 lines = str.split('\n')
1705 # Only keep instructions and labels not starting with a dot.
1706 metadata = re.compile('^[ \t]*\\..*$')
1707 out = []
1708 for line in lines:
1709 # Drop metadata directives (e.g. ".type")
1710 if not metadata.match(line):
1711 line = re.sub('@plt', '', line)
1712 instr = line.lstrip().split()
1713 # Drop empty lines.
1714 if not instr:
1715 continue
1716 # Drop operands, except for call instructions.
1717 elif instr[0] == 'call':
1718 out.append(instr[0] + ' ' + instr[1])
1719 else:
1720 out.append(instr[0])
1721 out = '\n'.join(out)
1722 return out
1723
1724 def if_verbose( n, str ):
1725 if config.verbose >= n:
1726 print str
1727
1728 def if_verbose_dump( n, f ):
1729 if config.verbose >= n:
1730 try:
1731 print open(f).read()
1732 except:
1733 print ''
1734
1735 def rawSystem(cmd_and_args):
1736 # We prefer subprocess.call to os.spawnv as the latter
1737 # seems to send its arguments through a shell or something
1738 # with the Windows (non-cygwin) python. An argument "a b c"
1739 # turns into three arguments ["a", "b", "c"].
1740
1741 # However, subprocess is new in python 2.4, so fall back to
1742 # using spawnv if we don't have it
1743
1744 if have_subprocess:
1745 return subprocess.call(cmd_and_args)
1746 else:
1747 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1748
1749 # Note that this doesn't handle the timeout itself; it is just used for
1750 # commands that have timeout handling built-in.
1751 def rawSystemWithTimeout(cmd_and_args):
1752 r = rawSystem(cmd_and_args)
1753 if r == 98:
1754 # The python timeout program uses 98 to signal that ^C was pressed
1755 stopNow()
1756 return r
1757
1758 # cmd is a complex command in Bourne-shell syntax
1759 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1760 # Hence it must ultimately be run by a Bourne shell
1761 #
1762 # Mostly it invokes the command wrapped in 'timeout' thus
1763 # timeout 300 'cd . && ...blah blah'
1764 # so it's timeout's job to invoke the Bourne shell
1765 #
1766 # But watch out for the case when there is no timeout program!
1767 # Then, when using the native Python, os.system will invoke the cmd shell
1768
1769 def runCmd( cmd ):
1770 if_verbose( 1, cmd )
1771 r = 0
1772 if config.os == 'mingw32':
1773 # On MinGW, we will always have timeout
1774 assert config.timeout_prog!=''
1775
1776 if config.timeout_prog != '':
1777 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1778 else:
1779 r = os.system(cmd)
1780 return r << 8
1781
1782 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1783 if_verbose( 1, cmd )
1784 r = 0
1785 if config.os == 'mingw32':
1786 # On MinGW, we will always have timeout
1787 assert config.timeout_prog!=''
1788 timeout = int(ceil(config.timeout * timeout_multiplier))
1789
1790 if config.timeout_prog != '':
1791 if config.check_files_written:
1792 fn = name + ".strace"
1793 r = rawSystemWithTimeout(
1794 ["strace", "-o", fn, "-fF",
1795 "-e", "creat,open,chdir,clone,vfork",
1796 config.timeout_prog, str(timeout), cmd])
1797 addTestFilesWritten(name, fn)
1798 rm_no_fail(fn)
1799 else:
1800 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1801 else:
1802 r = os.system(cmd)
1803 return r << 8
1804
1805 def runCmdExitCode( cmd ):
1806 return (runCmd(cmd) >> 8);
1807
1808
1809 # -----------------------------------------------------------------------------
1810 # checking for files being written to by multiple tests
1811
1812 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1813 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1814 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1815 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1816 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1817 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1818 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1819 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1820 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1821 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1822 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1823 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1824 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1825 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1826 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1827 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1828 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1829
1830 # Files that are read or written but shouldn't be:
1831 # * ghci_history shouldn't be read or written by tests
1832 # * things under package.conf.d shouldn't be written by tests
1833 bad_file_usages = {}
1834
1835 # Mapping from tests to the list of files that they write
1836 files_written = {}
1837
1838 # Mapping from tests to the list of files that they write but don't clean
1839 files_written_not_removed = {}
1840
1841 def add_bad_file_usage(name, file):
1842 try:
1843 if not file in bad_file_usages[name]:
1844 bad_file_usages[name].append(file)
1845 except:
1846 bad_file_usages[name] = [file]
1847
1848 def mkPath(curdir, path):
1849 # Given the current full directory is 'curdir', what is the full
1850 # path to 'path'?
1851 return os.path.realpath(os.path.join(curdir, path))
1852
1853 def addTestFilesWritten(name, fn):
1854 if config.use_threads:
1855 with t.lockFilesWritten:
1856 addTestFilesWrittenHelper(name, fn)
1857 else:
1858 addTestFilesWrittenHelper(name, fn)
1859
1860 def addTestFilesWrittenHelper(name, fn):
1861 started = False
1862 working_directories = {}
1863
1864 with open(fn, 'r') as f:
1865 for line in f:
1866 m_pid = re_strace_pid.match(line)
1867 if m_pid:
1868 pid = m_pid.group(1)
1869 content = m_pid.group(2)
1870 elif re_strace_unavailable.match(line):
1871 next
1872 else:
1873 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1874
1875 m_open = re_strace_open.match(content)
1876 m_chdir = re_strace_chdir.match(content)
1877 m_clone = re_strace_clone.match(content)
1878 m_vfork = re_strace_vfork.match(content)
1879
1880 if not started:
1881 working_directories[pid] = os.getcwd()
1882 started = True
1883
1884 if m_open:
1885 file = m_open.group(1)
1886 file = mkPath(working_directories[pid], file)
1887 if file.endswith("ghci_history"):
1888 add_bad_file_usage(name, file)
1889 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1890 flags = m_open.group(2).split('|')
1891 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1892 if re.match('package\.conf\.d', file):
1893 add_bad_file_usage(name, file)
1894 else:
1895 try:
1896 if not file in files_written[name]:
1897 files_written[name].append(file)
1898 except:
1899 files_written[name] = [file]
1900 elif 'O_RDONLY' in flags:
1901 pass
1902 else:
1903 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1904 elif m_chdir:
1905 # We optimistically assume that unfinished chdir's are going to succeed
1906 dir = m_chdir.group(1)
1907 working_directories[pid] = mkPath(working_directories[pid], dir)
1908 elif m_clone:
1909 working_directories[m_clone.group(2)] = working_directories[pid]
1910 elif m_vfork:
1911 working_directories[m_vfork.group(2)] = working_directories[pid]
1912 elif re_strace_open_resumed.match(content):
1913 pass
1914 elif re_strace_chdir_resumed.match(content):
1915 pass
1916 elif re_strace_vfork_unfinished.match(content):
1917 pass
1918 elif re_strace_clone_unfinished.match(content):
1919 pass
1920 elif re_strace_ignore_sigchild.match(content):
1921 pass
1922 elif re_strace_ignore_sigvtalarm.match(content):
1923 pass
1924 elif re_strace_ignore_sigint.match(content):
1925 pass
1926 elif re_strace_ignore_sigfpe.match(content):
1927 pass
1928 elif re_strace_ignore_sigsegv.match(content):
1929 pass
1930 elif re_strace_ignore_sigpipe.match(content):
1931 pass
1932 else:
1933 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1934
1935 def checkForFilesWrittenProblems(file):
1936 foundProblem = False
1937
1938 files_written_inverted = {}
1939 for t in files_written.keys():
1940 for f in files_written[t]:
1941 try:
1942 files_written_inverted[f].append(t)
1943 except:
1944 files_written_inverted[f] = [t]
1945
1946 for f in files_written_inverted.keys():
1947 if len(files_written_inverted[f]) > 1:
1948 if not foundProblem:
1949 foundProblem = True
1950 file.write("\n")
1951 file.write("\nSome files are written by multiple tests:\n")
1952 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1953 if foundProblem:
1954 file.write("\n")
1955
1956 # -----
1957
1958 if len(files_written_not_removed) > 0:
1959 file.write("\n")
1960 file.write("\nSome files written but not removed:\n")
1961 tests = files_written_not_removed.keys()
1962 tests.sort()
1963 for t in tests:
1964 for f in files_written_not_removed[t]:
1965 file.write(" " + t + ": " + f + "\n")
1966 file.write("\n")
1967
1968 # -----
1969
1970 if len(bad_file_usages) > 0:
1971 file.write("\n")
1972 file.write("\nSome bad file usages:\n")
1973 tests = bad_file_usages.keys()
1974 tests.sort()
1975 for t in tests:
1976 for f in bad_file_usages[t]:
1977 file.write(" " + t + ": " + f + "\n")
1978 file.write("\n")
1979
1980 # -----------------------------------------------------------------------------
1981 # checking if ghostscript is available for checking the output of hp2ps
1982
1983 def genGSCmd(psfile):
1984 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1985
1986 def gsNotWorking():
1987 global gs_working
1988 print "GhostScript not available for hp2ps tests"
1989
1990 global gs_working
1991 gs_working = 0
1992 if config.have_profiling:
1993 if config.gs != '':
1994 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
1995 if resultGood == 0:
1996 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
1997 if resultBad != 0:
1998 print "GhostScript available for hp2ps tests"
1999 gs_working = 1;
2000 else:
2001 gsNotWorking();
2002 else:
2003 gsNotWorking();
2004 else:
2005 gsNotWorking();
2006
2007 def rm_no_fail( file ):
2008 try:
2009 os.remove( file )
2010 finally:
2011 return
2012
2013 def add_suffix( name, suffix ):
2014 if suffix == '':
2015 return name
2016 else:
2017 return name + '.' + suffix
2018
2019 def add_hs_lhs_suffix(name):
2020 if getTestOpts().c_src:
2021 return add_suffix(name, 'c')
2022 elif getTestOpts().cmm_src:
2023 return add_suffix(name, 'cmm')
2024 elif getTestOpts().objc_src:
2025 return add_suffix(name, 'm')
2026 elif getTestOpts().objcpp_src:
2027 return add_suffix(name, 'mm')
2028 elif getTestOpts().literate:
2029 return add_suffix(name, 'lhs')
2030 else:
2031 return add_suffix(name, 'hs')
2032
2033 def replace_suffix( name, suffix ):
2034 base, suf = os.path.splitext(name)
2035 return base + '.' + suffix
2036
2037 def in_testdir( name ):
2038 return (getTestOpts().testdir + '/' + name)
2039
2040 def qualify( name, suff ):
2041 return in_testdir(add_suffix(name, suff))
2042
2043
2044 # Finding the sample output. The filename is of the form
2045 #
2046 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2047 #
2048 # and we pick the most specific version available. The <version> is
2049 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2050 # more fine-grained control use if_compiler_lt().
2051 #
2052 def platform_wordsize_qualify( name, suff ):
2053
2054 basepath = qualify(name, suff)
2055
2056 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2057 for (platformSpecific, plat) in [(1, '-' + config.platform),
2058 (1, '-' + config.os),
2059 (0, '')]
2060 for ws in ['-ws-' + config.wordsize, '']
2061 for comp in ['-' + config.compiler_type, '']
2062 for vers in ['-' + config.compiler_maj_version, '']]
2063
2064 dir = glob.glob(basepath + '*')
2065 dir = map (lambda d: normalise_slashes_(d), dir)
2066
2067 for (platformSpecific, f) in paths:
2068 if f in dir:
2069 return (platformSpecific,f)
2070
2071 return (0, basepath)
2072
2073 # Clean up prior to the test, so that we can't spuriously conclude
2074 # that it passed on the basis of old run outputs.
2075 def pretest_cleanup(name):
2076 if getTestOpts().outputdir != None:
2077 odir = in_testdir(getTestOpts().outputdir)
2078 try:
2079 shutil.rmtree(odir)
2080 except:
2081 pass
2082 os.mkdir(odir)
2083
2084 rm_no_fail(qualify(name,'comp.stderr'))
2085 rm_no_fail(qualify(name,'run.stderr'))
2086 rm_no_fail(qualify(name,'run.stdout'))
2087 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2088 # simple_build zaps the following:
2089 # rm_nofail(qualify("o"))
2090 # rm_nofail(qualify(""))
2091 # not interested in the return code
2092
2093 # -----------------------------------------------------------------------------
2094 # Return a list of all the files ending in '.T' below the directory dir.
2095
2096 def findTFiles(roots):
2097 return concat(map(findTFiles_,roots))
2098
2099 def findTFiles_(path):
2100 if os.path.isdir(path):
2101 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2102 return findTFiles(paths)
2103 elif path[-2:] == '.T':
2104 return [path]
2105 else:
2106 return []
2107
2108 # -----------------------------------------------------------------------------
2109 # Output a test summary to the specified file object
2110
2111 def summary(t, file):
2112
2113 file.write('\n')
2114 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2115 file.write('OVERALL SUMMARY for test run started at '
2116 + t.start_time + '\n'
2117 + string.rjust(`t.total_tests`, 8)
2118 + ' total tests, which gave rise to\n'
2119 + string.rjust(`t.total_test_cases`, 8)
2120 + ' test cases, of which\n'
2121 + string.rjust(`t.n_tests_skipped`, 8)
2122 + ' were skipped\n'
2123 + '\n'
2124 + string.rjust(`t.n_missing_libs`, 8)
2125 + ' had missing libraries\n'
2126 + string.rjust(`t.n_expected_passes`, 8)
2127 + ' expected passes\n'
2128 + string.rjust(`t.n_expected_failures`, 8)
2129 + ' expected failures\n'
2130 + '\n'
2131 + string.rjust(`t.n_framework_failures`, 8)
2132 + ' caused framework failures\n'
2133 + string.rjust(`t.n_unexpected_passes`, 8)
2134 + ' unexpected passes\n'
2135 + string.rjust(`t.n_unexpected_failures`, 8)
2136 + ' unexpected failures\n'
2137 + '\n')
2138
2139 if t.n_unexpected_passes > 0:
2140 file.write('Unexpected passes:\n')
2141 printPassingTestInfosSummary(file, t.unexpected_passes)
2142
2143 if t.n_unexpected_failures > 0:
2144 file.write('Unexpected failures:\n')
2145 printFailingTestInfosSummary(file, t.unexpected_failures)
2146
2147 if config.check_files_written:
2148 checkForFilesWrittenProblems(file)
2149
2150 if stopping():
2151 file.write('WARNING: Testsuite run was terminated early\n')
2152
2153 def printUnexpectedTests(file, testInfoss):
2154 unexpected = []
2155 for testInfos in testInfoss:
2156 directories = testInfos.keys()
2157 for directory in directories:
2158 tests = testInfos[directory].keys()
2159 unexpected += tests
2160 if unexpected != []:
2161 file.write('Unexpected results from:\n')
2162 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2163 file.write('\n')
2164
2165 def printPassingTestInfosSummary(file, testInfos):
2166 directories = testInfos.keys()
2167 directories.sort()
2168 maxDirLen = max(map ((lambda x : len(x)), directories))
2169 for directory in directories:
2170 tests = testInfos[directory].keys()
2171 tests.sort()
2172 for test in tests:
2173 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2174 ' (' + join(testInfos[directory][test],',') + ')\n')
2175 file.write('\n')
2176
2177 def printFailingTestInfosSummary(file, testInfos):
2178 directories = testInfos.keys()
2179 directories.sort()
2180 maxDirLen = max(map ((lambda x : len(x)), directories))
2181 for directory in directories:
2182 tests = testInfos[directory].keys()
2183 tests.sort()
2184 for test in tests:
2185 reasons = testInfos[directory][test].keys()
2186 for reason in reasons:
2187 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2188 ' [' + reason + ']' + \
2189 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2190 file.write('\n')
2191
2192 def getStdout(cmd):
2193 if have_subprocess:
2194 p = subprocess.Popen(cmd,
2195 stdout=subprocess.PIPE,
2196 stderr=subprocess.PIPE)
2197 (stdout, stderr) = p.communicate()
2198 r = p.wait()
2199 if r != 0:
2200 raise Exception("Command failed: " + str(cmd))
2201 if stderr != '':
2202 raise Exception("stderr from command: " + str(cmd))
2203 return stdout
2204 else:
2205 raise Exception("Need subprocess to get stdout, but don't have it")