Merge branch 'master' of darcs.haskell.org:/srv/darcs//testsuite
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = [thisdir_settings, f]
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 record_broken(name, opts, bug)
155 opts.expect = 'fail';
156
157 def expect_broken_for( bug, ways ):
158 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
159
160 def _expect_broken_for( name, opts, bug, ways ):
161 record_broken(name, opts, bug)
162 opts.expect_fail_for = ways
163
164 def record_broken(name, opts, bug):
165 global brokens
166 me = (bug, opts.testdir, name)
167 if not me in brokens:
168 brokens.append(me)
169
170 # -----
171
172 def omit_ways( ways ):
173 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174
175 def _omit_ways( name, opts, ways ):
176 opts.omit_ways = ways
177
178 # -----
179
180 def only_ways( ways ):
181 return lambda name, opts, w=ways: _only_ways( name, opts, w )
182
183 def _only_ways( name, opts, ways ):
184 opts.only_ways = ways
185
186 # -----
187
188 def extra_ways( ways ):
189 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190
191 def _extra_ways( name, opts, ways ):
192 opts.extra_ways = ways
193
194 # -----
195
196 def omit_compiler_types( compiler_types ):
197 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
198
199 def _omit_compiler_types( name, opts, compiler_types ):
200 if config.compiler_type in compiler_types:
201 opts.skip = 1
202
203 # -----
204
205 def only_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
207
208 def _only_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type not in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def set_stdin( file ):
215 return lambda name, opts, f=file: _set_stdin(name, opts, f);
216
217 def _set_stdin( name, opts, f ):
218 opts.stdin = f
219
220 # -----
221
222 def exit_code( val ):
223 return lambda name, opts, v=val: _exit_code(name, opts, v);
224
225 def _exit_code( name, opts, v ):
226 opts.exit_code = v
227
228 # -----
229
230 def timeout_multiplier( val ):
231 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232
233 def _timeout_multiplier( name, opts, v ):
234 opts.timeout_multiplier = v
235
236 # -----
237
238 def extra_run_opts( val ):
239 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240
241 def _extra_run_opts( name, opts, v ):
242 opts.extra_run_opts = v
243
244 # -----
245
246 def extra_hc_opts( val ):
247 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
248
249 def _extra_hc_opts( name, opts, v ):
250 opts.extra_hc_opts = v
251
252 # -----
253
254 def extra_clean( files ):
255 return lambda name, opts, v=files: _extra_clean(name, opts, v);
256
257 def _extra_clean( name, opts, v ):
258 opts.clean_files = v
259
260 # -----
261
262 def stats_num_field( field, expecteds ):
263 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
264
265 def _stats_num_field( name, opts, field, expecteds ):
266 if field in opts.stats_range_fields:
267 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
268
269 if type(expecteds) is types.ListType:
270 for (b, expected, dev) in expecteds:
271 if b:
272 opts.stats_range_fields[field] = (expected, dev)
273 return
274 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275
276 else:
277 (expected, dev) = expecteds
278 opts.stats_range_fields[field] = (expected, dev)
279
280 def compiler_stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282
283 def _compiler_stats_num_field( name, opts, field, expecteds ):
284 if field in opts.compiler_stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 for (b, expected, dev) in expecteds:
288 if b:
289 opts.compiler_stats_range_fields[field] = (expected, dev)
290 return
291
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 # -----
295
296 def when(b, f):
297 # When list_brokens is on, we want to see all expect_broken calls,
298 # so we always do f
299 if b or config.list_broken:
300 return f
301 else:
302 return normal
303
304 def unless(b, f):
305 return when(not b, f)
306
307 def doing_ghci():
308 return 'ghci' in config.run_ways
309
310 def ghci_dynamic( ):
311 return config.ghc_dynamic_by_default
312
313 def fast():
314 return config.fast
315
316 def platform( plat ):
317 return config.platform == plat
318
319 def opsys( os ):
320 return config.os == os
321
322 def arch( arch ):
323 return config.arch == arch
324
325 def wordsize( ws ):
326 return config.wordsize == str(ws)
327
328 def msys( ):
329 return config.msys
330
331 def cygwin( ):
332 return config.cygwin
333
334 def have_vanilla( ):
335 return config.have_vanilla
336
337 def have_dynamic( ):
338 return config.have_dynamic
339
340 def have_profiling( ):
341 return config.have_profiling
342
343 def in_tree_compiler( ):
344 return config.in_tree_compiler
345
346 def compiler_type( compiler ):
347 return config.compiler_type == compiler
348
349 def compiler_lt( compiler, version ):
350 return config.compiler_type == compiler and \
351 version_lt(config.compiler_version, version)
352
353 def compiler_le( compiler, version ):
354 return config.compiler_type == compiler and \
355 version_le(config.compiler_version, version)
356
357 def compiler_gt( compiler, version ):
358 return config.compiler_type == compiler and \
359 version_gt(config.compiler_version, version)
360
361 def compiler_ge( compiler, version ):
362 return config.compiler_type == compiler and \
363 version_ge(config.compiler_version, version)
364
365 def unregisterised( ):
366 return config.unregisterised
367
368 def compiler_profiled( ):
369 return config.compiler_profiled
370
371 def compiler_debugged( ):
372 return config.compiler_debugged
373
374 def tag( t ):
375 return t in config.compiler_tags
376
377 # ---
378
379 def namebase( nb ):
380 return lambda opts, nb=nb: _namebase(opts, nb)
381
382 def _namebase( opts, nb ):
383 opts.with_namebase = nb
384
385 # ---
386
387 def high_memory_usage(name, opts):
388 opts.alone = True
389
390 # ---
391 def literate( name, opts ):
392 opts.literate = 1;
393
394 def c_src( name, opts ):
395 opts.c_src = 1;
396
397 def objc_src( name, opts ):
398 opts.objc_src = 1;
399
400 def objcpp_src( name, opts ):
401 opts.objcpp_src = 1;
402
403 def cmm_src( name, opts ):
404 opts.cmm_src = 1;
405
406 def outputdir( odir ):
407 return lambda name, opts, d=odir: _outputdir(name, opts, d)
408
409 def _outputdir( name, opts, odir ):
410 opts.outputdir = odir;
411
412 # ----
413
414 def pre_cmd( cmd ):
415 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
416
417 def _pre_cmd( name, opts, cmd ):
418 opts.pre_cmd = cmd
419
420 # ----
421
422 def clean_cmd( cmd ):
423 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
424
425 def _clean_cmd( name, opts, cmd ):
426 opts.clean_cmd = cmd
427
428 # ----
429
430 def cmd_prefix( prefix ):
431 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
432
433 def _cmd_prefix( name, opts, prefix ):
434 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
435
436 # ----
437
438 def cmd_wrapper( fun ):
439 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
440
441 def _cmd_wrapper( name, opts, fun ):
442 opts.cmd_wrapper = fun
443
444 # ----
445
446 def compile_cmd_prefix( prefix ):
447 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
448
449 def _compile_cmd_prefix( name, opts, prefix ):
450 opts.compile_cmd_prefix = prefix
451
452 # ----
453
454 def normalise_slashes( name, opts ):
455 opts.extra_normaliser = normalise_slashes_
456
457 def normalise_exe( name, opts ):
458 opts.extra_normaliser = normalise_exe_
459
460 def normalise_fun( fun ):
461 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
462
463 def _normalise_fun( name, opts, f ):
464 opts.extra_normaliser = f
465
466 def normalise_errmsg_fun( fun ):
467 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
468
469 def _normalise_errmsg_fun( name, opts, f ):
470 opts.extra_errmsg_normaliser = f
471
472 def two_normalisers(f, g):
473 return lambda x, f=f, g=g: f(g(x))
474
475 # ----
476 # Function for composing two opt-fns together
477
478 def executeSetups(fs, name, opts):
479 if type(fs) is types.ListType:
480 # If we have a list of setups, then execute each one
481 map (lambda f : executeSetups(f, name, opts), fs)
482 else:
483 # fs is a single function, so just apply it
484 fs(name, opts)
485
486 # -----------------------------------------------------------------------------
487 # The current directory of tests
488
489 def newTestDir( dir ):
490 global thisdir_settings
491 # reset the options for this test directory
492 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
493
494 def _newTestDir( name, opts, dir ):
495 opts.testdir = dir
496 opts.compiler_always_flags = config.compiler_always_flags
497
498 # -----------------------------------------------------------------------------
499 # Actually doing tests
500
501 parallelTests = []
502 aloneTests = []
503 allTestNames = set([])
504
505 def runTest (opts, name, func, args):
506 ok = 0
507
508 if config.use_threads:
509 t.thread_pool.acquire()
510 try:
511 while config.threads<(t.running_threads+1):
512 t.thread_pool.wait()
513 t.running_threads = t.running_threads+1
514 ok=1
515 t.thread_pool.release()
516 thread.start_new_thread(test_common_thread, (name, opts, func, args))
517 except:
518 if not ok:
519 t.thread_pool.release()
520 else:
521 test_common_work (name, opts, func, args)
522
523 # name :: String
524 # setup :: TestOpts -> IO ()
525 def test (name, setup, func, args):
526 global aloneTests
527 global parallelTests
528 global allTestNames
529 global thisdir_settings
530 if name in allTestNames:
531 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
532 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
533 framework_fail(name, 'bad_name', 'This test has an invalid name')
534
535 # Make a deep copy of the default_testopts, as we need our own copy
536 # of any dictionaries etc inside it. Otherwise, if one test modifies
537 # them, all tests will see the modified version!
538 myTestOpts = copy.deepcopy(default_testopts)
539
540 executeSetups([thisdir_settings, setup], name, myTestOpts)
541
542 thisTest = lambda : runTest(myTestOpts, name, func, args)
543 if myTestOpts.alone:
544 aloneTests.append(thisTest)
545 else:
546 parallelTests.append(thisTest)
547 allTestNames.add(name)
548
549 if config.use_threads:
550 def test_common_thread(name, opts, func, args):
551 t.lock.acquire()
552 try:
553 test_common_work(name,opts,func,args)
554 finally:
555 t.lock.release()
556 t.thread_pool.acquire()
557 t.running_threads = t.running_threads - 1
558 t.thread_pool.notify()
559 t.thread_pool.release()
560
561 def get_package_cache_timestamp():
562 if config.package_conf_cache_file == '':
563 return 0.0
564 else:
565 try:
566 return os.stat(config.package_conf_cache_file).st_mtime
567 except:
568 return 0.0
569
570
571 def test_common_work (name, opts, func, args):
572 try:
573 t.total_tests = t.total_tests+1
574 setLocalTestOpts(opts)
575
576 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
577
578 # All the ways we might run this test
579 if func == compile or func == multimod_compile:
580 all_ways = config.compile_ways
581 elif func == compile_and_run or func == multimod_compile_and_run:
582 all_ways = config.run_ways
583 elif func == ghci_script:
584 if 'ghci' in config.run_ways:
585 all_ways = ['ghci']
586 else:
587 all_ways = []
588 else:
589 all_ways = ['normal']
590
591 # A test itself can request extra ways by setting opts.extra_ways
592 all_ways = all_ways + filter(lambda way: way not in all_ways,
593 opts.extra_ways)
594
595 t.total_test_cases = t.total_test_cases + len(all_ways)
596
597 ok_way = lambda way: \
598 not getTestOpts().skip \
599 and (config.only == [] or name in config.only) \
600 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
601 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
602 and way not in getTestOpts().omit_ways
603
604 # Which ways we are asked to skip
605 do_ways = filter (ok_way,all_ways)
606
607 # In fast mode, we skip all but one way
608 if config.fast and len(do_ways) > 0:
609 do_ways = [do_ways[0]]
610
611 if not config.clean_only:
612 # Run the required tests...
613 for way in do_ways:
614 if stopping():
615 break
616 do_test (name, way, func, args)
617
618 for way in all_ways:
619 if way not in do_ways:
620 skiptest (name,way)
621
622 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
623 clean(map (lambda suff: name + suff,
624 ['', '.exe', '.exe.manifest', '.genscript',
625 '.stderr.normalised', '.stdout.normalised',
626 '.run.stderr', '.run.stdout',
627 '.run.stderr.normalised', '.run.stdout.normalised',
628 '.comp.stderr', '.comp.stdout',
629 '.comp.stderr.normalised', '.comp.stdout.normalised',
630 '.interp.stderr', '.interp.stdout',
631 '.interp.stderr.normalised', '.interp.stdout.normalised',
632 '.stats', '.comp.stats',
633 '.hi', '.o', '.prof', '.exe.prof', '.hc',
634 '_stub.h', '_stub.c', '_stub.o',
635 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
636
637 if func == multi_compile or func == multi_compile_fail:
638 extra_mods = args[1]
639 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
640 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
641
642 clean(getTestOpts().clean_files)
643
644 if getTestOpts().outputdir != None:
645 odir = in_testdir(getTestOpts().outputdir)
646 try:
647 shutil.rmtree(odir)
648 except:
649 pass
650
651 try:
652 shutil.rmtree(in_testdir('.hpc.' + name))
653 except:
654 pass
655
656 try:
657 cleanCmd = getTestOpts().clean_cmd
658 if cleanCmd != None:
659 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
660 if result != 0:
661 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
662 except e:
663 framework_fail(name, 'cleaning', 'clean-command exception')
664
665 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
666
667 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
668 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
669
670 try:
671 for f in files_written[name]:
672 if os.path.exists(f):
673 try:
674 if not f in files_written_not_removed[name]:
675 files_written_not_removed[name].append(f)
676 except:
677 files_written_not_removed[name] = [f]
678 except:
679 pass
680 except Exception, e:
681 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
682
683 def clean(strs):
684 for str in strs:
685 for name in glob.glob(in_testdir(str)):
686 clean_full_path(name)
687
688 def clean_full_path(name):
689 try:
690 # Remove files...
691 os.remove(name)
692 except OSError, e1:
693 try:
694 # ... and empty directories
695 os.rmdir(name)
696 except OSError, e2:
697 # We don't want to fail here, but we do want to know
698 # what went wrong, so print out the exceptions.
699 # ENOENT isn't a problem, though, as we clean files
700 # that don't necessarily exist.
701 if e1.errno != errno.ENOENT:
702 print e1
703 if e2.errno != errno.ENOENT:
704 print e2
705
706 def do_test(name, way, func, args):
707 full_name = name + '(' + way + ')'
708
709 try:
710 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
711 str([t.n_unexpected_passes, \
712 t.n_unexpected_failures, \
713 t.n_framework_failures])
714
715 if config.use_threads:
716 t.lock.release()
717
718 try:
719 preCmd = getTestOpts().pre_cmd
720 if preCmd != None:
721 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
722 if result != 0:
723 framework_fail(name, way, 'pre-command failed: ' + str(result))
724 except e:
725 framework_fail(name, way, 'pre-command exception')
726
727 try:
728 result = apply(func, [name,way] + args)
729 finally:
730 if config.use_threads:
731 t.lock.acquire()
732
733 if getTestOpts().expect != 'pass' and \
734 getTestOpts().expect != 'fail' and \
735 getTestOpts().expect != 'missing-lib':
736 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
737
738 try:
739 passFail = result['passFail']
740 except:
741 passFail = 'No passFail found'
742
743 if passFail == 'pass':
744 if getTestOpts().expect == 'pass' \
745 and way not in getTestOpts().expect_fail_for:
746 t.n_expected_passes = t.n_expected_passes + 1
747 if name in t.expected_passes:
748 t.expected_passes[name].append(way)
749 else:
750 t.expected_passes[name] = [way]
751 else:
752 print '*** unexpected pass for', full_name
753 t.n_unexpected_passes = t.n_unexpected_passes + 1
754 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
755 elif passFail == 'fail':
756 if getTestOpts().expect == 'pass' \
757 and way not in getTestOpts().expect_fail_for:
758 print '*** unexpected failure for', full_name
759 t.n_unexpected_failures = t.n_unexpected_failures + 1
760 reason = result['reason']
761 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
762 else:
763 if getTestOpts().expect == 'missing-lib':
764 t.n_missing_libs = t.n_missing_libs + 1
765 if name in t.missing_libs:
766 t.missing_libs[name].append(way)
767 else:
768 t.missing_libs[name] = [way]
769 else:
770 t.n_expected_failures = t.n_expected_failures + 1
771 if name in t.expected_failures:
772 t.expected_failures[name].append(way)
773 else:
774 t.expected_failures[name] = [way]
775 else:
776 framework_fail(name, way, 'bad result ' + passFail)
777 except KeyboardInterrupt:
778 stopNow()
779 except:
780 framework_fail(name, way, 'do_test exception')
781 traceback.print_exc()
782
783 def addPassingTestInfo (testInfos, directory, name, way):
784 directory = re.sub('^\\.[/\\\\]', '', directory)
785
786 if not directory in testInfos:
787 testInfos[directory] = {}
788
789 if not name in testInfos[directory]:
790 testInfos[directory][name] = []
791
792 testInfos[directory][name].append(way)
793
794 def addFailingTestInfo (testInfos, directory, name, reason, way):
795 directory = re.sub('^\\.[/\\\\]', '', directory)
796
797 if not directory in testInfos:
798 testInfos[directory] = {}
799
800 if not name in testInfos[directory]:
801 testInfos[directory][name] = {}
802
803 if not reason in testInfos[directory][name]:
804 testInfos[directory][name][reason] = []
805
806 testInfos[directory][name][reason].append(way)
807
808 def skiptest (name, way):
809 # print 'Skipping test \"', name, '\"'
810 t.n_tests_skipped = t.n_tests_skipped + 1
811 if name in t.tests_skipped:
812 t.tests_skipped[name].append(way)
813 else:
814 t.tests_skipped[name] = [way]
815
816 def framework_fail( name, way, reason ):
817 full_name = name + '(' + way + ')'
818 print '*** framework failure for', full_name, reason
819 t.n_framework_failures = t.n_framework_failures + 1
820 if name in t.framework_failures:
821 t.framework_failures[name].append(way)
822 else:
823 t.framework_failures[name] = [way]
824
825 def badResult(result):
826 try:
827 if result['passFail'] == 'pass':
828 return False
829 return True
830 except:
831 return True
832
833 def passed():
834 return {'passFail': 'pass'}
835
836 def failBecause(reason):
837 return {'passFail': 'fail', 'reason': reason}
838
839 # -----------------------------------------------------------------------------
840 # Generic command tests
841
842 # A generic command test is expected to run and exit successfully.
843 #
844 # The expected exit code can be changed via exit_code() as normal, and
845 # the expected stdout/stderr are stored in <testname>.stdout and
846 # <testname>.stderr. The output of the command can be ignored
847 # altogether by using run_command_ignore_output instead of
848 # run_command.
849
850 def run_command( name, way, cmd ):
851 return simple_run( name, '', cmd, '' )
852
853 # -----------------------------------------------------------------------------
854 # GHCi tests
855
856 def ghci_script( name, way, script ):
857 # filter out -fforce-recomp from compiler_always_flags, because we're
858 # actually testing the recompilation behaviour in the GHCi tests.
859 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
860 flags.append(getTestOpts().extra_hc_opts)
861 if getTestOpts().outputdir != None:
862 flags.extend(["-outputdir", getTestOpts().outputdir])
863
864 # We pass HC and HC_OPTS as environment variables, so that the
865 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
866 cmd = "HC='" + config.compiler + "' " + \
867 "HC_OPTS='" + join(flags,' ') + "' " + \
868 "'" + config.compiler + "'" + \
869 ' --interactive -v0 -ignore-dot-ghci ' + \
870 join(flags,' ')
871
872 getTestOpts().stdin = script
873 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
874
875 # -----------------------------------------------------------------------------
876 # Compile-only tests
877
878 def compile( name, way, extra_hc_opts ):
879 return do_compile( name, way, 0, '', [], extra_hc_opts )
880
881 def compile_fail( name, way, extra_hc_opts ):
882 return do_compile( name, way, 1, '', [], extra_hc_opts )
883
884 def multimod_compile( name, way, top_mod, extra_hc_opts ):
885 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
886
887 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
888 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
889
890 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
891 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
892
893 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
894 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
895
896 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
897 # print 'Compile only, extra args = ', extra_hc_opts
898 pretest_cleanup(name)
899
900 result = extras_build( way, extra_mods, extra_hc_opts )
901 if badResult(result):
902 return result
903 extra_hc_opts = result['hc_opts']
904
905 force = 0
906 if extra_mods:
907 force = 1
908 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
909
910 if badResult(result):
911 return result
912
913 # the actual stderr should always match the expected, regardless
914 # of whether we expected the compilation to fail or not (successful
915 # compilations may generate warnings).
916
917 if getTestOpts().with_namebase == None:
918 namebase = name
919 else:
920 namebase = getTestOpts().with_namebase
921
922 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
923 actual_stderr_file = qualify(name, 'comp.stderr')
924
925 if not compare_outputs('stderr', \
926 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
927 expected_stderr_file, actual_stderr_file):
928 return failBecause('stderr mismatch')
929
930 # no problems found, this test passed
931 return passed()
932
933 def compile_cmp_asm( name, way, extra_hc_opts ):
934 print 'Compile only, extra args = ', extra_hc_opts
935 pretest_cleanup(name)
936 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
937
938 if badResult(result):
939 return result
940
941 # the actual stderr should always match the expected, regardless
942 # of whether we expected the compilation to fail or not (successful
943 # compilations may generate warnings).
944
945 if getTestOpts().with_namebase == None:
946 namebase = name
947 else:
948 namebase = getTestOpts().with_namebase
949
950 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
951 actual_asm_file = qualify(name, 's')
952
953 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
954 expected_asm_file, actual_asm_file):
955 return failBecause('asm mismatch')
956
957 # no problems found, this test passed
958 return passed()
959
960 # -----------------------------------------------------------------------------
961 # Compile-and-run tests
962
963 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
964 # print 'Compile and run, extra args = ', extra_hc_opts
965 pretest_cleanup(name)
966
967 result = extras_build( way, extra_mods, extra_hc_opts )
968 if badResult(result):
969 return result
970 extra_hc_opts = result['hc_opts']
971
972 if way == 'ghci': # interpreted...
973 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
974 elif way == 'extcore' or way == 'optextcore' :
975 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
976 else: # compiled...
977 force = 0
978 if extra_mods:
979 force = 1
980
981 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
982 if badResult(result):
983 return result
984
985 cmd = './' + name;
986
987 # we don't check the compiler's stderr for a compile-and-run test
988 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
989
990 def compile_and_run( name, way, extra_hc_opts ):
991 return compile_and_run__( name, way, '', [], extra_hc_opts)
992
993 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
994 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
995
996 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
997 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
998
999 def stats( name, way, stats_file ):
1000 opts = getTestOpts()
1001 return checkStats(stats_file, opts.stats_range_fields)
1002
1003 # -----------------------------------------------------------------------------
1004 # Check -t stats info
1005
1006 def checkStats(stats_file, range_fields):
1007 result = passed()
1008 if len(range_fields) > 0:
1009 f = open(in_testdir(stats_file))
1010 contents = f.read()
1011 f.close()
1012
1013 for (field, (expected, dev)) in range_fields.items():
1014 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1015 if m == None:
1016 print 'Failed to find field: ', field
1017 result = failBecause('no such stats field')
1018 val = int(m.group(1))
1019
1020 lowerBound = trunc( expected * ((100 - float(dev))/100));
1021 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1022
1023 if val < lowerBound:
1024 print field, 'value is too low:'
1025 print '(If this is because you have improved GHC, please'
1026 print 'update the test so that GHC doesn\'t regress again)'
1027 result = failBecause('stat too good')
1028 if val > upperBound:
1029 print field, 'value is too high:'
1030 result = failBecause('stat not good enough')
1031
1032 if val < lowerBound or val > upperBound:
1033 valStr = str(val)
1034 valLen = len(valStr)
1035 expectedStr = str(expected)
1036 expectedLen = len(expectedStr)
1037 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1038 def display(descr, val, extra):
1039 print descr, string.rjust(str(val), length), extra
1040 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1041 display(' Lower bound ' + field + ':', lowerBound, '')
1042 display(' Upper bound ' + field + ':', upperBound, '')
1043 display(' Actual ' + field + ':', val, '')
1044
1045 return result
1046
1047 # -----------------------------------------------------------------------------
1048 # Build a single-module program
1049
1050 def extras_build( way, extra_mods, extra_hc_opts ):
1051 for modopts in extra_mods:
1052 mod, opts = modopts
1053 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1054 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1055 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1056 if badResult(result):
1057 return result
1058
1059 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1060
1061
1062 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1063 opts = getTestOpts()
1064 errname = add_suffix(name, 'comp.stderr')
1065 rm_no_fail( qualify(errname, '') )
1066
1067 if top_mod != '':
1068 srcname = top_mod
1069 rm_no_fail( qualify(name, '') )
1070 base, suf = os.path.splitext(top_mod)
1071 rm_no_fail( qualify(base, '') )
1072 rm_no_fail( qualify(base, 'exe') )
1073 elif addsuf:
1074 srcname = add_hs_lhs_suffix(name)
1075 rm_no_fail( qualify(name, '') )
1076 else:
1077 srcname = name
1078 rm_no_fail( qualify(name, 'o') )
1079
1080 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1081
1082 to_do = ''
1083 if top_mod != '':
1084 to_do = '--make '
1085 if link:
1086 to_do = to_do + '-o ' + name
1087 elif link:
1088 to_do = '-o ' + name
1089 elif opts.compile_to_hc:
1090 to_do = '-C'
1091 else:
1092 to_do = '-c' # just compile
1093
1094 stats_file = name + '.comp.stats'
1095 if len(opts.compiler_stats_range_fields) > 0:
1096 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1097
1098 # Required by GHC 7.3+, harmless for earlier versions:
1099 if (getTestOpts().c_src or
1100 getTestOpts().objc_src or
1101 getTestOpts().objcpp_src or
1102 getTestOpts().cmm_src):
1103 extra_hc_opts += ' -no-hs-main '
1104
1105 if getTestOpts().compile_cmd_prefix == '':
1106 cmd_prefix = ''
1107 else:
1108 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1109
1110 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1111 if noforce:
1112 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1113 if getTestOpts().outputdir != None:
1114 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1115
1116 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1117 + config.compiler + "' " \
1118 + join(comp_flags,' ') + ' ' \
1119 + to_do + ' ' + srcname + ' ' \
1120 + join(config.way_flags(name)[way],' ') + ' ' \
1121 + extra_hc_opts + ' ' \
1122 + opts.extra_hc_opts + ' ' \
1123 + '>' + errname + ' 2>&1'
1124
1125 result = runCmdFor(name, cmd)
1126
1127 if result != 0 and not should_fail:
1128 actual_stderr = qualify(name, 'comp.stderr')
1129 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1130 if_verbose_dump(1,actual_stderr)
1131
1132 # ToDo: if the sub-shell was killed by ^C, then exit
1133
1134 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1135
1136 if badResult(statsResult):
1137 return statsResult
1138
1139 if should_fail:
1140 if result == 0:
1141 return failBecause('exit code 0')
1142 else:
1143 if result != 0:
1144 return failBecause('exit code non-0')
1145
1146 return passed()
1147
1148 # -----------------------------------------------------------------------------
1149 # Run a program and check its output
1150 #
1151 # If testname.stdin exists, route input from that, else
1152 # from /dev/null. Route output to testname.run.stdout and
1153 # testname.run.stderr. Returns the exit code of the run.
1154
1155 def simple_run( name, way, prog, args ):
1156 opts = getTestOpts()
1157
1158 # figure out what to use for stdin
1159 if opts.stdin != '':
1160 use_stdin = opts.stdin
1161 else:
1162 stdin_file = add_suffix(name, 'stdin')
1163 if os.path.exists(in_testdir(stdin_file)):
1164 use_stdin = stdin_file
1165 else:
1166 use_stdin = '/dev/null'
1167
1168 run_stdout = add_suffix(name,'run.stdout')
1169 run_stderr = add_suffix(name,'run.stderr')
1170
1171 rm_no_fail(qualify(name,'run.stdout'))
1172 rm_no_fail(qualify(name,'run.stderr'))
1173 rm_no_fail(qualify(name, 'hp'))
1174 rm_no_fail(qualify(name,'ps'))
1175 rm_no_fail(qualify(name, 'prof'))
1176
1177 my_rts_flags = rts_flags(way)
1178
1179 stats_file = name + '.stats'
1180 if len(opts.stats_range_fields) > 0:
1181 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1182
1183 if opts.no_stdin:
1184 stdin_comes_from = ''
1185 else:
1186 stdin_comes_from = ' <' + use_stdin
1187
1188 if opts.combined_output:
1189 redirection = ' >' + run_stdout \
1190 + ' 2>&1'
1191 else:
1192 redirection = ' >' + run_stdout \
1193 + ' 2>' + run_stderr
1194
1195 cmd = prog + ' ' + args + ' ' \
1196 + my_rts_flags + ' ' \
1197 + stdin_comes_from \
1198 + redirection
1199
1200 if opts.cmd_wrapper != None:
1201 cmd = opts.cmd_wrapper(cmd);
1202
1203 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1204
1205 # run the command
1206 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1207
1208 exit_code = result >> 8
1209 signal = result & 0xff
1210
1211 # check the exit code
1212 if exit_code != opts.exit_code:
1213 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1214 dump_stdout(name)
1215 dump_stderr(name)
1216 return failBecause('bad exit code')
1217
1218 check_hp = my_rts_flags.find("-h") != -1
1219 check_prof = my_rts_flags.find("-p") != -1
1220
1221 if not opts.ignore_output:
1222 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1223 bad_stdout = not check_stdout_ok(name)
1224 if bad_stderr:
1225 return failBecause('bad stderr')
1226 if bad_stdout:
1227 return failBecause('bad stdout')
1228 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1229 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1230 return failBecause('bad heap profile')
1231 if check_prof and not check_prof_ok(name):
1232 return failBecause('bad profile')
1233
1234 return checkStats(stats_file, opts.stats_range_fields)
1235
1236 def rts_flags(way):
1237 if (way == ''):
1238 return ''
1239 else:
1240 args = config.way_rts_flags[way]
1241
1242 if args == []:
1243 return ''
1244 else:
1245 return '+RTS ' + join(args,' ') + ' -RTS'
1246
1247 # -----------------------------------------------------------------------------
1248 # Run a program in the interpreter and check its output
1249
1250 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1251 outname = add_suffix(name, 'interp.stdout')
1252 errname = add_suffix(name, 'interp.stderr')
1253 rm_no_fail(outname)
1254 rm_no_fail(errname)
1255 rm_no_fail(name)
1256
1257 if (top_mod == ''):
1258 srcname = add_hs_lhs_suffix(name)
1259 else:
1260 srcname = top_mod
1261
1262 scriptname = add_suffix(name, 'genscript')
1263 qscriptname = in_testdir(scriptname)
1264 rm_no_fail(qscriptname)
1265
1266 delimiter = '===== program output begins here\n'
1267
1268 script = open(qscriptname, 'w')
1269 if not compile_only:
1270 # set the prog name and command-line args to match the compiled
1271 # environment.
1272 script.write(':set prog ' + name + '\n')
1273 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1274 # Add marker lines to the stdout and stderr output files, so we
1275 # can separate GHCi's output from the program's.
1276 script.write(':! echo ' + delimiter)
1277 script.write(':! echo 1>&2 ' + delimiter)
1278 # Set stdout to be line-buffered to match the compiled environment.
1279 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1280 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1281 # in the event of an exception as for the compiled program.
1282 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1283 script.close()
1284
1285 # figure out what to use for stdin
1286 if getTestOpts().stdin != '':
1287 stdin_file = in_testdir(getTestOpts().stdin)
1288 else:
1289 stdin_file = qualify(name, 'stdin')
1290
1291 if os.path.exists(stdin_file):
1292 stdin = open(stdin_file, 'r')
1293 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1294
1295 script.close()
1296
1297 flags = copy.copy(getTestOpts().compiler_always_flags)
1298 if getTestOpts().outputdir != None:
1299 flags.extend(["-outputdir", getTestOpts().outputdir])
1300
1301 cmd = "'" + config.compiler + "' " \
1302 + join(flags,' ') + ' ' \
1303 + srcname + ' ' \
1304 + join(config.way_flags(name)[way],' ') + ' ' \
1305 + extra_hc_opts + ' ' \
1306 + getTestOpts().extra_hc_opts + ' ' \
1307 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1308
1309 if getTestOpts().cmd_wrapper != None:
1310 cmd = getTestOpts().cmd_wrapper(cmd);
1311
1312 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1313
1314 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1315
1316 exit_code = result >> 8
1317 signal = result & 0xff
1318
1319 # split the stdout into compilation/program output
1320 split_file(in_testdir(outname), delimiter,
1321 qualify(name, 'comp.stdout'),
1322 qualify(name, 'run.stdout'))
1323 split_file(in_testdir(errname), delimiter,
1324 qualify(name, 'comp.stderr'),
1325 qualify(name, 'run.stderr'))
1326
1327 # check the exit code
1328 if exit_code != getTestOpts().exit_code:
1329 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1330 dump_stdout(name)
1331 dump_stderr(name)
1332 return failBecause('bad exit code')
1333
1334 # ToDo: if the sub-shell was killed by ^C, then exit
1335
1336 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1337 check_stdout_ok(name)):
1338 return passed()
1339 else:
1340 return failBecause('bad stdout or stderr')
1341
1342
1343 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1344 infile = open(in_fn)
1345 out1 = open(out1_fn, 'w')
1346 out2 = open(out2_fn, 'w')
1347
1348 line = infile.readline()
1349 line = re.sub('\r', '', line) # ignore Windows EOL
1350 while (re.sub('^\s*','',line) != delimiter and line != ''):
1351 out1.write(line)
1352 line = infile.readline()
1353 line = re.sub('\r', '', line)
1354 out1.close()
1355
1356 line = infile.readline()
1357 while (line != ''):
1358 out2.write(line)
1359 line = infile.readline()
1360 out2.close()
1361
1362 # -----------------------------------------------------------------------------
1363 # Generate External Core for the given program, then compile the resulting Core
1364 # and compare its output to the expected output
1365
1366 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1367
1368 depsfilename = qualify(name, 'deps')
1369 errname = add_suffix(name, 'comp.stderr')
1370 qerrname = qualify(errname,'')
1371
1372 hcname = qualify(name, 'hc')
1373 oname = qualify(name, 'o')
1374
1375 rm_no_fail( qerrname )
1376 rm_no_fail( qualify(name, '') )
1377
1378 if (top_mod == ''):
1379 srcname = add_hs_lhs_suffix(name)
1380 else:
1381 srcname = top_mod
1382
1383 qcorefilename = qualify(name, 'hcr')
1384 corefilename = add_suffix(name, 'hcr')
1385 rm_no_fail(qcorefilename)
1386
1387 # Generate External Core
1388
1389 if (top_mod == ''):
1390 to_do = ' ' + srcname + ' '
1391 else:
1392 to_do = ' --make ' + top_mod + ' '
1393
1394 flags = copy.copy(getTestOpts().compiler_always_flags)
1395 if getTestOpts().outputdir != None:
1396 flags.extend(["-outputdir", getTestOpts().outputdir])
1397 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1398 + config.compiler + "' " \
1399 + join(flags,' ') + ' ' \
1400 + join(config.way_flags(name)[way],' ') + ' ' \
1401 + extra_hc_opts + ' ' \
1402 + getTestOpts().extra_hc_opts \
1403 + to_do \
1404 + '>' + errname + ' 2>&1'
1405 result = runCmdFor(name, cmd)
1406
1407 exit_code = result >> 8
1408
1409 if exit_code != 0:
1410 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1411 if_verbose_dump(1,qerrname)
1412 return failBecause('ext core exit code non-0')
1413
1414 # Compile the resulting files -- if there's more than one module, we need to read the output
1415 # of the previous compilation in order to find the dependencies
1416 if (top_mod == ''):
1417 to_compile = corefilename
1418 else:
1419 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1420 deps = open(depsfilename).read()
1421 deplist = string.replace(deps, '\n',' ');
1422 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1423 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1424
1425 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1426 if getTestOpts().outputdir != None:
1427 flags.extend(["-outputdir", getTestOpts().outputdir])
1428
1429 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1430 + config.compiler + "' " \
1431 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1432 + to_compile + ' ' \
1433 + extra_hc_opts + ' ' \
1434 + getTestOpts().extra_hc_opts + ' ' \
1435 + flags \
1436 + ' -fglasgow-exts -o ' + name \
1437 + '>' + errname + ' 2>&1'
1438
1439 result = runCmdFor(name, cmd)
1440 exit_code = result >> 8
1441
1442 if exit_code != 0:
1443 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1444 if_verbose_dump(1,qerrname)
1445 return failBecause('ext core exit code non-0')
1446
1447 # Clean up
1448 rm_no_fail ( oname )
1449 rm_no_fail ( hcname )
1450 rm_no_fail ( qcorefilename )
1451 rm_no_fail ( depsfilename )
1452
1453 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1454
1455 # -----------------------------------------------------------------------------
1456 # Utils
1457
1458 def check_stdout_ok( name ):
1459 if getTestOpts().with_namebase == None:
1460 namebase = name
1461 else:
1462 namebase = getTestOpts().with_namebase
1463
1464 actual_stdout_file = qualify(name, 'run.stdout')
1465 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1466
1467 def norm(str):
1468 if platform_specific:
1469 return str
1470 else:
1471 return normalise_output(str)
1472
1473 return compare_outputs('stdout', \
1474 two_normalisers(norm, getTestOpts().extra_normaliser), \
1475 expected_stdout_file, actual_stdout_file)
1476
1477 def dump_stdout( name ):
1478 print 'Stdout:'
1479 print read_no_crs(qualify(name, 'run.stdout'))
1480
1481 def check_stderr_ok( name ):
1482 if getTestOpts().with_namebase == None:
1483 namebase = name
1484 else:
1485 namebase = getTestOpts().with_namebase
1486
1487 actual_stderr_file = qualify(name, 'run.stderr')
1488 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1489
1490 def norm(str):
1491 if platform_specific:
1492 return str
1493 else:
1494 return normalise_errmsg(str)
1495
1496 return compare_outputs('stderr', \
1497 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1498 expected_stderr_file, actual_stderr_file)
1499
1500 def dump_stderr( name ):
1501 print "Stderr:"
1502 print read_no_crs(qualify(name, 'run.stderr'))
1503
1504 def read_no_crs(file):
1505 str = ''
1506 try:
1507 h = open(file)
1508 str = h.read()
1509 h.close
1510 except:
1511 # On Windows, if the program fails very early, it seems the
1512 # files stdout/stderr are redirected to may not get created
1513 pass
1514 return re.sub('\r', '', str)
1515
1516 def write_file(file, str):
1517 h = open(file, 'w')
1518 h.write(str)
1519 h.close
1520
1521 def check_hp_ok(name):
1522
1523 # do not qualify for hp2ps because we should be in the right directory
1524 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1525
1526 hp2psResult = runCmdExitCode(hp2psCmd)
1527
1528 actual_ps_file = qualify(name, 'ps')
1529
1530 if(hp2psResult == 0):
1531 if (os.path.exists(actual_ps_file)):
1532 if gs_working:
1533 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1534 if (gsResult == 0):
1535 return (True)
1536 else:
1537 print "hp2ps output for " + name + "is not valid PostScript"
1538 else: return (True) # assume postscript is valid without ghostscript
1539 else:
1540 print "hp2ps did not generate PostScript for " + name
1541 return (False)
1542 else:
1543 print "hp2ps error when processing heap profile for " + name
1544 return(False)
1545
1546 def check_prof_ok(name):
1547
1548 prof_file = qualify(name,'prof')
1549
1550 if not os.path.exists(prof_file):
1551 print prof_file + " does not exist"
1552 return(False)
1553
1554 if os.path.getsize(qualify(name,'prof')) == 0:
1555 print prof_file + " is empty"
1556 return(False)
1557
1558 if getTestOpts().with_namebase == None:
1559 namebase = name
1560 else:
1561 namebase = getTestOpts().with_namebase
1562
1563 (platform_specific, expected_prof_file) = \
1564 platform_wordsize_qualify(namebase, 'prof.sample')
1565
1566 # sample prof file is not required
1567 if not os.path.exists(expected_prof_file):
1568 return True
1569 else:
1570 return compare_outputs('prof', \
1571 two_normalisers(normalise_whitespace,normalise_prof), \
1572 expected_prof_file, prof_file)
1573
1574 # Compare expected output to actual output, and optionally accept the
1575 # new output. Returns true if output matched or was accepted, false
1576 # otherwise.
1577 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1578 if os.path.exists(expected_file):
1579 expected_raw = read_no_crs(expected_file)
1580 # print "norm:", normaliser(expected_raw)
1581 expected_str = normaliser(expected_raw)
1582 expected_file_for_diff = expected_file
1583 else:
1584 expected_str = ''
1585 expected_file_for_diff = '/dev/null'
1586
1587 actual_raw = read_no_crs(actual_file)
1588 actual_str = normaliser(actual_raw)
1589
1590 if expected_str == actual_str:
1591 return 1
1592 else:
1593 print 'Actual ' + kind + ' output differs from expected:'
1594
1595 if expected_file_for_diff == '/dev/null':
1596 expected_normalised_file = '/dev/null'
1597 else:
1598 expected_normalised_file = expected_file + ".normalised"
1599 write_file(expected_normalised_file, expected_str)
1600
1601 actual_normalised_file = actual_file + ".normalised"
1602 write_file(actual_normalised_file, actual_str)
1603
1604 # Ignore whitespace when diffing. We should only get to this
1605 # point if there are non-whitespace differences
1606 #
1607 # Note we are diffing the *actual* output, not the normalised
1608 # output. The normalised output may have whitespace squashed
1609 # (including newlines) so the diff would be hard to read.
1610 # This does mean that the diff might contain changes that
1611 # would be normalised away.
1612 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1613 ' ' + actual_file )
1614
1615 # If for some reason there were no non-whitespace differences,
1616 # then do a full diff
1617 if r == 0:
1618 r = os.system( 'diff -u ' + expected_file_for_diff + \
1619 ' ' + actual_file )
1620
1621 if config.accept:
1622 print 'Accepting new output.'
1623 write_file(expected_file, actual_raw)
1624 return 1
1625 else:
1626 return 0
1627
1628
1629 def normalise_whitespace( str ):
1630 # Merge contiguous whitespace characters into a single space.
1631 str = re.sub('[ \t\n]+', ' ', str)
1632 return str
1633
1634 def normalise_errmsg( str ):
1635 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1636 # the colon is there because it appears in error messages; this
1637 # hacky solution is used in place of more sophisticated filename
1638 # mangling
1639 str = re.sub('([^\\s])\\.exe', '\\1', str)
1640 # normalise slashes, minimise Windows/Unix filename differences
1641 str = re.sub('\\\\', '/', str)
1642 # The inplace ghc's are called ghc-stage[123] to avoid filename
1643 # collisions, so we need to normalise that to just "ghc"
1644 str = re.sub('ghc-stage[123]', 'ghc', str)
1645 # We sometimes see the name of the integer-gmp package on stderr,
1646 # but this can change (either the implementation name or the
1647 # version number), so we canonicalise it here
1648 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1649 return str
1650
1651 # normalise a .prof file, so that we can reasonably compare it against
1652 # a sample. This doesn't compare any of the actual profiling data,
1653 # only the shape of the profile and the number of entries.
1654 def normalise_prof (str):
1655 # strip everything up to the line beginning "COST CENTRE"
1656 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1657
1658 # strip results for CAFs, these tend to change unpredictably
1659 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1660
1661 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1662 # sometimes under MAIN.
1663 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1664
1665 # We have somthing like this:
1666
1667 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1668 # k Main 204 1 0.0 0.0 0.0 0.0
1669 # foo Main 205 1 0.0 0.0 0.0 0.0
1670 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1671
1672 # then we remove all the specific profiling data, leaving only the
1673 # cost centre name, module, and entries, to end up with this:
1674
1675 # MAIN MAIN 0
1676 # k Main 1
1677 # foo Main 1
1678 # foo.bar Main 1
1679
1680 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1681 return str
1682
1683 def normalise_slashes_( str ):
1684 str = re.sub('\\\\', '/', str)
1685 return str
1686
1687 def normalise_exe_( str ):
1688 str = re.sub('\.exe', '', str)
1689 return str
1690
1691 def normalise_output( str ):
1692 # Remove a .exe extension (for Windows)
1693 # This can occur in error messages generated by the program.
1694 str = re.sub('([^\\s])\\.exe', '\\1', str)
1695 return str
1696
1697 def normalise_asm( str ):
1698 lines = str.split('\n')
1699 # Only keep instructions and labels not starting with a dot.
1700 metadata = re.compile('^[ \t]*\\..*$')
1701 out = []
1702 for line in lines:
1703 # Drop metadata directives (e.g. ".type")
1704 if not metadata.match(line):
1705 line = re.sub('@plt', '', line)
1706 instr = line.lstrip().split()
1707 # Drop empty lines.
1708 if not instr:
1709 continue
1710 # Drop operands, except for call instructions.
1711 elif instr[0] == 'call':
1712 out.append(instr[0] + ' ' + instr[1])
1713 else:
1714 out.append(instr[0])
1715 out = '\n'.join(out)
1716 return out
1717
1718 def if_verbose( n, str ):
1719 if config.verbose >= n:
1720 print str
1721
1722 def if_verbose_dump( n, f ):
1723 if config.verbose >= n:
1724 try:
1725 print open(f).read()
1726 except:
1727 print ''
1728
1729 def rawSystem(cmd_and_args):
1730 # We prefer subprocess.call to os.spawnv as the latter
1731 # seems to send its arguments through a shell or something
1732 # with the Windows (non-cygwin) python. An argument "a b c"
1733 # turns into three arguments ["a", "b", "c"].
1734
1735 # However, subprocess is new in python 2.4, so fall back to
1736 # using spawnv if we don't have it
1737
1738 if have_subprocess:
1739 return subprocess.call(cmd_and_args)
1740 else:
1741 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1742
1743 # Note that this doesn't handle the timeout itself; it is just used for
1744 # commands that have timeout handling built-in.
1745 def rawSystemWithTimeout(cmd_and_args):
1746 r = rawSystem(cmd_and_args)
1747 if r == 98:
1748 # The python timeout program uses 98 to signal that ^C was pressed
1749 stopNow()
1750 return r
1751
1752 # cmd is a complex command in Bourne-shell syntax
1753 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1754 # Hence it must ultimately be run by a Bourne shell
1755 #
1756 # Mostly it invokes the command wrapped in 'timeout' thus
1757 # timeout 300 'cd . && ...blah blah'
1758 # so it's timeout's job to invoke the Bourne shell
1759 #
1760 # But watch out for the case when there is no timeout program!
1761 # Then, when using the native Python, os.system will invoke the cmd shell
1762
1763 def runCmd( cmd ):
1764 if_verbose( 1, cmd )
1765 r = 0
1766 if config.os == 'mingw32':
1767 # On MinGW, we will always have timeout
1768 assert config.timeout_prog!=''
1769
1770 if config.timeout_prog != '':
1771 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1772 else:
1773 r = os.system(cmd)
1774 return r << 8
1775
1776 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1777 if_verbose( 1, cmd )
1778 r = 0
1779 if config.os == 'mingw32':
1780 # On MinGW, we will always have timeout
1781 assert config.timeout_prog!=''
1782 timeout = int(ceil(config.timeout * timeout_multiplier))
1783
1784 if config.timeout_prog != '':
1785 if config.check_files_written:
1786 fn = name + ".strace"
1787 r = rawSystemWithTimeout(
1788 ["strace", "-o", fn, "-fF",
1789 "-e", "creat,open,chdir,clone,vfork",
1790 config.timeout_prog, str(timeout), cmd])
1791 addTestFilesWritten(name, fn)
1792 rm_no_fail(fn)
1793 else:
1794 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1795 else:
1796 r = os.system(cmd)
1797 return r << 8
1798
1799 def runCmdExitCode( cmd ):
1800 return (runCmd(cmd) >> 8);
1801
1802
1803 # -----------------------------------------------------------------------------
1804 # checking for files being written to by multiple tests
1805
1806 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1807 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1808 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1809 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1810 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1811 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1812 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1813 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1814 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1815 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1816 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1817 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1818 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1819 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1820 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1821 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1822 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1823
1824 # Files that are read or written but shouldn't be:
1825 # * ghci_history shouldn't be read or written by tests
1826 # * things under package.conf.d shouldn't be written by tests
1827 bad_file_usages = {}
1828
1829 # Mapping from tests to the list of files that they write
1830 files_written = {}
1831
1832 # Mapping from tests to the list of files that they write but don't clean
1833 files_written_not_removed = {}
1834
1835 def add_bad_file_usage(name, file):
1836 try:
1837 if not file in bad_file_usages[name]:
1838 bad_file_usages[name].append(file)
1839 except:
1840 bad_file_usages[name] = [file]
1841
1842 def mkPath(curdir, path):
1843 # Given the current full directory is 'curdir', what is the full
1844 # path to 'path'?
1845 return os.path.realpath(os.path.join(curdir, path))
1846
1847 def addTestFilesWritten(name, fn):
1848 if config.use_threads:
1849 with t.lockFilesWritten:
1850 addTestFilesWrittenHelper(name, fn)
1851 else:
1852 addTestFilesWrittenHelper(name, fn)
1853
1854 def addTestFilesWrittenHelper(name, fn):
1855 started = False
1856 working_directories = {}
1857
1858 with open(fn, 'r') as f:
1859 for line in f:
1860 m_pid = re_strace_pid.match(line)
1861 if m_pid:
1862 pid = m_pid.group(1)
1863 content = m_pid.group(2)
1864 elif re_strace_unavailable.match(line):
1865 next
1866 else:
1867 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1868
1869 m_open = re_strace_open.match(content)
1870 m_chdir = re_strace_chdir.match(content)
1871 m_clone = re_strace_clone.match(content)
1872 m_vfork = re_strace_vfork.match(content)
1873
1874 if not started:
1875 working_directories[pid] = os.getcwd()
1876 started = True
1877
1878 if m_open:
1879 file = m_open.group(1)
1880 file = mkPath(working_directories[pid], file)
1881 if file.endswith("ghci_history"):
1882 add_bad_file_usage(name, file)
1883 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1884 flags = m_open.group(2).split('|')
1885 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1886 if re.match('package\.conf\.d', file):
1887 add_bad_file_usage(name, file)
1888 else:
1889 try:
1890 if not file in files_written[name]:
1891 files_written[name].append(file)
1892 except:
1893 files_written[name] = [file]
1894 elif 'O_RDONLY' in flags:
1895 pass
1896 else:
1897 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1898 elif m_chdir:
1899 # We optimistically assume that unfinished chdir's are going to succeed
1900 dir = m_chdir.group(1)
1901 working_directories[pid] = mkPath(working_directories[pid], dir)
1902 elif m_clone:
1903 working_directories[m_clone.group(2)] = working_directories[pid]
1904 elif m_vfork:
1905 working_directories[m_vfork.group(2)] = working_directories[pid]
1906 elif re_strace_open_resumed.match(content):
1907 pass
1908 elif re_strace_chdir_resumed.match(content):
1909 pass
1910 elif re_strace_vfork_unfinished.match(content):
1911 pass
1912 elif re_strace_clone_unfinished.match(content):
1913 pass
1914 elif re_strace_ignore_sigchild.match(content):
1915 pass
1916 elif re_strace_ignore_sigvtalarm.match(content):
1917 pass
1918 elif re_strace_ignore_sigint.match(content):
1919 pass
1920 elif re_strace_ignore_sigfpe.match(content):
1921 pass
1922 elif re_strace_ignore_sigsegv.match(content):
1923 pass
1924 elif re_strace_ignore_sigpipe.match(content):
1925 pass
1926 else:
1927 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1928
1929 def checkForFilesWrittenProblems(file):
1930 foundProblem = False
1931
1932 files_written_inverted = {}
1933 for t in files_written.keys():
1934 for f in files_written[t]:
1935 try:
1936 files_written_inverted[f].append(t)
1937 except:
1938 files_written_inverted[f] = [t]
1939
1940 for f in files_written_inverted.keys():
1941 if len(files_written_inverted[f]) > 1:
1942 if not foundProblem:
1943 foundProblem = True
1944 file.write("\n")
1945 file.write("\nSome files are written by multiple tests:\n")
1946 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1947 if foundProblem:
1948 file.write("\n")
1949
1950 # -----
1951
1952 if len(files_written_not_removed) > 0:
1953 file.write("\n")
1954 file.write("\nSome files written but not removed:\n")
1955 tests = files_written_not_removed.keys()
1956 tests.sort()
1957 for t in tests:
1958 for f in files_written_not_removed[t]:
1959 file.write(" " + t + ": " + f + "\n")
1960 file.write("\n")
1961
1962 # -----
1963
1964 if len(bad_file_usages) > 0:
1965 file.write("\n")
1966 file.write("\nSome bad file usages:\n")
1967 tests = bad_file_usages.keys()
1968 tests.sort()
1969 for t in tests:
1970 for f in bad_file_usages[t]:
1971 file.write(" " + t + ": " + f + "\n")
1972 file.write("\n")
1973
1974 # -----------------------------------------------------------------------------
1975 # checking if ghostscript is available for checking the output of hp2ps
1976
1977 def genGSCmd(psfile):
1978 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1979
1980 def gsNotWorking():
1981 global gs_working
1982 print "GhostScript not available for hp2ps tests"
1983
1984 global gs_working
1985 gs_working = 0
1986 if config.have_profiling:
1987 if config.gs != '':
1988 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
1989 if resultGood == 0:
1990 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
1991 if resultBad != 0:
1992 print "GhostScript available for hp2ps tests"
1993 gs_working = 1;
1994 else:
1995 gsNotWorking();
1996 else:
1997 gsNotWorking();
1998 else:
1999 gsNotWorking();
2000
2001 def rm_no_fail( file ):
2002 try:
2003 os.remove( file )
2004 finally:
2005 return
2006
2007 def add_suffix( name, suffix ):
2008 if suffix == '':
2009 return name
2010 else:
2011 return name + '.' + suffix
2012
2013 def add_hs_lhs_suffix(name):
2014 if getTestOpts().c_src:
2015 return add_suffix(name, 'c')
2016 elif getTestOpts().cmm_src:
2017 return add_suffix(name, 'cmm')
2018 elif getTestOpts().objc_src:
2019 return add_suffix(name, 'm')
2020 elif getTestOpts().objcpp_src:
2021 return add_suffix(name, 'mm')
2022 elif getTestOpts().literate:
2023 return add_suffix(name, 'lhs')
2024 else:
2025 return add_suffix(name, 'hs')
2026
2027 def replace_suffix( name, suffix ):
2028 base, suf = os.path.splitext(name)
2029 return base + '.' + suffix
2030
2031 def in_testdir( name ):
2032 return (getTestOpts().testdir + '/' + name)
2033
2034 def qualify( name, suff ):
2035 return in_testdir(add_suffix(name, suff))
2036
2037
2038 # Finding the sample output. The filename is of the form
2039 #
2040 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2041 #
2042 # and we pick the most specific version available. The <version> is
2043 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2044 # more fine-grained control use if_compiler_lt().
2045 #
2046 def platform_wordsize_qualify( name, suff ):
2047
2048 basepath = qualify(name, suff)
2049
2050 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2051 for (platformSpecific, plat) in [(1, '-' + config.platform),
2052 (1, '-' + config.os),
2053 (0, '')]
2054 for ws in ['-ws-' + config.wordsize, '']
2055 for comp in ['-' + config.compiler_type, '']
2056 for vers in ['-' + config.compiler_maj_version, '']]
2057
2058 dir = glob.glob(basepath + '*')
2059 dir = map (lambda d: normalise_slashes_(d), dir)
2060
2061 for (platformSpecific, f) in paths:
2062 if f in dir:
2063 return (platformSpecific,f)
2064
2065 return (0, basepath)
2066
2067 # Clean up prior to the test, so that we can't spuriously conclude
2068 # that it passed on the basis of old run outputs.
2069 def pretest_cleanup(name):
2070 if getTestOpts().outputdir != None:
2071 odir = in_testdir(getTestOpts().outputdir)
2072 try:
2073 shutil.rmtree(odir)
2074 except:
2075 pass
2076 os.mkdir(odir)
2077
2078 rm_no_fail(qualify(name,'comp.stderr'))
2079 rm_no_fail(qualify(name,'run.stderr'))
2080 rm_no_fail(qualify(name,'run.stdout'))
2081 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2082 # simple_build zaps the following:
2083 # rm_nofail(qualify("o"))
2084 # rm_nofail(qualify(""))
2085 # not interested in the return code
2086
2087 # -----------------------------------------------------------------------------
2088 # Return a list of all the files ending in '.T' below the directory dir.
2089
2090 def findTFiles(roots):
2091 return concat(map(findTFiles_,roots))
2092
2093 def findTFiles_(path):
2094 if os.path.isdir(path):
2095 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2096 return findTFiles(paths)
2097 elif path[-2:] == '.T':
2098 return [path]
2099 else:
2100 return []
2101
2102 # -----------------------------------------------------------------------------
2103 # Output a test summary to the specified file object
2104
2105 def summary(t, file):
2106
2107 file.write('\n')
2108 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2109 file.write('OVERALL SUMMARY for test run started at '
2110 + t.start_time + '\n'
2111 + string.rjust(`t.total_tests`, 8)
2112 + ' total tests, which gave rise to\n'
2113 + string.rjust(`t.total_test_cases`, 8)
2114 + ' test cases, of which\n'
2115 + string.rjust(`t.n_tests_skipped`, 8)
2116 + ' were skipped\n'
2117 + '\n'
2118 + string.rjust(`t.n_missing_libs`, 8)
2119 + ' had missing libraries\n'
2120 + string.rjust(`t.n_expected_passes`, 8)
2121 + ' expected passes\n'
2122 + string.rjust(`t.n_expected_failures`, 8)
2123 + ' expected failures\n'
2124 + '\n'
2125 + string.rjust(`t.n_framework_failures`, 8)
2126 + ' caused framework failures\n'
2127 + string.rjust(`t.n_unexpected_passes`, 8)
2128 + ' unexpected passes\n'
2129 + string.rjust(`t.n_unexpected_failures`, 8)
2130 + ' unexpected failures\n'
2131 + '\n')
2132
2133 if t.n_unexpected_passes > 0:
2134 file.write('Unexpected passes:\n')
2135 printPassingTestInfosSummary(file, t.unexpected_passes)
2136
2137 if t.n_unexpected_failures > 0:
2138 file.write('Unexpected failures:\n')
2139 printFailingTestInfosSummary(file, t.unexpected_failures)
2140
2141 if config.check_files_written:
2142 checkForFilesWrittenProblems(file)
2143
2144 if stopping():
2145 file.write('WARNING: Testsuite run was terminated early\n')
2146
2147 def printUnexpectedTests(file, testInfoss):
2148 unexpected = []
2149 for testInfos in testInfoss:
2150 directories = testInfos.keys()
2151 for directory in directories:
2152 tests = testInfos[directory].keys()
2153 unexpected += tests
2154 if unexpected != []:
2155 file.write('Unexpected results from:\n')
2156 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2157 file.write('\n')
2158
2159 def printPassingTestInfosSummary(file, testInfos):
2160 directories = testInfos.keys()
2161 directories.sort()
2162 maxDirLen = max(map ((lambda x : len(x)), directories))
2163 for directory in directories:
2164 tests = testInfos[directory].keys()
2165 tests.sort()
2166 for test in tests:
2167 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2168 ' (' + join(testInfos[directory][test],',') + ')\n')
2169 file.write('\n')
2170
2171 def printFailingTestInfosSummary(file, testInfos):
2172 directories = testInfos.keys()
2173 directories.sort()
2174 maxDirLen = max(map ((lambda x : len(x)), directories))
2175 for directory in directories:
2176 tests = testInfos[directory].keys()
2177 tests.sort()
2178 for test in tests:
2179 reasons = testInfos[directory][test].keys()
2180 for reason in reasons:
2181 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2182 ' [' + reason + ']' + \
2183 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2184 file.write('\n')
2185
2186 def getStdout(cmd):
2187 if have_subprocess:
2188 p = subprocess.Popen(cmd,
2189 stdout=subprocess.PIPE,
2190 stderr=subprocess.PIPE)
2191 (stdout, stderr) = p.communicate()
2192 r = p.wait()
2193 if r != 0:
2194 raise Exception("Command failed: " + str(cmd))
2195 if stderr != '':
2196 raise Exception("stderr from command: " + str(cmd))
2197 return stdout
2198 else:
2199 raise Exception("Need subprocess to get stdout, but don't have it")