Make test cleaning a little more consistent
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = [thisdir_settings, f]
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 record_broken(name, opts, bug)
155 opts.expect = 'fail';
156
157 def expect_broken_for( bug, ways ):
158 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
159
160 def _expect_broken_for( name, opts, bug, ways ):
161 record_broken(name, opts, bug)
162 opts.expect_fail_for = ways
163
164 def record_broken(name, opts, bug):
165 global brokens
166 me = (bug, opts.testdir, name)
167 if not me in brokens:
168 brokens.append(me)
169
170 # -----
171
172 def omit_ways( ways ):
173 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174
175 def _omit_ways( name, opts, ways ):
176 opts.omit_ways = ways
177
178 # -----
179
180 def only_ways( ways ):
181 return lambda name, opts, w=ways: _only_ways( name, opts, w )
182
183 def _only_ways( name, opts, ways ):
184 opts.only_ways = ways
185
186 # -----
187
188 def extra_ways( ways ):
189 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190
191 def _extra_ways( name, opts, ways ):
192 opts.extra_ways = ways
193
194 # -----
195
196 def omit_compiler_types( compiler_types ):
197 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
198
199 def _omit_compiler_types( name, opts, compiler_types ):
200 if config.compiler_type in compiler_types:
201 opts.skip = 1
202
203 # -----
204
205 def only_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
207
208 def _only_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type not in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def set_stdin( file ):
215 return lambda name, opts, f=file: _set_stdin(name, opts, f);
216
217 def _set_stdin( name, opts, f ):
218 opts.stdin = f
219
220 # -----
221
222 def exit_code( val ):
223 return lambda name, opts, v=val: _exit_code(name, opts, v);
224
225 def _exit_code( name, opts, v ):
226 opts.exit_code = v
227
228 # -----
229
230 def timeout_multiplier( val ):
231 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232
233 def _timeout_multiplier( name, opts, v ):
234 opts.timeout_multiplier = v
235
236 # -----
237
238 def extra_run_opts( val ):
239 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240
241 def _extra_run_opts( name, opts, v ):
242 opts.extra_run_opts = v
243
244 # -----
245
246 def extra_hc_opts( val ):
247 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
248
249 def _extra_hc_opts( name, opts, v ):
250 opts.extra_hc_opts = v
251
252 # -----
253
254 def extra_clean( files ):
255 return lambda name, opts, v=files: _extra_clean(name, opts, v);
256
257 def _extra_clean( name, opts, v ):
258 opts.clean_files = v
259
260 # -----
261
262 def stats_num_field( field, expecteds ):
263 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
264
265 def _stats_num_field( name, opts, field, expecteds ):
266 if field in opts.stats_range_fields:
267 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
268
269 if type(expecteds) is types.ListType:
270 for (b, expected, dev) in expecteds:
271 if b:
272 opts.stats_range_fields[field] = (expected, dev)
273 return
274 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275
276 else:
277 (expected, dev) = expecteds
278 opts.stats_range_fields[field] = (expected, dev)
279
280 def compiler_stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282
283 def _compiler_stats_num_field( name, opts, field, expecteds ):
284 if field in opts.compiler_stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 for (b, expected, dev) in expecteds:
288 if b:
289 opts.compiler_stats_range_fields[field] = (expected, dev)
290 return
291
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 # -----
295
296 def when(b, f):
297 # When list_brokens is on, we want to see all expect_broken calls,
298 # so we always do f
299 if b or config.list_broken:
300 return f
301 else:
302 return normal
303
304 def unless(b, f):
305 return when(not b, f)
306
307 def doing_ghci():
308 return 'ghci' in config.run_ways
309
310 def ghci_dynamic( ):
311 return config.ghc_dynamic
312
313 def fast():
314 return config.fast
315
316 def platform( plat ):
317 return config.platform == plat
318
319 def opsys( os ):
320 return config.os == os
321
322 def arch( arch ):
323 return config.arch == arch
324
325 def wordsize( ws ):
326 return config.wordsize == str(ws)
327
328 def msys( ):
329 return config.msys
330
331 def cygwin( ):
332 return config.cygwin
333
334 def have_vanilla( ):
335 return config.have_vanilla
336
337 def have_dynamic( ):
338 return config.have_dynamic
339
340 def have_profiling( ):
341 return config.have_profiling
342
343 def in_tree_compiler( ):
344 return config.in_tree_compiler
345
346 def compiler_type( compiler ):
347 return config.compiler_type == compiler
348
349 def compiler_lt( compiler, version ):
350 return config.compiler_type == compiler and \
351 version_lt(config.compiler_version, version)
352
353 def compiler_le( compiler, version ):
354 return config.compiler_type == compiler and \
355 version_le(config.compiler_version, version)
356
357 def compiler_gt( compiler, version ):
358 return config.compiler_type == compiler and \
359 version_gt(config.compiler_version, version)
360
361 def compiler_ge( compiler, version ):
362 return config.compiler_type == compiler and \
363 version_ge(config.compiler_version, version)
364
365 def unregisterised( ):
366 return config.unregisterised
367
368 def compiler_profiled( ):
369 return config.compiler_profiled
370
371 def compiler_debugged( ):
372 return config.compiler_debugged
373
374 def tag( t ):
375 return t in config.compiler_tags
376
377 # ---
378
379 def namebase( nb ):
380 return lambda opts, nb=nb: _namebase(opts, nb)
381
382 def _namebase( opts, nb ):
383 opts.with_namebase = nb
384
385 # ---
386
387 def high_memory_usage(name, opts):
388 opts.alone = True
389
390 # If a test is for a multi-CPU race, then running the test alone
391 # increases the chance that we'll actually see it.
392 def multi_cpu_race(name, opts):
393 opts.alone = True
394
395 # ---
396 def literate( name, opts ):
397 opts.literate = 1;
398
399 def c_src( name, opts ):
400 opts.c_src = 1;
401
402 def objc_src( name, opts ):
403 opts.objc_src = 1;
404
405 def objcpp_src( name, opts ):
406 opts.objcpp_src = 1;
407
408 def cmm_src( name, opts ):
409 opts.cmm_src = 1;
410
411 def outputdir( odir ):
412 return lambda name, opts, d=odir: _outputdir(name, opts, d)
413
414 def _outputdir( name, opts, odir ):
415 opts.outputdir = odir;
416
417 # ----
418
419 def pre_cmd( cmd ):
420 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
421
422 def _pre_cmd( name, opts, cmd ):
423 opts.pre_cmd = cmd
424
425 # ----
426
427 def clean_cmd( cmd ):
428 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
429
430 def _clean_cmd( name, opts, cmd ):
431 opts.clean_cmd = cmd
432
433 # ----
434
435 def cmd_prefix( prefix ):
436 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
437
438 def _cmd_prefix( name, opts, prefix ):
439 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
440
441 # ----
442
443 def cmd_wrapper( fun ):
444 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
445
446 def _cmd_wrapper( name, opts, fun ):
447 opts.cmd_wrapper = fun
448
449 # ----
450
451 def compile_cmd_prefix( prefix ):
452 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
453
454 def _compile_cmd_prefix( name, opts, prefix ):
455 opts.compile_cmd_prefix = prefix
456
457 # ----
458
459 def normalise_slashes( name, opts ):
460 opts.extra_normaliser = normalise_slashes_
461
462 def normalise_exe( name, opts ):
463 opts.extra_normaliser = normalise_exe_
464
465 def normalise_fun( fun ):
466 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
467
468 def _normalise_fun( name, opts, f ):
469 opts.extra_normaliser = f
470
471 def normalise_errmsg_fun( fun ):
472 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
473
474 def _normalise_errmsg_fun( name, opts, f ):
475 opts.extra_errmsg_normaliser = f
476
477 def two_normalisers(f, g):
478 return lambda x, f=f, g=g: f(g(x))
479
480 # ----
481 # Function for composing two opt-fns together
482
483 def executeSetups(fs, name, opts):
484 if type(fs) is types.ListType:
485 # If we have a list of setups, then execute each one
486 map (lambda f : executeSetups(f, name, opts), fs)
487 else:
488 # fs is a single function, so just apply it
489 fs(name, opts)
490
491 # -----------------------------------------------------------------------------
492 # The current directory of tests
493
494 def newTestDir( dir ):
495 global thisdir_settings
496 # reset the options for this test directory
497 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
498
499 def _newTestDir( name, opts, dir ):
500 opts.testdir = dir
501 opts.compiler_always_flags = config.compiler_always_flags
502
503 # -----------------------------------------------------------------------------
504 # Actually doing tests
505
506 parallelTests = []
507 aloneTests = []
508 allTestNames = set([])
509
510 def runTest (opts, name, func, args):
511 ok = 0
512
513 if config.use_threads:
514 t.thread_pool.acquire()
515 try:
516 while config.threads<(t.running_threads+1):
517 t.thread_pool.wait()
518 t.running_threads = t.running_threads+1
519 ok=1
520 t.thread_pool.release()
521 thread.start_new_thread(test_common_thread, (name, opts, func, args))
522 except:
523 if not ok:
524 t.thread_pool.release()
525 else:
526 test_common_work (name, opts, func, args)
527
528 # name :: String
529 # setup :: TestOpts -> IO ()
530 def test (name, setup, func, args):
531 global aloneTests
532 global parallelTests
533 global allTestNames
534 global thisdir_settings
535 if name in allTestNames:
536 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
537 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
538 framework_fail(name, 'bad_name', 'This test has an invalid name')
539
540 # Make a deep copy of the default_testopts, as we need our own copy
541 # of any dictionaries etc inside it. Otherwise, if one test modifies
542 # them, all tests will see the modified version!
543 myTestOpts = copy.deepcopy(default_testopts)
544
545 executeSetups([thisdir_settings, setup], name, myTestOpts)
546
547 thisTest = lambda : runTest(myTestOpts, name, func, args)
548 if myTestOpts.alone:
549 aloneTests.append(thisTest)
550 else:
551 parallelTests.append(thisTest)
552 allTestNames.add(name)
553
554 if config.use_threads:
555 def test_common_thread(name, opts, func, args):
556 t.lock.acquire()
557 try:
558 test_common_work(name,opts,func,args)
559 finally:
560 t.lock.release()
561 t.thread_pool.acquire()
562 t.running_threads = t.running_threads - 1
563 t.thread_pool.notify()
564 t.thread_pool.release()
565
566 def get_package_cache_timestamp():
567 if config.package_conf_cache_file == '':
568 return 0.0
569 else:
570 try:
571 return os.stat(config.package_conf_cache_file).st_mtime
572 except:
573 return 0.0
574
575
576 def test_common_work (name, opts, func, args):
577 try:
578 t.total_tests = t.total_tests+1
579 setLocalTestOpts(opts)
580
581 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
582
583 # All the ways we might run this test
584 if func == compile or func == multimod_compile:
585 all_ways = config.compile_ways
586 elif func == compile_and_run or func == multimod_compile_and_run:
587 all_ways = config.run_ways
588 elif func == ghci_script:
589 if 'ghci' in config.run_ways:
590 all_ways = ['ghci']
591 else:
592 all_ways = []
593 else:
594 all_ways = ['normal']
595
596 # A test itself can request extra ways by setting opts.extra_ways
597 all_ways = all_ways + filter(lambda way: way not in all_ways,
598 opts.extra_ways)
599
600 t.total_test_cases = t.total_test_cases + len(all_ways)
601
602 ok_way = lambda way: \
603 not getTestOpts().skip \
604 and (config.only == [] or name in config.only) \
605 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
606 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
607 and way not in getTestOpts().omit_ways
608
609 # Which ways we are asked to skip
610 do_ways = filter (ok_way,all_ways)
611
612 # In fast mode, we skip all but one way
613 if config.fast and len(do_ways) > 0:
614 do_ways = [do_ways[0]]
615
616 if not config.clean_only:
617 # Run the required tests...
618 for way in do_ways:
619 if stopping():
620 break
621 do_test (name, way, func, args)
622
623 for way in all_ways:
624 if way not in do_ways:
625 skiptest (name,way)
626
627 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
628 pretest_cleanup(name)
629 clean(map (lambda suff: name + suff,
630 ['', '.exe', '.exe.manifest', '.genscript',
631 '.stderr.normalised', '.stdout.normalised',
632 '.run.stderr.normalised', '.run.stdout.normalised',
633 '.comp.stderr.normalised', '.comp.stdout.normalised',
634 '.interp.stderr.normalised', '.interp.stdout.normalised',
635 '.stats', '.comp.stats',
636 '.hi', '.o', '.prof', '.exe.prof', '.hc',
637 '_stub.h', '_stub.c', '_stub.o',
638 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
639
640 if func == multi_compile or func == multi_compile_fail:
641 extra_mods = args[1]
642 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
643 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
644
645 clean(getTestOpts().clean_files)
646
647 if getTestOpts().outputdir != None:
648 odir = in_testdir(getTestOpts().outputdir)
649 try:
650 shutil.rmtree(odir)
651 except:
652 pass
653
654 try:
655 shutil.rmtree(in_testdir('.hpc.' + name))
656 except:
657 pass
658
659 try:
660 cleanCmd = getTestOpts().clean_cmd
661 if cleanCmd != None:
662 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
663 if result != 0:
664 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
665 except e:
666 framework_fail(name, 'cleaning', 'clean-command exception')
667
668 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
669
670 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
671 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
672
673 try:
674 for f in files_written[name]:
675 if os.path.exists(f):
676 try:
677 if not f in files_written_not_removed[name]:
678 files_written_not_removed[name].append(f)
679 except:
680 files_written_not_removed[name] = [f]
681 except:
682 pass
683 except Exception, e:
684 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
685
686 def clean(strs):
687 for str in strs:
688 for name in glob.glob(in_testdir(str)):
689 clean_full_path(name)
690
691 def clean_full_path(name):
692 try:
693 # Remove files...
694 os.remove(name)
695 except OSError, e1:
696 try:
697 # ... and empty directories
698 os.rmdir(name)
699 except OSError, e2:
700 # We don't want to fail here, but we do want to know
701 # what went wrong, so print out the exceptions.
702 # ENOENT isn't a problem, though, as we clean files
703 # that don't necessarily exist.
704 if e1.errno != errno.ENOENT:
705 print e1
706 if e2.errno != errno.ENOENT:
707 print e2
708
709 def do_test(name, way, func, args):
710 full_name = name + '(' + way + ')'
711
712 try:
713 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
714 str([t.n_unexpected_passes, \
715 t.n_unexpected_failures, \
716 t.n_framework_failures])
717
718 if config.use_threads:
719 t.lock.release()
720
721 try:
722 preCmd = getTestOpts().pre_cmd
723 if preCmd != None:
724 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
725 if result != 0:
726 framework_fail(name, way, 'pre-command failed: ' + str(result))
727 except e:
728 framework_fail(name, way, 'pre-command exception')
729
730 try:
731 result = apply(func, [name,way] + args)
732 finally:
733 if config.use_threads:
734 t.lock.acquire()
735
736 if getTestOpts().expect != 'pass' and \
737 getTestOpts().expect != 'fail' and \
738 getTestOpts().expect != 'missing-lib':
739 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
740
741 try:
742 passFail = result['passFail']
743 except:
744 passFail = 'No passFail found'
745
746 if passFail == 'pass':
747 if getTestOpts().expect == 'pass' \
748 and way not in getTestOpts().expect_fail_for:
749 t.n_expected_passes = t.n_expected_passes + 1
750 if name in t.expected_passes:
751 t.expected_passes[name].append(way)
752 else:
753 t.expected_passes[name] = [way]
754 else:
755 print '*** unexpected pass for', full_name
756 t.n_unexpected_passes = t.n_unexpected_passes + 1
757 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
758 elif passFail == 'fail':
759 if getTestOpts().expect == 'pass' \
760 and way not in getTestOpts().expect_fail_for:
761 print '*** unexpected failure for', full_name
762 t.n_unexpected_failures = t.n_unexpected_failures + 1
763 reason = result['reason']
764 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
765 else:
766 if getTestOpts().expect == 'missing-lib':
767 t.n_missing_libs = t.n_missing_libs + 1
768 if name in t.missing_libs:
769 t.missing_libs[name].append(way)
770 else:
771 t.missing_libs[name] = [way]
772 else:
773 t.n_expected_failures = t.n_expected_failures + 1
774 if name in t.expected_failures:
775 t.expected_failures[name].append(way)
776 else:
777 t.expected_failures[name] = [way]
778 else:
779 framework_fail(name, way, 'bad result ' + passFail)
780 except KeyboardInterrupt:
781 stopNow()
782 except:
783 framework_fail(name, way, 'do_test exception')
784 traceback.print_exc()
785
786 def addPassingTestInfo (testInfos, directory, name, way):
787 directory = re.sub('^\\.[/\\\\]', '', directory)
788
789 if not directory in testInfos:
790 testInfos[directory] = {}
791
792 if not name in testInfos[directory]:
793 testInfos[directory][name] = []
794
795 testInfos[directory][name].append(way)
796
797 def addFailingTestInfo (testInfos, directory, name, reason, way):
798 directory = re.sub('^\\.[/\\\\]', '', directory)
799
800 if not directory in testInfos:
801 testInfos[directory] = {}
802
803 if not name in testInfos[directory]:
804 testInfos[directory][name] = {}
805
806 if not reason in testInfos[directory][name]:
807 testInfos[directory][name][reason] = []
808
809 testInfos[directory][name][reason].append(way)
810
811 def skiptest (name, way):
812 # print 'Skipping test \"', name, '\"'
813 t.n_tests_skipped = t.n_tests_skipped + 1
814 if name in t.tests_skipped:
815 t.tests_skipped[name].append(way)
816 else:
817 t.tests_skipped[name] = [way]
818
819 def framework_fail( name, way, reason ):
820 full_name = name + '(' + way + ')'
821 print '*** framework failure for', full_name, reason
822 t.n_framework_failures = t.n_framework_failures + 1
823 if name in t.framework_failures:
824 t.framework_failures[name].append(way)
825 else:
826 t.framework_failures[name] = [way]
827
828 def badResult(result):
829 try:
830 if result['passFail'] == 'pass':
831 return False
832 return True
833 except:
834 return True
835
836 def passed():
837 return {'passFail': 'pass'}
838
839 def failBecause(reason):
840 return {'passFail': 'fail', 'reason': reason}
841
842 # -----------------------------------------------------------------------------
843 # Generic command tests
844
845 # A generic command test is expected to run and exit successfully.
846 #
847 # The expected exit code can be changed via exit_code() as normal, and
848 # the expected stdout/stderr are stored in <testname>.stdout and
849 # <testname>.stderr. The output of the command can be ignored
850 # altogether by using run_command_ignore_output instead of
851 # run_command.
852
853 def run_command( name, way, cmd ):
854 return simple_run( name, '', cmd, '' )
855
856 # -----------------------------------------------------------------------------
857 # GHCi tests
858
859 def ghci_script( name, way, script ):
860 # filter out -fforce-recomp from compiler_always_flags, because we're
861 # actually testing the recompilation behaviour in the GHCi tests.
862 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
863 flags.append(getTestOpts().extra_hc_opts)
864 if getTestOpts().outputdir != None:
865 flags.extend(["-outputdir", getTestOpts().outputdir])
866
867 # We pass HC and HC_OPTS as environment variables, so that the
868 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
869 cmd = "HC='" + config.compiler + "' " + \
870 "HC_OPTS='" + join(flags,' ') + "' " + \
871 "'" + config.compiler + "'" + \
872 ' --interactive -v0 -ignore-dot-ghci ' + \
873 join(flags,' ')
874
875 getTestOpts().stdin = script
876 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
877
878 # -----------------------------------------------------------------------------
879 # Compile-only tests
880
881 def compile( name, way, extra_hc_opts ):
882 return do_compile( name, way, 0, '', [], extra_hc_opts )
883
884 def compile_fail( name, way, extra_hc_opts ):
885 return do_compile( name, way, 1, '', [], extra_hc_opts )
886
887 def multimod_compile( name, way, top_mod, extra_hc_opts ):
888 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
889
890 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
891 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
892
893 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
894 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
895
896 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
897 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
898
899 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
900 # print 'Compile only, extra args = ', extra_hc_opts
901 pretest_cleanup(name)
902
903 result = extras_build( way, extra_mods, extra_hc_opts )
904 if badResult(result):
905 return result
906 extra_hc_opts = result['hc_opts']
907
908 force = 0
909 if extra_mods:
910 force = 1
911 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
912
913 if badResult(result):
914 return result
915
916 # the actual stderr should always match the expected, regardless
917 # of whether we expected the compilation to fail or not (successful
918 # compilations may generate warnings).
919
920 if getTestOpts().with_namebase == None:
921 namebase = name
922 else:
923 namebase = getTestOpts().with_namebase
924
925 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
926 actual_stderr_file = qualify(name, 'comp.stderr')
927
928 if not compare_outputs('stderr', \
929 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
930 expected_stderr_file, actual_stderr_file):
931 return failBecause('stderr mismatch')
932
933 # no problems found, this test passed
934 return passed()
935
936 def compile_cmp_asm( name, way, extra_hc_opts ):
937 print 'Compile only, extra args = ', extra_hc_opts
938 pretest_cleanup(name)
939 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
940
941 if badResult(result):
942 return result
943
944 # the actual stderr should always match the expected, regardless
945 # of whether we expected the compilation to fail or not (successful
946 # compilations may generate warnings).
947
948 if getTestOpts().with_namebase == None:
949 namebase = name
950 else:
951 namebase = getTestOpts().with_namebase
952
953 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
954 actual_asm_file = qualify(name, 's')
955
956 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
957 expected_asm_file, actual_asm_file):
958 return failBecause('asm mismatch')
959
960 # no problems found, this test passed
961 return passed()
962
963 # -----------------------------------------------------------------------------
964 # Compile-and-run tests
965
966 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
967 # print 'Compile and run, extra args = ', extra_hc_opts
968 pretest_cleanup(name)
969
970 result = extras_build( way, extra_mods, extra_hc_opts )
971 if badResult(result):
972 return result
973 extra_hc_opts = result['hc_opts']
974
975 if way == 'ghci': # interpreted...
976 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
977 elif way == 'extcore' or way == 'optextcore' :
978 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
979 else: # compiled...
980 force = 0
981 if extra_mods:
982 force = 1
983
984 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
985 if badResult(result):
986 return result
987
988 cmd = './' + name;
989
990 # we don't check the compiler's stderr for a compile-and-run test
991 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
992
993 def compile_and_run( name, way, extra_hc_opts ):
994 return compile_and_run__( name, way, '', [], extra_hc_opts)
995
996 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
997 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
998
999 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1000 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1001
1002 def stats( name, way, stats_file ):
1003 opts = getTestOpts()
1004 return checkStats(stats_file, opts.stats_range_fields)
1005
1006 # -----------------------------------------------------------------------------
1007 # Check -t stats info
1008
1009 def checkStats(stats_file, range_fields):
1010 result = passed()
1011 if len(range_fields) > 0:
1012 f = open(in_testdir(stats_file))
1013 contents = f.read()
1014 f.close()
1015
1016 for (field, (expected, dev)) in range_fields.items():
1017 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1018 if m == None:
1019 print 'Failed to find field: ', field
1020 result = failBecause('no such stats field')
1021 val = int(m.group(1))
1022
1023 lowerBound = trunc( expected * ((100 - float(dev))/100));
1024 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1025
1026 if val < lowerBound:
1027 print field, 'value is too low:'
1028 print '(If this is because you have improved GHC, please'
1029 print 'update the test so that GHC doesn\'t regress again)'
1030 result = failBecause('stat too good')
1031 if val > upperBound:
1032 print field, 'value is too high:'
1033 result = failBecause('stat not good enough')
1034
1035 if val < lowerBound or val > upperBound:
1036 valStr = str(val)
1037 valLen = len(valStr)
1038 expectedStr = str(expected)
1039 expectedLen = len(expectedStr)
1040 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1041 def display(descr, val, extra):
1042 print descr, string.rjust(str(val), length), extra
1043 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1044 display(' Lower bound ' + field + ':', lowerBound, '')
1045 display(' Upper bound ' + field + ':', upperBound, '')
1046 display(' Actual ' + field + ':', val, '')
1047
1048 return result
1049
1050 # -----------------------------------------------------------------------------
1051 # Build a single-module program
1052
1053 def extras_build( way, extra_mods, extra_hc_opts ):
1054 for modopts in extra_mods:
1055 mod, opts = modopts
1056 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1057 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1058 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1059 if badResult(result):
1060 return result
1061
1062 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1063
1064
1065 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1066 opts = getTestOpts()
1067 errname = add_suffix(name, 'comp.stderr')
1068 rm_no_fail( qualify(errname, '') )
1069
1070 if top_mod != '':
1071 srcname = top_mod
1072 rm_no_fail( qualify(name, '') )
1073 base, suf = os.path.splitext(top_mod)
1074 rm_no_fail( qualify(base, '') )
1075 rm_no_fail( qualify(base, 'exe') )
1076 elif addsuf:
1077 srcname = add_hs_lhs_suffix(name)
1078 rm_no_fail( qualify(name, '') )
1079 else:
1080 srcname = name
1081 rm_no_fail( qualify(name, 'o') )
1082
1083 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1084
1085 to_do = ''
1086 if top_mod != '':
1087 to_do = '--make '
1088 if link:
1089 to_do = to_do + '-o ' + name
1090 elif link:
1091 to_do = '-o ' + name
1092 elif opts.compile_to_hc:
1093 to_do = '-C'
1094 else:
1095 to_do = '-c' # just compile
1096
1097 stats_file = name + '.comp.stats'
1098 if len(opts.compiler_stats_range_fields) > 0:
1099 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1100
1101 # Required by GHC 7.3+, harmless for earlier versions:
1102 if (getTestOpts().c_src or
1103 getTestOpts().objc_src or
1104 getTestOpts().objcpp_src or
1105 getTestOpts().cmm_src):
1106 extra_hc_opts += ' -no-hs-main '
1107
1108 if getTestOpts().compile_cmd_prefix == '':
1109 cmd_prefix = ''
1110 else:
1111 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1112
1113 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1114 if noforce:
1115 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1116 if getTestOpts().outputdir != None:
1117 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1118
1119 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1120 + config.compiler + "' " \
1121 + join(comp_flags,' ') + ' ' \
1122 + to_do + ' ' + srcname + ' ' \
1123 + join(config.way_flags(name)[way],' ') + ' ' \
1124 + extra_hc_opts + ' ' \
1125 + opts.extra_hc_opts + ' ' \
1126 + '>' + errname + ' 2>&1'
1127
1128 result = runCmdFor(name, cmd)
1129
1130 if result != 0 and not should_fail:
1131 actual_stderr = qualify(name, 'comp.stderr')
1132 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1133 if_verbose_dump(1,actual_stderr)
1134
1135 # ToDo: if the sub-shell was killed by ^C, then exit
1136
1137 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1138
1139 if badResult(statsResult):
1140 return statsResult
1141
1142 if should_fail:
1143 if result == 0:
1144 return failBecause('exit code 0')
1145 else:
1146 if result != 0:
1147 return failBecause('exit code non-0')
1148
1149 return passed()
1150
1151 # -----------------------------------------------------------------------------
1152 # Run a program and check its output
1153 #
1154 # If testname.stdin exists, route input from that, else
1155 # from /dev/null. Route output to testname.run.stdout and
1156 # testname.run.stderr. Returns the exit code of the run.
1157
1158 def simple_run( name, way, prog, args ):
1159 opts = getTestOpts()
1160
1161 # figure out what to use for stdin
1162 if opts.stdin != '':
1163 use_stdin = opts.stdin
1164 else:
1165 stdin_file = add_suffix(name, 'stdin')
1166 if os.path.exists(in_testdir(stdin_file)):
1167 use_stdin = stdin_file
1168 else:
1169 use_stdin = '/dev/null'
1170
1171 run_stdout = add_suffix(name,'run.stdout')
1172 run_stderr = add_suffix(name,'run.stderr')
1173
1174 rm_no_fail(qualify(name,'run.stdout'))
1175 rm_no_fail(qualify(name,'run.stderr'))
1176 rm_no_fail(qualify(name, 'hp'))
1177 rm_no_fail(qualify(name,'ps'))
1178 rm_no_fail(qualify(name, 'prof'))
1179
1180 my_rts_flags = rts_flags(way)
1181
1182 stats_file = name + '.stats'
1183 if len(opts.stats_range_fields) > 0:
1184 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1185
1186 if opts.no_stdin:
1187 stdin_comes_from = ''
1188 else:
1189 stdin_comes_from = ' <' + use_stdin
1190
1191 if opts.combined_output:
1192 redirection = ' >' + run_stdout \
1193 + ' 2>&1'
1194 else:
1195 redirection = ' >' + run_stdout \
1196 + ' 2>' + run_stderr
1197
1198 cmd = prog + ' ' + args + ' ' \
1199 + my_rts_flags + ' ' \
1200 + stdin_comes_from \
1201 + redirection
1202
1203 if opts.cmd_wrapper != None:
1204 cmd = opts.cmd_wrapper(cmd);
1205
1206 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1207
1208 # run the command
1209 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1210
1211 exit_code = result >> 8
1212 signal = result & 0xff
1213
1214 # check the exit code
1215 if exit_code != opts.exit_code:
1216 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1217 dump_stdout(name)
1218 dump_stderr(name)
1219 return failBecause('bad exit code')
1220
1221 check_hp = my_rts_flags.find("-h") != -1
1222 check_prof = my_rts_flags.find("-p") != -1
1223
1224 if not opts.ignore_output:
1225 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1226 bad_stdout = not check_stdout_ok(name)
1227 if bad_stderr:
1228 return failBecause('bad stderr')
1229 if bad_stdout:
1230 return failBecause('bad stdout')
1231 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1232 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1233 return failBecause('bad heap profile')
1234 if check_prof and not check_prof_ok(name):
1235 return failBecause('bad profile')
1236
1237 return checkStats(stats_file, opts.stats_range_fields)
1238
1239 def rts_flags(way):
1240 if (way == ''):
1241 return ''
1242 else:
1243 args = config.way_rts_flags[way]
1244
1245 if args == []:
1246 return ''
1247 else:
1248 return '+RTS ' + join(args,' ') + ' -RTS'
1249
1250 # -----------------------------------------------------------------------------
1251 # Run a program in the interpreter and check its output
1252
1253 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1254 outname = add_suffix(name, 'interp.stdout')
1255 errname = add_suffix(name, 'interp.stderr')
1256 rm_no_fail(outname)
1257 rm_no_fail(errname)
1258 rm_no_fail(name)
1259
1260 if (top_mod == ''):
1261 srcname = add_hs_lhs_suffix(name)
1262 else:
1263 srcname = top_mod
1264
1265 scriptname = add_suffix(name, 'genscript')
1266 qscriptname = in_testdir(scriptname)
1267 rm_no_fail(qscriptname)
1268
1269 delimiter = '===== program output begins here\n'
1270
1271 script = open(qscriptname, 'w')
1272 if not compile_only:
1273 # set the prog name and command-line args to match the compiled
1274 # environment.
1275 script.write(':set prog ' + name + '\n')
1276 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1277 # Add marker lines to the stdout and stderr output files, so we
1278 # can separate GHCi's output from the program's.
1279 script.write(':! echo ' + delimiter)
1280 script.write(':! echo 1>&2 ' + delimiter)
1281 # Set stdout to be line-buffered to match the compiled environment.
1282 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1283 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1284 # in the event of an exception as for the compiled program.
1285 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1286 script.close()
1287
1288 # figure out what to use for stdin
1289 if getTestOpts().stdin != '':
1290 stdin_file = in_testdir(getTestOpts().stdin)
1291 else:
1292 stdin_file = qualify(name, 'stdin')
1293
1294 if os.path.exists(stdin_file):
1295 stdin = open(stdin_file, 'r')
1296 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1297
1298 script.close()
1299
1300 flags = copy.copy(getTestOpts().compiler_always_flags)
1301 if getTestOpts().outputdir != None:
1302 flags.extend(["-outputdir", getTestOpts().outputdir])
1303
1304 cmd = "'" + config.compiler + "' " \
1305 + join(flags,' ') + ' ' \
1306 + srcname + ' ' \
1307 + join(config.way_flags(name)[way],' ') + ' ' \
1308 + extra_hc_opts + ' ' \
1309 + getTestOpts().extra_hc_opts + ' ' \
1310 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1311
1312 if getTestOpts().cmd_wrapper != None:
1313 cmd = getTestOpts().cmd_wrapper(cmd);
1314
1315 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1316
1317 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1318
1319 exit_code = result >> 8
1320 signal = result & 0xff
1321
1322 # split the stdout into compilation/program output
1323 split_file(in_testdir(outname), delimiter,
1324 qualify(name, 'comp.stdout'),
1325 qualify(name, 'run.stdout'))
1326 split_file(in_testdir(errname), delimiter,
1327 qualify(name, 'comp.stderr'),
1328 qualify(name, 'run.stderr'))
1329
1330 # check the exit code
1331 if exit_code != getTestOpts().exit_code:
1332 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1333 dump_stdout(name)
1334 dump_stderr(name)
1335 return failBecause('bad exit code')
1336
1337 # ToDo: if the sub-shell was killed by ^C, then exit
1338
1339 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1340 check_stdout_ok(name)):
1341 return passed()
1342 else:
1343 return failBecause('bad stdout or stderr')
1344
1345
1346 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1347 infile = open(in_fn)
1348 out1 = open(out1_fn, 'w')
1349 out2 = open(out2_fn, 'w')
1350
1351 line = infile.readline()
1352 line = re.sub('\r', '', line) # ignore Windows EOL
1353 while (re.sub('^\s*','',line) != delimiter and line != ''):
1354 out1.write(line)
1355 line = infile.readline()
1356 line = re.sub('\r', '', line)
1357 out1.close()
1358
1359 line = infile.readline()
1360 while (line != ''):
1361 out2.write(line)
1362 line = infile.readline()
1363 out2.close()
1364
1365 # -----------------------------------------------------------------------------
1366 # Generate External Core for the given program, then compile the resulting Core
1367 # and compare its output to the expected output
1368
1369 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1370
1371 depsfilename = qualify(name, 'deps')
1372 errname = add_suffix(name, 'comp.stderr')
1373 qerrname = qualify(errname,'')
1374
1375 hcname = qualify(name, 'hc')
1376 oname = qualify(name, 'o')
1377
1378 rm_no_fail( qerrname )
1379 rm_no_fail( qualify(name, '') )
1380
1381 if (top_mod == ''):
1382 srcname = add_hs_lhs_suffix(name)
1383 else:
1384 srcname = top_mod
1385
1386 qcorefilename = qualify(name, 'hcr')
1387 corefilename = add_suffix(name, 'hcr')
1388 rm_no_fail(qcorefilename)
1389
1390 # Generate External Core
1391
1392 if (top_mod == ''):
1393 to_do = ' ' + srcname + ' '
1394 else:
1395 to_do = ' --make ' + top_mod + ' '
1396
1397 flags = copy.copy(getTestOpts().compiler_always_flags)
1398 if getTestOpts().outputdir != None:
1399 flags.extend(["-outputdir", getTestOpts().outputdir])
1400 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1401 + config.compiler + "' " \
1402 + join(flags,' ') + ' ' \
1403 + join(config.way_flags(name)[way],' ') + ' ' \
1404 + extra_hc_opts + ' ' \
1405 + getTestOpts().extra_hc_opts \
1406 + to_do \
1407 + '>' + errname + ' 2>&1'
1408 result = runCmdFor(name, cmd)
1409
1410 exit_code = result >> 8
1411
1412 if exit_code != 0:
1413 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1414 if_verbose_dump(1,qerrname)
1415 return failBecause('ext core exit code non-0')
1416
1417 # Compile the resulting files -- if there's more than one module, we need to read the output
1418 # of the previous compilation in order to find the dependencies
1419 if (top_mod == ''):
1420 to_compile = corefilename
1421 else:
1422 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1423 deps = open(depsfilename).read()
1424 deplist = string.replace(deps, '\n',' ');
1425 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1426 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1427
1428 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1429 if getTestOpts().outputdir != None:
1430 flags.extend(["-outputdir", getTestOpts().outputdir])
1431
1432 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1433 + config.compiler + "' " \
1434 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1435 + to_compile + ' ' \
1436 + extra_hc_opts + ' ' \
1437 + getTestOpts().extra_hc_opts + ' ' \
1438 + flags \
1439 + ' -fglasgow-exts -o ' + name \
1440 + '>' + errname + ' 2>&1'
1441
1442 result = runCmdFor(name, cmd)
1443 exit_code = result >> 8
1444
1445 if exit_code != 0:
1446 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1447 if_verbose_dump(1,qerrname)
1448 return failBecause('ext core exit code non-0')
1449
1450 # Clean up
1451 rm_no_fail ( oname )
1452 rm_no_fail ( hcname )
1453 rm_no_fail ( qcorefilename )
1454 rm_no_fail ( depsfilename )
1455
1456 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1457
1458 # -----------------------------------------------------------------------------
1459 # Utils
1460
1461 def check_stdout_ok( name ):
1462 if getTestOpts().with_namebase == None:
1463 namebase = name
1464 else:
1465 namebase = getTestOpts().with_namebase
1466
1467 actual_stdout_file = qualify(name, 'run.stdout')
1468 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1469
1470 def norm(str):
1471 if platform_specific:
1472 return str
1473 else:
1474 return normalise_output(str)
1475
1476 return compare_outputs('stdout', \
1477 two_normalisers(norm, getTestOpts().extra_normaliser), \
1478 expected_stdout_file, actual_stdout_file)
1479
1480 def dump_stdout( name ):
1481 print 'Stdout:'
1482 print read_no_crs(qualify(name, 'run.stdout'))
1483
1484 def check_stderr_ok( name ):
1485 if getTestOpts().with_namebase == None:
1486 namebase = name
1487 else:
1488 namebase = getTestOpts().with_namebase
1489
1490 actual_stderr_file = qualify(name, 'run.stderr')
1491 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1492
1493 def norm(str):
1494 if platform_specific:
1495 return str
1496 else:
1497 return normalise_errmsg(str)
1498
1499 return compare_outputs('stderr', \
1500 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1501 expected_stderr_file, actual_stderr_file)
1502
1503 def dump_stderr( name ):
1504 print "Stderr:"
1505 print read_no_crs(qualify(name, 'run.stderr'))
1506
1507 def read_no_crs(file):
1508 str = ''
1509 try:
1510 h = open(file)
1511 str = h.read()
1512 h.close
1513 except:
1514 # On Windows, if the program fails very early, it seems the
1515 # files stdout/stderr are redirected to may not get created
1516 pass
1517 return re.sub('\r', '', str)
1518
1519 def write_file(file, str):
1520 h = open(file, 'w')
1521 h.write(str)
1522 h.close
1523
1524 def check_hp_ok(name):
1525
1526 # do not qualify for hp2ps because we should be in the right directory
1527 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1528
1529 hp2psResult = runCmdExitCode(hp2psCmd)
1530
1531 actual_ps_file = qualify(name, 'ps')
1532
1533 if(hp2psResult == 0):
1534 if (os.path.exists(actual_ps_file)):
1535 if gs_working:
1536 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1537 if (gsResult == 0):
1538 return (True)
1539 else:
1540 print "hp2ps output for " + name + "is not valid PostScript"
1541 else: return (True) # assume postscript is valid without ghostscript
1542 else:
1543 print "hp2ps did not generate PostScript for " + name
1544 return (False)
1545 else:
1546 print "hp2ps error when processing heap profile for " + name
1547 return(False)
1548
1549 def check_prof_ok(name):
1550
1551 prof_file = qualify(name,'prof')
1552
1553 if not os.path.exists(prof_file):
1554 print prof_file + " does not exist"
1555 return(False)
1556
1557 if os.path.getsize(qualify(name,'prof')) == 0:
1558 print prof_file + " is empty"
1559 return(False)
1560
1561 if getTestOpts().with_namebase == None:
1562 namebase = name
1563 else:
1564 namebase = getTestOpts().with_namebase
1565
1566 (platform_specific, expected_prof_file) = \
1567 platform_wordsize_qualify(namebase, 'prof.sample')
1568
1569 # sample prof file is not required
1570 if not os.path.exists(expected_prof_file):
1571 return True
1572 else:
1573 return compare_outputs('prof', \
1574 two_normalisers(normalise_whitespace,normalise_prof), \
1575 expected_prof_file, prof_file)
1576
1577 # Compare expected output to actual output, and optionally accept the
1578 # new output. Returns true if output matched or was accepted, false
1579 # otherwise.
1580 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1581 if os.path.exists(expected_file):
1582 expected_raw = read_no_crs(expected_file)
1583 # print "norm:", normaliser(expected_raw)
1584 expected_str = normaliser(expected_raw)
1585 expected_file_for_diff = expected_file
1586 else:
1587 expected_str = ''
1588 expected_file_for_diff = '/dev/null'
1589
1590 actual_raw = read_no_crs(actual_file)
1591 actual_str = normaliser(actual_raw)
1592
1593 if expected_str == actual_str:
1594 return 1
1595 else:
1596 print 'Actual ' + kind + ' output differs from expected:'
1597
1598 if expected_file_for_diff == '/dev/null':
1599 expected_normalised_file = '/dev/null'
1600 else:
1601 expected_normalised_file = expected_file + ".normalised"
1602 write_file(expected_normalised_file, expected_str)
1603
1604 actual_normalised_file = actual_file + ".normalised"
1605 write_file(actual_normalised_file, actual_str)
1606
1607 # Ignore whitespace when diffing. We should only get to this
1608 # point if there are non-whitespace differences
1609 #
1610 # Note we are diffing the *actual* output, not the normalised
1611 # output. The normalised output may have whitespace squashed
1612 # (including newlines) so the diff would be hard to read.
1613 # This does mean that the diff might contain changes that
1614 # would be normalised away.
1615 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1616 ' ' + actual_file )
1617
1618 # If for some reason there were no non-whitespace differences,
1619 # then do a full diff
1620 if r == 0:
1621 r = os.system( 'diff -u ' + expected_file_for_diff + \
1622 ' ' + actual_file )
1623
1624 if config.accept:
1625 print 'Accepting new output.'
1626 write_file(expected_file, actual_raw)
1627 return 1
1628 else:
1629 return 0
1630
1631
1632 def normalise_whitespace( str ):
1633 # Merge contiguous whitespace characters into a single space.
1634 str = re.sub('[ \t\n]+', ' ', str)
1635 return str
1636
1637 def normalise_errmsg( str ):
1638 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1639 # the colon is there because it appears in error messages; this
1640 # hacky solution is used in place of more sophisticated filename
1641 # mangling
1642 str = re.sub('([^\\s])\\.exe', '\\1', str)
1643 # normalise slashes, minimise Windows/Unix filename differences
1644 str = re.sub('\\\\', '/', str)
1645 # The inplace ghc's are called ghc-stage[123] to avoid filename
1646 # collisions, so we need to normalise that to just "ghc"
1647 str = re.sub('ghc-stage[123]', 'ghc', str)
1648 # We sometimes see the name of the integer-gmp package on stderr,
1649 # but this can change (either the implementation name or the
1650 # version number), so we canonicalise it here
1651 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1652 return str
1653
1654 # normalise a .prof file, so that we can reasonably compare it against
1655 # a sample. This doesn't compare any of the actual profiling data,
1656 # only the shape of the profile and the number of entries.
1657 def normalise_prof (str):
1658 # strip everything up to the line beginning "COST CENTRE"
1659 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1660
1661 # strip results for CAFs, these tend to change unpredictably
1662 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1663
1664 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1665 # sometimes under MAIN.
1666 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1667
1668 # We have somthing like this:
1669
1670 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1671 # k Main 204 1 0.0 0.0 0.0 0.0
1672 # foo Main 205 1 0.0 0.0 0.0 0.0
1673 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1674
1675 # then we remove all the specific profiling data, leaving only the
1676 # cost centre name, module, and entries, to end up with this:
1677
1678 # MAIN MAIN 0
1679 # k Main 1
1680 # foo Main 1
1681 # foo.bar Main 1
1682
1683 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1684 return str
1685
1686 def normalise_slashes_( str ):
1687 str = re.sub('\\\\', '/', str)
1688 return str
1689
1690 def normalise_exe_( str ):
1691 str = re.sub('\.exe', '', str)
1692 return str
1693
1694 def normalise_output( str ):
1695 # Remove a .exe extension (for Windows)
1696 # This can occur in error messages generated by the program.
1697 str = re.sub('([^\\s])\\.exe', '\\1', str)
1698 return str
1699
1700 def normalise_asm( str ):
1701 lines = str.split('\n')
1702 # Only keep instructions and labels not starting with a dot.
1703 metadata = re.compile('^[ \t]*\\..*$')
1704 out = []
1705 for line in lines:
1706 # Drop metadata directives (e.g. ".type")
1707 if not metadata.match(line):
1708 line = re.sub('@plt', '', line)
1709 instr = line.lstrip().split()
1710 # Drop empty lines.
1711 if not instr:
1712 continue
1713 # Drop operands, except for call instructions.
1714 elif instr[0] == 'call':
1715 out.append(instr[0] + ' ' + instr[1])
1716 else:
1717 out.append(instr[0])
1718 out = '\n'.join(out)
1719 return out
1720
1721 def if_verbose( n, str ):
1722 if config.verbose >= n:
1723 print str
1724
1725 def if_verbose_dump( n, f ):
1726 if config.verbose >= n:
1727 try:
1728 print open(f).read()
1729 except:
1730 print ''
1731
1732 def rawSystem(cmd_and_args):
1733 # We prefer subprocess.call to os.spawnv as the latter
1734 # seems to send its arguments through a shell or something
1735 # with the Windows (non-cygwin) python. An argument "a b c"
1736 # turns into three arguments ["a", "b", "c"].
1737
1738 # However, subprocess is new in python 2.4, so fall back to
1739 # using spawnv if we don't have it
1740
1741 if have_subprocess:
1742 return subprocess.call(cmd_and_args)
1743 else:
1744 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1745
1746 # Note that this doesn't handle the timeout itself; it is just used for
1747 # commands that have timeout handling built-in.
1748 def rawSystemWithTimeout(cmd_and_args):
1749 r = rawSystem(cmd_and_args)
1750 if r == 98:
1751 # The python timeout program uses 98 to signal that ^C was pressed
1752 stopNow()
1753 return r
1754
1755 # cmd is a complex command in Bourne-shell syntax
1756 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1757 # Hence it must ultimately be run by a Bourne shell
1758 #
1759 # Mostly it invokes the command wrapped in 'timeout' thus
1760 # timeout 300 'cd . && ...blah blah'
1761 # so it's timeout's job to invoke the Bourne shell
1762 #
1763 # But watch out for the case when there is no timeout program!
1764 # Then, when using the native Python, os.system will invoke the cmd shell
1765
1766 def runCmd( cmd ):
1767 if_verbose( 1, cmd )
1768 r = 0
1769 if config.os == 'mingw32':
1770 # On MinGW, we will always have timeout
1771 assert config.timeout_prog!=''
1772
1773 if config.timeout_prog != '':
1774 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1775 else:
1776 r = os.system(cmd)
1777 return r << 8
1778
1779 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1780 if_verbose( 1, cmd )
1781 r = 0
1782 if config.os == 'mingw32':
1783 # On MinGW, we will always have timeout
1784 assert config.timeout_prog!=''
1785 timeout = int(ceil(config.timeout * timeout_multiplier))
1786
1787 if config.timeout_prog != '':
1788 if config.check_files_written:
1789 fn = name + ".strace"
1790 r = rawSystemWithTimeout(
1791 ["strace", "-o", fn, "-fF",
1792 "-e", "creat,open,chdir,clone,vfork",
1793 config.timeout_prog, str(timeout), cmd])
1794 addTestFilesWritten(name, fn)
1795 rm_no_fail(fn)
1796 else:
1797 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1798 else:
1799 r = os.system(cmd)
1800 return r << 8
1801
1802 def runCmdExitCode( cmd ):
1803 return (runCmd(cmd) >> 8);
1804
1805
1806 # -----------------------------------------------------------------------------
1807 # checking for files being written to by multiple tests
1808
1809 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1810 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1811 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1812 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1813 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1814 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1815 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1816 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1817 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1818 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1819 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1820 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1821 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1822 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1823 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1824 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1825 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1826
1827 # Files that are read or written but shouldn't be:
1828 # * ghci_history shouldn't be read or written by tests
1829 # * things under package.conf.d shouldn't be written by tests
1830 bad_file_usages = {}
1831
1832 # Mapping from tests to the list of files that they write
1833 files_written = {}
1834
1835 # Mapping from tests to the list of files that they write but don't clean
1836 files_written_not_removed = {}
1837
1838 def add_bad_file_usage(name, file):
1839 try:
1840 if not file in bad_file_usages[name]:
1841 bad_file_usages[name].append(file)
1842 except:
1843 bad_file_usages[name] = [file]
1844
1845 def mkPath(curdir, path):
1846 # Given the current full directory is 'curdir', what is the full
1847 # path to 'path'?
1848 return os.path.realpath(os.path.join(curdir, path))
1849
1850 def addTestFilesWritten(name, fn):
1851 if config.use_threads:
1852 with t.lockFilesWritten:
1853 addTestFilesWrittenHelper(name, fn)
1854 else:
1855 addTestFilesWrittenHelper(name, fn)
1856
1857 def addTestFilesWrittenHelper(name, fn):
1858 started = False
1859 working_directories = {}
1860
1861 with open(fn, 'r') as f:
1862 for line in f:
1863 m_pid = re_strace_pid.match(line)
1864 if m_pid:
1865 pid = m_pid.group(1)
1866 content = m_pid.group(2)
1867 elif re_strace_unavailable.match(line):
1868 next
1869 else:
1870 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1871
1872 m_open = re_strace_open.match(content)
1873 m_chdir = re_strace_chdir.match(content)
1874 m_clone = re_strace_clone.match(content)
1875 m_vfork = re_strace_vfork.match(content)
1876
1877 if not started:
1878 working_directories[pid] = os.getcwd()
1879 started = True
1880
1881 if m_open:
1882 file = m_open.group(1)
1883 file = mkPath(working_directories[pid], file)
1884 if file.endswith("ghci_history"):
1885 add_bad_file_usage(name, file)
1886 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1887 flags = m_open.group(2).split('|')
1888 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1889 if re.match('package\.conf\.d', file):
1890 add_bad_file_usage(name, file)
1891 else:
1892 try:
1893 if not file in files_written[name]:
1894 files_written[name].append(file)
1895 except:
1896 files_written[name] = [file]
1897 elif 'O_RDONLY' in flags:
1898 pass
1899 else:
1900 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1901 elif m_chdir:
1902 # We optimistically assume that unfinished chdir's are going to succeed
1903 dir = m_chdir.group(1)
1904 working_directories[pid] = mkPath(working_directories[pid], dir)
1905 elif m_clone:
1906 working_directories[m_clone.group(2)] = working_directories[pid]
1907 elif m_vfork:
1908 working_directories[m_vfork.group(2)] = working_directories[pid]
1909 elif re_strace_open_resumed.match(content):
1910 pass
1911 elif re_strace_chdir_resumed.match(content):
1912 pass
1913 elif re_strace_vfork_unfinished.match(content):
1914 pass
1915 elif re_strace_clone_unfinished.match(content):
1916 pass
1917 elif re_strace_ignore_sigchild.match(content):
1918 pass
1919 elif re_strace_ignore_sigvtalarm.match(content):
1920 pass
1921 elif re_strace_ignore_sigint.match(content):
1922 pass
1923 elif re_strace_ignore_sigfpe.match(content):
1924 pass
1925 elif re_strace_ignore_sigsegv.match(content):
1926 pass
1927 elif re_strace_ignore_sigpipe.match(content):
1928 pass
1929 else:
1930 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1931
1932 def checkForFilesWrittenProblems(file):
1933 foundProblem = False
1934
1935 files_written_inverted = {}
1936 for t in files_written.keys():
1937 for f in files_written[t]:
1938 try:
1939 files_written_inverted[f].append(t)
1940 except:
1941 files_written_inverted[f] = [t]
1942
1943 for f in files_written_inverted.keys():
1944 if len(files_written_inverted[f]) > 1:
1945 if not foundProblem:
1946 foundProblem = True
1947 file.write("\n")
1948 file.write("\nSome files are written by multiple tests:\n")
1949 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1950 if foundProblem:
1951 file.write("\n")
1952
1953 # -----
1954
1955 if len(files_written_not_removed) > 0:
1956 file.write("\n")
1957 file.write("\nSome files written but not removed:\n")
1958 tests = files_written_not_removed.keys()
1959 tests.sort()
1960 for t in tests:
1961 for f in files_written_not_removed[t]:
1962 file.write(" " + t + ": " + f + "\n")
1963 file.write("\n")
1964
1965 # -----
1966
1967 if len(bad_file_usages) > 0:
1968 file.write("\n")
1969 file.write("\nSome bad file usages:\n")
1970 tests = bad_file_usages.keys()
1971 tests.sort()
1972 for t in tests:
1973 for f in bad_file_usages[t]:
1974 file.write(" " + t + ": " + f + "\n")
1975 file.write("\n")
1976
1977 # -----------------------------------------------------------------------------
1978 # checking if ghostscript is available for checking the output of hp2ps
1979
1980 def genGSCmd(psfile):
1981 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1982
1983 def gsNotWorking():
1984 global gs_working
1985 print "GhostScript not available for hp2ps tests"
1986
1987 global gs_working
1988 gs_working = 0
1989 if config.have_profiling:
1990 if config.gs != '':
1991 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
1992 if resultGood == 0:
1993 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
1994 if resultBad != 0:
1995 print "GhostScript available for hp2ps tests"
1996 gs_working = 1;
1997 else:
1998 gsNotWorking();
1999 else:
2000 gsNotWorking();
2001 else:
2002 gsNotWorking();
2003
2004 def rm_no_fail( file ):
2005 try:
2006 os.remove( file )
2007 finally:
2008 return
2009
2010 def add_suffix( name, suffix ):
2011 if suffix == '':
2012 return name
2013 else:
2014 return name + '.' + suffix
2015
2016 def add_hs_lhs_suffix(name):
2017 if getTestOpts().c_src:
2018 return add_suffix(name, 'c')
2019 elif getTestOpts().cmm_src:
2020 return add_suffix(name, 'cmm')
2021 elif getTestOpts().objc_src:
2022 return add_suffix(name, 'm')
2023 elif getTestOpts().objcpp_src:
2024 return add_suffix(name, 'mm')
2025 elif getTestOpts().literate:
2026 return add_suffix(name, 'lhs')
2027 else:
2028 return add_suffix(name, 'hs')
2029
2030 def replace_suffix( name, suffix ):
2031 base, suf = os.path.splitext(name)
2032 return base + '.' + suffix
2033
2034 def in_testdir( name ):
2035 return (getTestOpts().testdir + '/' + name)
2036
2037 def qualify( name, suff ):
2038 return in_testdir(add_suffix(name, suff))
2039
2040
2041 # Finding the sample output. The filename is of the form
2042 #
2043 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2044 #
2045 # and we pick the most specific version available. The <version> is
2046 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2047 # more fine-grained control use if_compiler_lt().
2048 #
2049 def platform_wordsize_qualify( name, suff ):
2050
2051 basepath = qualify(name, suff)
2052
2053 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2054 for (platformSpecific, plat) in [(1, '-' + config.platform),
2055 (1, '-' + config.os),
2056 (0, '')]
2057 for ws in ['-ws-' + config.wordsize, '']
2058 for comp in ['-' + config.compiler_type, '']
2059 for vers in ['-' + config.compiler_maj_version, '']]
2060
2061 dir = glob.glob(basepath + '*')
2062 dir = map (lambda d: normalise_slashes_(d), dir)
2063
2064 for (platformSpecific, f) in paths:
2065 if f in dir:
2066 return (platformSpecific,f)
2067
2068 return (0, basepath)
2069
2070 # Clean up prior to the test, so that we can't spuriously conclude
2071 # that it passed on the basis of old run outputs.
2072 def pretest_cleanup(name):
2073 if getTestOpts().outputdir != None:
2074 odir = in_testdir(getTestOpts().outputdir)
2075 try:
2076 shutil.rmtree(odir)
2077 except:
2078 pass
2079 os.mkdir(odir)
2080
2081 rm_no_fail(qualify(name,'interp.stderr'))
2082 rm_no_fail(qualify(name,'interp.stdout'))
2083 rm_no_fail(qualify(name,'comp.stderr'))
2084 rm_no_fail(qualify(name,'comp.stdout'))
2085 rm_no_fail(qualify(name,'run.stderr'))
2086 rm_no_fail(qualify(name,'run.stdout'))
2087 rm_no_fail(qualify(name,'tix'))
2088 rm_no_fail(qualify(name,'exe.tix'))
2089 # simple_build zaps the following:
2090 # rm_nofail(qualify("o"))
2091 # rm_nofail(qualify(""))
2092 # not interested in the return code
2093
2094 # -----------------------------------------------------------------------------
2095 # Return a list of all the files ending in '.T' below the directory dir.
2096
2097 def findTFiles(roots):
2098 return concat(map(findTFiles_,roots))
2099
2100 def findTFiles_(path):
2101 if os.path.isdir(path):
2102 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2103 return findTFiles(paths)
2104 elif path[-2:] == '.T':
2105 return [path]
2106 else:
2107 return []
2108
2109 # -----------------------------------------------------------------------------
2110 # Output a test summary to the specified file object
2111
2112 def summary(t, file):
2113
2114 file.write('\n')
2115 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2116 file.write('OVERALL SUMMARY for test run started at '
2117 + t.start_time + '\n'
2118 + string.rjust(`t.total_tests`, 8)
2119 + ' total tests, which gave rise to\n'
2120 + string.rjust(`t.total_test_cases`, 8)
2121 + ' test cases, of which\n'
2122 + string.rjust(`t.n_tests_skipped`, 8)
2123 + ' were skipped\n'
2124 + '\n'
2125 + string.rjust(`t.n_missing_libs`, 8)
2126 + ' had missing libraries\n'
2127 + string.rjust(`t.n_expected_passes`, 8)
2128 + ' expected passes\n'
2129 + string.rjust(`t.n_expected_failures`, 8)
2130 + ' expected failures\n'
2131 + '\n'
2132 + string.rjust(`t.n_framework_failures`, 8)
2133 + ' caused framework failures\n'
2134 + string.rjust(`t.n_unexpected_passes`, 8)
2135 + ' unexpected passes\n'
2136 + string.rjust(`t.n_unexpected_failures`, 8)
2137 + ' unexpected failures\n'
2138 + '\n')
2139
2140 if t.n_unexpected_passes > 0:
2141 file.write('Unexpected passes:\n')
2142 printPassingTestInfosSummary(file, t.unexpected_passes)
2143
2144 if t.n_unexpected_failures > 0:
2145 file.write('Unexpected failures:\n')
2146 printFailingTestInfosSummary(file, t.unexpected_failures)
2147
2148 if config.check_files_written:
2149 checkForFilesWrittenProblems(file)
2150
2151 if stopping():
2152 file.write('WARNING: Testsuite run was terminated early\n')
2153
2154 def printUnexpectedTests(file, testInfoss):
2155 unexpected = []
2156 for testInfos in testInfoss:
2157 directories = testInfos.keys()
2158 for directory in directories:
2159 tests = testInfos[directory].keys()
2160 unexpected += tests
2161 if unexpected != []:
2162 file.write('Unexpected results from:\n')
2163 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2164 file.write('\n')
2165
2166 def printPassingTestInfosSummary(file, testInfos):
2167 directories = testInfos.keys()
2168 directories.sort()
2169 maxDirLen = max(map ((lambda x : len(x)), directories))
2170 for directory in directories:
2171 tests = testInfos[directory].keys()
2172 tests.sort()
2173 for test in tests:
2174 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2175 ' (' + join(testInfos[directory][test],',') + ')\n')
2176 file.write('\n')
2177
2178 def printFailingTestInfosSummary(file, testInfos):
2179 directories = testInfos.keys()
2180 directories.sort()
2181 maxDirLen = max(map ((lambda x : len(x)), directories))
2182 for directory in directories:
2183 tests = testInfos[directory].keys()
2184 tests.sort()
2185 for test in tests:
2186 reasons = testInfos[directory][test].keys()
2187 for reason in reasons:
2188 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2189 ' [' + reason + ']' + \
2190 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2191 file.write('\n')
2192
2193 def getStdout(cmd):
2194 if have_subprocess:
2195 p = subprocess.Popen(cmd,
2196 stdout=subprocess.PIPE,
2197 stderr=subprocess.PIPE)
2198 (stdout, stderr) = p.communicate()
2199 r = p.wait()
2200 if r != 0:
2201 raise Exception("Command failed: " + str(cmd))
2202 if stderr != '':
2203 raise Exception("stderr from command: " + str(cmd))
2204 return stdout
2205 else:
2206 raise Exception("Need subprocess to get stdout, but don't have it")