Fold base.git into ghc.git (re #8545)
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 import types
20 from math import ceil, trunc
21
22 have_subprocess = False
23 try:
24 import subprocess
25 have_subprocess = True
26 except:
27 print "Warning: subprocess not found, will fall back to spawnv"
28
29 from string import join
30 from testglobals import *
31 from testutil import *
32
33 if config.use_threads:
34 import threading
35 import thread
36
37 global wantToStop
38 wantToStop = False
39 def stopNow():
40 global wantToStop
41 wantToStop = True
42 def stopping():
43 return wantToStop
44
45 # Options valid for the current test only (these get reset to
46 # testdir_testopts after each test).
47
48 global testopts_local
49 if config.use_threads:
50 testopts_local = threading.local()
51 else:
52 class TestOpts_Local:
53 pass
54 testopts_local = TestOpts_Local()
55
56 def getTestOpts():
57 return testopts_local.x
58
59 def setLocalTestOpts(opts):
60 global testopts_local
61 testopts_local.x=opts
62
63 def isStatsTest():
64 opts = getTestOpts()
65 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
66
67
68 # This can be called at the top of a file of tests, to set default test options
69 # for the following tests.
70 def setTestOpts( f ):
71 global thisdir_settings
72 thisdir_settings = [thisdir_settings, f]
73
74 # -----------------------------------------------------------------------------
75 # Canned setup functions for common cases. eg. for a test you might say
76 #
77 # test('test001', normal, compile, [''])
78 #
79 # to run it without any options, but change it to
80 #
81 # test('test001', expect_fail, compile, [''])
82 #
83 # to expect failure for this test.
84
85 def normal( name, opts ):
86 return;
87
88 def skip( name, opts ):
89 opts.skip = 1
90
91 def expect_fail( name, opts ):
92 opts.expect = 'fail';
93
94 def reqlib( lib ):
95 return lambda name, opts, l=lib: _reqlib (name, opts, l )
96
97 # Cache the results of looking to see if we have a library or not.
98 # This makes quite a difference, especially on Windows.
99 have_lib = {}
100
101 def _reqlib( name, opts, lib ):
102 if have_lib.has_key(lib):
103 got_it = have_lib[lib]
104 else:
105 if have_subprocess:
106 # By preference we use subprocess, as the alternative uses
107 # /dev/null which mingw doesn't have.
108 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 stdout=subprocess.PIPE,
110 stderr=subprocess.PIPE)
111 # read from stdout and stderr to avoid blocking due to
112 # buffers filling
113 p.communicate()
114 r = p.wait()
115 else:
116 r = os.system(config.ghc_pkg + ' describe ' + lib
117 + ' > /dev/null 2> /dev/null')
118 got_it = r == 0
119 have_lib[lib] = got_it
120
121 if not got_it:
122 opts.expect = 'missing-lib'
123
124 def req_profiling( name, opts ):
125 if not config.have_profiling:
126 opts.expect = 'fail'
127
128 def req_shared_libs( name, opts ):
129 if not config.have_shared_libs:
130 opts.expect = 'fail'
131
132 def req_interp( name, opts ):
133 if not config.have_interp:
134 opts.expect = 'fail'
135
136 def req_smp( name, opts ):
137 if not config.have_smp:
138 opts.expect = 'fail'
139
140 def ignore_output( name, opts ):
141 opts.ignore_output = 1
142
143 def no_stdin( name, opts ):
144 opts.no_stdin = 1
145
146 def combined_output( name, opts ):
147 opts.combined_output = True
148
149 # -----
150
151 def expect_fail_for( ways ):
152 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153
154 def _expect_fail_for( name, opts, ways ):
155 opts.expect_fail_for = ways
156
157 def expect_broken( bug ):
158 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
159
160 def _expect_broken( name, opts, bug ):
161 record_broken(name, opts, bug)
162 opts.expect = 'fail';
163
164 def expect_broken_for( bug, ways ):
165 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
166
167 def _expect_broken_for( name, opts, bug, ways ):
168 record_broken(name, opts, bug)
169 opts.expect_fail_for = ways
170
171 def record_broken(name, opts, bug):
172 global brokens
173 me = (bug, opts.testdir, name)
174 if not me in brokens:
175 brokens.append(me)
176
177 # -----
178
179 def omit_ways( ways ):
180 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181
182 def _omit_ways( name, opts, ways ):
183 opts.omit_ways = ways
184
185 # -----
186
187 def only_ways( ways ):
188 return lambda name, opts, w=ways: _only_ways( name, opts, w )
189
190 def _only_ways( name, opts, ways ):
191 opts.only_ways = ways
192
193 # -----
194
195 def extra_ways( ways ):
196 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197
198 def _extra_ways( name, opts, ways ):
199 opts.extra_ways = ways
200
201 # -----
202
203 def omit_compiler_types( compiler_types ):
204 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
205
206 def _omit_compiler_types( name, opts, compiler_types ):
207 if config.compiler_type in compiler_types:
208 opts.skip = 1
209
210 # -----
211
212 def only_compiler_types( compiler_types ):
213 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
214
215 def _only_compiler_types( name, opts, compiler_types ):
216 if config.compiler_type not in compiler_types:
217 opts.skip = 1
218
219 # -----
220
221 def set_stdin( file ):
222 return lambda name, opts, f=file: _set_stdin(name, opts, f);
223
224 def _set_stdin( name, opts, f ):
225 opts.stdin = f
226
227 # -----
228
229 def exit_code( val ):
230 return lambda name, opts, v=val: _exit_code(name, opts, v);
231
232 def _exit_code( name, opts, v ):
233 opts.exit_code = v
234
235 # -----
236
237 def timeout_multiplier( val ):
238 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239
240 def _timeout_multiplier( name, opts, v ):
241 opts.timeout_multiplier = v
242
243 # -----
244
245 def extra_run_opts( val ):
246 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247
248 def _extra_run_opts( name, opts, v ):
249 opts.extra_run_opts = v
250
251 # -----
252
253 def extra_hc_opts( val ):
254 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
255
256 def _extra_hc_opts( name, opts, v ):
257 opts.extra_hc_opts = v
258
259 # -----
260
261 def extra_clean( files ):
262 return lambda name, opts, v=files: _extra_clean(name, opts, v);
263
264 def _extra_clean( name, opts, v ):
265 opts.clean_files = v
266
267 # -----
268
269 def stats_num_field( field, expecteds ):
270 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
271
272 def _stats_num_field( name, opts, field, expecteds ):
273 if field in opts.stats_range_fields:
274 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
275
276 if type(expecteds) is types.ListType:
277 for (b, expected, dev) in expecteds:
278 if b:
279 opts.stats_range_fields[field] = (expected, dev)
280 return
281 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282
283 else:
284 (expected, dev) = expecteds
285 opts.stats_range_fields[field] = (expected, dev)
286
287 def compiler_stats_num_field( field, expecteds ):
288 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289
290 def _compiler_stats_num_field( name, opts, field, expecteds ):
291 if field in opts.compiler_stats_range_fields:
292 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
293
294 # Compiler performance numbers change when debugging is on, making the results
295 # useless and confusing. Therefore, skip if debugging is on.
296 if compiler_debugged():
297 skip(name, opts)
298
299 for (b, expected, dev) in expecteds:
300 if b:
301 opts.compiler_stats_range_fields[field] = (expected, dev)
302 return
303
304 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
305
306 # -----
307
308 def when(b, f):
309 # When list_brokens is on, we want to see all expect_broken calls,
310 # so we always do f
311 if b or config.list_broken:
312 return f
313 else:
314 return normal
315
316 def unless(b, f):
317 return when(not b, f)
318
319 def doing_ghci():
320 return 'ghci' in config.run_ways
321
322 def ghci_dynamic( ):
323 return config.ghc_dynamic
324
325 def fast():
326 return config.fast
327
328 def platform( plat ):
329 return config.platform == plat
330
331 def opsys( os ):
332 return config.os == os
333
334 def arch( arch ):
335 return config.arch == arch
336
337 def wordsize( ws ):
338 return config.wordsize == str(ws)
339
340 def msys( ):
341 return config.msys
342
343 def cygwin( ):
344 return config.cygwin
345
346 def have_vanilla( ):
347 return config.have_vanilla
348
349 def have_dynamic( ):
350 return config.have_dynamic
351
352 def have_profiling( ):
353 return config.have_profiling
354
355 def in_tree_compiler( ):
356 return config.in_tree_compiler
357
358 def compiler_type( compiler ):
359 return config.compiler_type == compiler
360
361 def compiler_lt( compiler, version ):
362 return config.compiler_type == compiler and \
363 version_lt(config.compiler_version, version)
364
365 def compiler_le( compiler, version ):
366 return config.compiler_type == compiler and \
367 version_le(config.compiler_version, version)
368
369 def compiler_gt( compiler, version ):
370 return config.compiler_type == compiler and \
371 version_gt(config.compiler_version, version)
372
373 def compiler_ge( compiler, version ):
374 return config.compiler_type == compiler and \
375 version_ge(config.compiler_version, version)
376
377 def unregisterised( ):
378 return config.unregisterised
379
380 def compiler_profiled( ):
381 return config.compiler_profiled
382
383 def compiler_debugged( ):
384 return config.compiler_debugged
385
386 def tag( t ):
387 return t in config.compiler_tags
388
389 # ---
390
391 def namebase( nb ):
392 return lambda opts, nb=nb: _namebase(opts, nb)
393
394 def _namebase( opts, nb ):
395 opts.with_namebase = nb
396
397 # ---
398
399 def high_memory_usage(name, opts):
400 opts.alone = True
401
402 # If a test is for a multi-CPU race, then running the test alone
403 # increases the chance that we'll actually see it.
404 def multi_cpu_race(name, opts):
405 opts.alone = True
406
407 # ---
408 def literate( name, opts ):
409 opts.literate = 1;
410
411 def c_src( name, opts ):
412 opts.c_src = 1;
413
414 def objc_src( name, opts ):
415 opts.objc_src = 1;
416
417 def objcpp_src( name, opts ):
418 opts.objcpp_src = 1;
419
420 def cmm_src( name, opts ):
421 opts.cmm_src = 1;
422
423 def outputdir( odir ):
424 return lambda name, opts, d=odir: _outputdir(name, opts, d)
425
426 def _outputdir( name, opts, odir ):
427 opts.outputdir = odir;
428
429 # ----
430
431 def pre_cmd( cmd ):
432 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
433
434 def _pre_cmd( name, opts, cmd ):
435 opts.pre_cmd = cmd
436
437 # ----
438
439 def clean_cmd( cmd ):
440 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
441
442 def _clean_cmd( name, opts, cmd ):
443 opts.clean_cmd = cmd
444
445 # ----
446
447 def cmd_prefix( prefix ):
448 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
449
450 def _cmd_prefix( name, opts, prefix ):
451 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
452
453 # ----
454
455 def cmd_wrapper( fun ):
456 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
457
458 def _cmd_wrapper( name, opts, fun ):
459 opts.cmd_wrapper = fun
460
461 # ----
462
463 def compile_cmd_prefix( prefix ):
464 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
465
466 def _compile_cmd_prefix( name, opts, prefix ):
467 opts.compile_cmd_prefix = prefix
468
469 # ----
470
471 def check_stdout( f ):
472 return lambda name, opts, f=f: _check_stdout(name, opts, f)
473
474 def _check_stdout( name, opts, f ):
475 opts.check_stdout = f
476
477 # ----
478
479 def normalise_slashes( name, opts ):
480 opts.extra_normaliser = normalise_slashes_
481
482 def normalise_exe( name, opts ):
483 opts.extra_normaliser = normalise_exe_
484
485 def normalise_fun( fun ):
486 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
487
488 def _normalise_fun( name, opts, f ):
489 opts.extra_normaliser = f
490
491 def normalise_errmsg_fun( fun ):
492 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
493
494 def _normalise_errmsg_fun( name, opts, f ):
495 opts.extra_errmsg_normaliser = f
496
497 def two_normalisers(f, g):
498 return lambda x, f=f, g=g: f(g(x))
499
500 # ----
501 # Function for composing two opt-fns together
502
503 def executeSetups(fs, name, opts):
504 if type(fs) is types.ListType:
505 # If we have a list of setups, then execute each one
506 map (lambda f : executeSetups(f, name, opts), fs)
507 else:
508 # fs is a single function, so just apply it
509 fs(name, opts)
510
511 # -----------------------------------------------------------------------------
512 # The current directory of tests
513
514 def newTestDir( dir ):
515 global thisdir_settings
516 # reset the options for this test directory
517 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
518
519 def _newTestDir( name, opts, dir ):
520 opts.testdir = dir
521 opts.compiler_always_flags = config.compiler_always_flags
522
523 # -----------------------------------------------------------------------------
524 # Actually doing tests
525
526 parallelTests = []
527 aloneTests = []
528 allTestNames = set([])
529
530 def runTest (opts, name, func, args):
531 ok = 0
532
533 if config.use_threads:
534 t.thread_pool.acquire()
535 try:
536 while config.threads<(t.running_threads+1):
537 t.thread_pool.wait()
538 t.running_threads = t.running_threads+1
539 ok=1
540 t.thread_pool.release()
541 thread.start_new_thread(test_common_thread, (name, opts, func, args))
542 except:
543 if not ok:
544 t.thread_pool.release()
545 else:
546 test_common_work (name, opts, func, args)
547
548 # name :: String
549 # setup :: TestOpts -> IO ()
550 def test (name, setup, func, args):
551 global aloneTests
552 global parallelTests
553 global allTestNames
554 global thisdir_settings
555 if name in allTestNames:
556 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
557 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
558 framework_fail(name, 'bad_name', 'This test has an invalid name')
559
560 # Make a deep copy of the default_testopts, as we need our own copy
561 # of any dictionaries etc inside it. Otherwise, if one test modifies
562 # them, all tests will see the modified version!
563 myTestOpts = copy.deepcopy(default_testopts)
564
565 executeSetups([thisdir_settings, setup], name, myTestOpts)
566
567 thisTest = lambda : runTest(myTestOpts, name, func, args)
568 if myTestOpts.alone:
569 aloneTests.append(thisTest)
570 else:
571 parallelTests.append(thisTest)
572 allTestNames.add(name)
573
574 if config.use_threads:
575 def test_common_thread(name, opts, func, args):
576 t.lock.acquire()
577 try:
578 test_common_work(name,opts,func,args)
579 finally:
580 t.lock.release()
581 t.thread_pool.acquire()
582 t.running_threads = t.running_threads - 1
583 t.thread_pool.notify()
584 t.thread_pool.release()
585
586 def get_package_cache_timestamp():
587 if config.package_conf_cache_file == '':
588 return 0.0
589 else:
590 try:
591 return os.stat(config.package_conf_cache_file).st_mtime
592 except:
593 return 0.0
594
595
596 def test_common_work (name, opts, func, args):
597 try:
598 t.total_tests = t.total_tests+1
599 setLocalTestOpts(opts)
600
601 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
602
603 # All the ways we might run this test
604 if func == compile or func == multimod_compile:
605 all_ways = config.compile_ways
606 elif func == compile_and_run or func == multimod_compile_and_run:
607 all_ways = config.run_ways
608 elif func == ghci_script:
609 if 'ghci' in config.run_ways:
610 all_ways = ['ghci']
611 else:
612 all_ways = []
613 else:
614 all_ways = ['normal']
615
616 # A test itself can request extra ways by setting opts.extra_ways
617 all_ways = all_ways + filter(lambda way: way not in all_ways,
618 opts.extra_ways)
619
620 t.total_test_cases = t.total_test_cases + len(all_ways)
621
622 ok_way = lambda way: \
623 not getTestOpts().skip \
624 and (config.only == [] or name in config.only) \
625 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
626 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
627 and (not (config.skip_perf_tests and isStatsTest())) \
628 and way not in getTestOpts().omit_ways
629
630 # Which ways we are asked to skip
631 do_ways = filter (ok_way,all_ways)
632
633 # In fast mode, we skip all but one way
634 if config.fast and len(do_ways) > 0:
635 do_ways = [do_ways[0]]
636
637 if not config.clean_only:
638 # Run the required tests...
639 for way in do_ways:
640 if stopping():
641 break
642 do_test (name, way, func, args)
643
644 for way in all_ways:
645 if way not in do_ways:
646 skiptest (name,way)
647
648 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
649 pretest_cleanup(name)
650 clean(map (lambda suff: name + suff,
651 ['', '.exe', '.exe.manifest', '.genscript',
652 '.stderr.normalised', '.stdout.normalised',
653 '.run.stderr.normalised', '.run.stdout.normalised',
654 '.comp.stderr.normalised', '.comp.stdout.normalised',
655 '.interp.stderr.normalised', '.interp.stdout.normalised',
656 '.stats', '.comp.stats',
657 '.hi', '.o', '.prof', '.exe.prof', '.hc',
658 '_stub.h', '_stub.c', '_stub.o',
659 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
660
661 if func == multi_compile or func == multi_compile_fail:
662 extra_mods = args[1]
663 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
664 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
665
666 clean(getTestOpts().clean_files)
667
668 if getTestOpts().outputdir != None:
669 odir = in_testdir(getTestOpts().outputdir)
670 try:
671 shutil.rmtree(odir)
672 except:
673 pass
674
675 try:
676 shutil.rmtree(in_testdir('.hpc.' + name))
677 except:
678 pass
679
680 try:
681 cleanCmd = getTestOpts().clean_cmd
682 if cleanCmd != None:
683 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
684 if result != 0:
685 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
686 except:
687 framework_fail(name, 'cleaning', 'clean-command exception')
688
689 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
690
691 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
692 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
693
694 try:
695 for f in files_written[name]:
696 if os.path.exists(f):
697 try:
698 if not f in files_written_not_removed[name]:
699 files_written_not_removed[name].append(f)
700 except:
701 files_written_not_removed[name] = [f]
702 except:
703 pass
704 except Exception, e:
705 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
706
707 def clean(strs):
708 for str in strs:
709 for name in glob.glob(in_testdir(str)):
710 clean_full_path(name)
711
712 def clean_full_path(name):
713 try:
714 # Remove files...
715 os.remove(name)
716 except OSError, e1:
717 try:
718 # ... and empty directories
719 os.rmdir(name)
720 except OSError, e2:
721 # We don't want to fail here, but we do want to know
722 # what went wrong, so print out the exceptions.
723 # ENOENT isn't a problem, though, as we clean files
724 # that don't necessarily exist.
725 if e1.errno != errno.ENOENT:
726 print e1
727 if e2.errno != errno.ENOENT:
728 print e2
729
730 def do_test(name, way, func, args):
731 full_name = name + '(' + way + ')'
732
733 try:
734 if_verbose(2, "=====> %s %d of %d %s " % \
735 (full_name, t.total_tests, len(allTestNames), \
736 [t.n_unexpected_passes, \
737 t.n_unexpected_failures, \
738 t.n_framework_failures]))
739
740 if config.use_threads:
741 t.lock.release()
742
743 try:
744 preCmd = getTestOpts().pre_cmd
745 if preCmd != None:
746 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
747 if result != 0:
748 framework_fail(name, way, 'pre-command failed: ' + str(result))
749 except:
750 framework_fail(name, way, 'pre-command exception')
751
752 try:
753 result = apply(func, [name,way] + args)
754 finally:
755 if config.use_threads:
756 t.lock.acquire()
757
758 if getTestOpts().expect != 'pass' and \
759 getTestOpts().expect != 'fail' and \
760 getTestOpts().expect != 'missing-lib':
761 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
762
763 try:
764 passFail = result['passFail']
765 except:
766 passFail = 'No passFail found'
767
768 if passFail == 'pass':
769 if getTestOpts().expect == 'pass' \
770 and way not in getTestOpts().expect_fail_for:
771 t.n_expected_passes = t.n_expected_passes + 1
772 if name in t.expected_passes:
773 t.expected_passes[name].append(way)
774 else:
775 t.expected_passes[name] = [way]
776 else:
777 if_verbose(1, '*** unexpected pass for %s' % full_name)
778 t.n_unexpected_passes = t.n_unexpected_passes + 1
779 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
780 elif passFail == 'fail':
781 if getTestOpts().expect == 'pass' \
782 and way not in getTestOpts().expect_fail_for:
783 if_verbose(1, '*** unexpected failure for %s' % full_name)
784 t.n_unexpected_failures = t.n_unexpected_failures + 1
785 reason = result['reason']
786 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
787 else:
788 if getTestOpts().expect == 'missing-lib':
789 t.n_missing_libs = t.n_missing_libs + 1
790 if name in t.missing_libs:
791 t.missing_libs[name].append(way)
792 else:
793 t.missing_libs[name] = [way]
794 else:
795 t.n_expected_failures = t.n_expected_failures + 1
796 if name in t.expected_failures:
797 t.expected_failures[name].append(way)
798 else:
799 t.expected_failures[name] = [way]
800 else:
801 framework_fail(name, way, 'bad result ' + passFail)
802 except KeyboardInterrupt:
803 stopNow()
804 except:
805 framework_fail(name, way, 'do_test exception')
806 traceback.print_exc()
807
808 def addPassingTestInfo (testInfos, directory, name, way):
809 directory = re.sub('^\\.[/\\\\]', '', directory)
810
811 if not directory in testInfos:
812 testInfos[directory] = {}
813
814 if not name in testInfos[directory]:
815 testInfos[directory][name] = []
816
817 testInfos[directory][name].append(way)
818
819 def addFailingTestInfo (testInfos, directory, name, reason, way):
820 directory = re.sub('^\\.[/\\\\]', '', directory)
821
822 if not directory in testInfos:
823 testInfos[directory] = {}
824
825 if not name in testInfos[directory]:
826 testInfos[directory][name] = {}
827
828 if not reason in testInfos[directory][name]:
829 testInfos[directory][name][reason] = []
830
831 testInfos[directory][name][reason].append(way)
832
833 def skiptest (name, way):
834 # print 'Skipping test \"', name, '\"'
835 t.n_tests_skipped = t.n_tests_skipped + 1
836 if name in t.tests_skipped:
837 t.tests_skipped[name].append(way)
838 else:
839 t.tests_skipped[name] = [way]
840
841 def framework_fail( name, way, reason ):
842 full_name = name + '(' + way + ')'
843 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
844 t.n_framework_failures = t.n_framework_failures + 1
845 if name in t.framework_failures:
846 t.framework_failures[name].append(way)
847 else:
848 t.framework_failures[name] = [way]
849
850 def badResult(result):
851 try:
852 if result['passFail'] == 'pass':
853 return False
854 return True
855 except:
856 return True
857
858 def passed():
859 return {'passFail': 'pass'}
860
861 def failBecause(reason):
862 return {'passFail': 'fail', 'reason': reason}
863
864 # -----------------------------------------------------------------------------
865 # Generic command tests
866
867 # A generic command test is expected to run and exit successfully.
868 #
869 # The expected exit code can be changed via exit_code() as normal, and
870 # the expected stdout/stderr are stored in <testname>.stdout and
871 # <testname>.stderr. The output of the command can be ignored
872 # altogether by using run_command_ignore_output instead of
873 # run_command.
874
875 def run_command( name, way, cmd ):
876 return simple_run( name, '', cmd, '' )
877
878 # -----------------------------------------------------------------------------
879 # GHCi tests
880
881 def ghci_script( name, way, script ):
882 # filter out -fforce-recomp from compiler_always_flags, because we're
883 # actually testing the recompilation behaviour in the GHCi tests.
884 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
885 flags.append(getTestOpts().extra_hc_opts)
886 if getTestOpts().outputdir != None:
887 flags.extend(["-outputdir", getTestOpts().outputdir])
888
889 # We pass HC and HC_OPTS as environment variables, so that the
890 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
891 cmd = "HC='" + config.compiler + "' " + \
892 "HC_OPTS='" + join(flags,' ') + "' " + \
893 "'" + config.compiler + "'" + \
894 ' --interactive -v0 -ignore-dot-ghci ' + \
895 join(flags,' ')
896
897 getTestOpts().stdin = script
898 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
899
900 # -----------------------------------------------------------------------------
901 # Compile-only tests
902
903 def compile( name, way, extra_hc_opts ):
904 return do_compile( name, way, 0, '', [], extra_hc_opts )
905
906 def compile_fail( name, way, extra_hc_opts ):
907 return do_compile( name, way, 1, '', [], extra_hc_opts )
908
909 def multimod_compile( name, way, top_mod, extra_hc_opts ):
910 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
911
912 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
913 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
914
915 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
916 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
917
918 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
919 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
920
921 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
922 # print 'Compile only, extra args = ', extra_hc_opts
923 pretest_cleanup(name)
924
925 result = extras_build( way, extra_mods, extra_hc_opts )
926 if badResult(result):
927 return result
928 extra_hc_opts = result['hc_opts']
929
930 force = 0
931 if extra_mods:
932 force = 1
933 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
934
935 if badResult(result):
936 return result
937
938 # the actual stderr should always match the expected, regardless
939 # of whether we expected the compilation to fail or not (successful
940 # compilations may generate warnings).
941
942 if getTestOpts().with_namebase == None:
943 namebase = name
944 else:
945 namebase = getTestOpts().with_namebase
946
947 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
948 actual_stderr_file = qualify(name, 'comp.stderr')
949
950 if not compare_outputs('stderr', \
951 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
952 expected_stderr_file, actual_stderr_file):
953 return failBecause('stderr mismatch')
954
955 # no problems found, this test passed
956 return passed()
957
958 def compile_cmp_asm( name, way, extra_hc_opts ):
959 print 'Compile only, extra args = ', extra_hc_opts
960 pretest_cleanup(name)
961 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
962
963 if badResult(result):
964 return result
965
966 # the actual stderr should always match the expected, regardless
967 # of whether we expected the compilation to fail or not (successful
968 # compilations may generate warnings).
969
970 if getTestOpts().with_namebase == None:
971 namebase = name
972 else:
973 namebase = getTestOpts().with_namebase
974
975 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
976 actual_asm_file = qualify(name, 's')
977
978 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
979 expected_asm_file, actual_asm_file):
980 return failBecause('asm mismatch')
981
982 # no problems found, this test passed
983 return passed()
984
985 # -----------------------------------------------------------------------------
986 # Compile-and-run tests
987
988 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
989 # print 'Compile and run, extra args = ', extra_hc_opts
990 pretest_cleanup(name)
991
992 result = extras_build( way, extra_mods, extra_hc_opts )
993 if badResult(result):
994 return result
995 extra_hc_opts = result['hc_opts']
996
997 if way == 'ghci': # interpreted...
998 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
999 elif way == 'extcore' or way == 'optextcore' :
1000 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1001 else: # compiled...
1002 force = 0
1003 if extra_mods:
1004 force = 1
1005
1006 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1007 if badResult(result):
1008 return result
1009
1010 cmd = './' + name;
1011
1012 # we don't check the compiler's stderr for a compile-and-run test
1013 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1014
1015 def compile_and_run( name, way, extra_hc_opts ):
1016 return compile_and_run__( name, way, '', [], extra_hc_opts)
1017
1018 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1019 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1020
1021 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1022 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1023
1024 def stats( name, way, stats_file ):
1025 opts = getTestOpts()
1026 return checkStats(stats_file, opts.stats_range_fields)
1027
1028 # -----------------------------------------------------------------------------
1029 # Check -t stats info
1030
1031 def checkStats(stats_file, range_fields):
1032 result = passed()
1033 if len(range_fields) > 0:
1034 f = open(in_testdir(stats_file))
1035 contents = f.read()
1036 f.close()
1037
1038 for (field, (expected, dev)) in range_fields.items():
1039 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1040 if m == None:
1041 print 'Failed to find field: ', field
1042 result = failBecause('no such stats field')
1043 val = int(m.group(1))
1044
1045 lowerBound = trunc( expected * ((100 - float(dev))/100));
1046 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1047
1048 if val < lowerBound:
1049 print field, 'value is too low:'
1050 print '(If this is because you have improved GHC, please'
1051 print 'update the test so that GHC doesn\'t regress again)'
1052 result = failBecause('stat too good')
1053 if val > upperBound:
1054 print field, 'value is too high:'
1055 result = failBecause('stat not good enough')
1056
1057 if val < lowerBound or val > upperBound:
1058 valStr = str(val)
1059 valLen = len(valStr)
1060 expectedStr = str(expected)
1061 expectedLen = len(expectedStr)
1062 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1063 def display(descr, val, extra):
1064 print descr, string.rjust(str(val), length), extra
1065 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1066 display(' Lower bound ' + field + ':', lowerBound, '')
1067 display(' Upper bound ' + field + ':', upperBound, '')
1068 display(' Actual ' + field + ':', val, '')
1069
1070 return result
1071
1072 # -----------------------------------------------------------------------------
1073 # Build a single-module program
1074
1075 def extras_build( way, extra_mods, extra_hc_opts ):
1076 for modopts in extra_mods:
1077 mod, opts = modopts
1078 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1079 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1080 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1081 if badResult(result):
1082 return result
1083
1084 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1085
1086
1087 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1088 opts = getTestOpts()
1089 errname = add_suffix(name, 'comp.stderr')
1090 rm_no_fail( qualify(errname, '') )
1091
1092 if top_mod != '':
1093 srcname = top_mod
1094 rm_no_fail( qualify(name, '') )
1095 base, suf = os.path.splitext(top_mod)
1096 rm_no_fail( qualify(base, '') )
1097 rm_no_fail( qualify(base, 'exe') )
1098 elif addsuf:
1099 srcname = add_hs_lhs_suffix(name)
1100 rm_no_fail( qualify(name, '') )
1101 else:
1102 srcname = name
1103 rm_no_fail( qualify(name, 'o') )
1104
1105 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1106
1107 to_do = ''
1108 if top_mod != '':
1109 to_do = '--make '
1110 if link:
1111 to_do = to_do + '-o ' + name
1112 elif link:
1113 to_do = '-o ' + name
1114 elif opts.compile_to_hc:
1115 to_do = '-C'
1116 else:
1117 to_do = '-c' # just compile
1118
1119 stats_file = name + '.comp.stats'
1120 if len(opts.compiler_stats_range_fields) > 0:
1121 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1122
1123 # Required by GHC 7.3+, harmless for earlier versions:
1124 if (getTestOpts().c_src or
1125 getTestOpts().objc_src or
1126 getTestOpts().objcpp_src or
1127 getTestOpts().cmm_src):
1128 extra_hc_opts += ' -no-hs-main '
1129
1130 if getTestOpts().compile_cmd_prefix == '':
1131 cmd_prefix = ''
1132 else:
1133 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1134
1135 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1136 if noforce:
1137 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1138 if getTestOpts().outputdir != None:
1139 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1140
1141 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1142 + config.compiler + "' " \
1143 + join(comp_flags,' ') + ' ' \
1144 + to_do + ' ' + srcname + ' ' \
1145 + join(config.way_flags(name)[way],' ') + ' ' \
1146 + extra_hc_opts + ' ' \
1147 + opts.extra_hc_opts + ' ' \
1148 + '>' + errname + ' 2>&1'
1149
1150 result = runCmdFor(name, cmd)
1151
1152 if result != 0 and not should_fail:
1153 actual_stderr = qualify(name, 'comp.stderr')
1154 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1155 if_verbose_dump(1,actual_stderr)
1156
1157 # ToDo: if the sub-shell was killed by ^C, then exit
1158
1159 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1160
1161 if badResult(statsResult):
1162 return statsResult
1163
1164 if should_fail:
1165 if result == 0:
1166 return failBecause('exit code 0')
1167 else:
1168 if result != 0:
1169 return failBecause('exit code non-0')
1170
1171 return passed()
1172
1173 # -----------------------------------------------------------------------------
1174 # Run a program and check its output
1175 #
1176 # If testname.stdin exists, route input from that, else
1177 # from /dev/null. Route output to testname.run.stdout and
1178 # testname.run.stderr. Returns the exit code of the run.
1179
1180 def simple_run( name, way, prog, args ):
1181 opts = getTestOpts()
1182
1183 # figure out what to use for stdin
1184 if opts.stdin != '':
1185 use_stdin = opts.stdin
1186 else:
1187 stdin_file = add_suffix(name, 'stdin')
1188 if os.path.exists(in_testdir(stdin_file)):
1189 use_stdin = stdin_file
1190 else:
1191 use_stdin = '/dev/null'
1192
1193 run_stdout = add_suffix(name,'run.stdout')
1194 run_stderr = add_suffix(name,'run.stderr')
1195
1196 rm_no_fail(qualify(name,'run.stdout'))
1197 rm_no_fail(qualify(name,'run.stderr'))
1198 rm_no_fail(qualify(name, 'hp'))
1199 rm_no_fail(qualify(name,'ps'))
1200 rm_no_fail(qualify(name, 'prof'))
1201
1202 my_rts_flags = rts_flags(way)
1203
1204 stats_file = name + '.stats'
1205 if len(opts.stats_range_fields) > 0:
1206 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1207
1208 if opts.no_stdin:
1209 stdin_comes_from = ''
1210 else:
1211 stdin_comes_from = ' <' + use_stdin
1212
1213 if opts.combined_output:
1214 redirection = ' >' + run_stdout \
1215 + ' 2>&1'
1216 else:
1217 redirection = ' >' + run_stdout \
1218 + ' 2>' + run_stderr
1219
1220 cmd = prog + ' ' + args + ' ' \
1221 + my_rts_flags + ' ' \
1222 + stdin_comes_from \
1223 + redirection
1224
1225 if opts.cmd_wrapper != None:
1226 cmd = opts.cmd_wrapper(cmd);
1227
1228 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1229
1230 # run the command
1231 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1232
1233 exit_code = result >> 8
1234 signal = result & 0xff
1235
1236 # check the exit code
1237 if exit_code != opts.exit_code:
1238 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1239 dump_stdout(name)
1240 dump_stderr(name)
1241 return failBecause('bad exit code')
1242
1243 check_hp = my_rts_flags.find("-h") != -1
1244 check_prof = my_rts_flags.find("-p") != -1
1245
1246 if not opts.ignore_output:
1247 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1248 bad_stdout = not check_stdout_ok(name)
1249 if bad_stderr:
1250 return failBecause('bad stderr')
1251 if bad_stdout:
1252 return failBecause('bad stdout')
1253 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1254 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1255 return failBecause('bad heap profile')
1256 if check_prof and not check_prof_ok(name):
1257 return failBecause('bad profile')
1258
1259 return checkStats(stats_file, opts.stats_range_fields)
1260
1261 def rts_flags(way):
1262 if (way == ''):
1263 return ''
1264 else:
1265 args = config.way_rts_flags[way]
1266
1267 if args == []:
1268 return ''
1269 else:
1270 return '+RTS ' + join(args,' ') + ' -RTS'
1271
1272 # -----------------------------------------------------------------------------
1273 # Run a program in the interpreter and check its output
1274
1275 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1276 outname = add_suffix(name, 'interp.stdout')
1277 errname = add_suffix(name, 'interp.stderr')
1278 rm_no_fail(outname)
1279 rm_no_fail(errname)
1280 rm_no_fail(name)
1281
1282 if (top_mod == ''):
1283 srcname = add_hs_lhs_suffix(name)
1284 else:
1285 srcname = top_mod
1286
1287 scriptname = add_suffix(name, 'genscript')
1288 qscriptname = in_testdir(scriptname)
1289 rm_no_fail(qscriptname)
1290
1291 delimiter = '===== program output begins here\n'
1292
1293 script = open(qscriptname, 'w')
1294 if not compile_only:
1295 # set the prog name and command-line args to match the compiled
1296 # environment.
1297 script.write(':set prog ' + name + '\n')
1298 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1299 # Add marker lines to the stdout and stderr output files, so we
1300 # can separate GHCi's output from the program's.
1301 script.write(':! echo ' + delimiter)
1302 script.write(':! echo 1>&2 ' + delimiter)
1303 # Set stdout to be line-buffered to match the compiled environment.
1304 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1305 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1306 # in the event of an exception as for the compiled program.
1307 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1308 script.close()
1309
1310 # figure out what to use for stdin
1311 if getTestOpts().stdin != '':
1312 stdin_file = in_testdir(getTestOpts().stdin)
1313 else:
1314 stdin_file = qualify(name, 'stdin')
1315
1316 if os.path.exists(stdin_file):
1317 stdin = open(stdin_file, 'r')
1318 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1319
1320 script.close()
1321
1322 flags = copy.copy(getTestOpts().compiler_always_flags)
1323 if getTestOpts().outputdir != None:
1324 flags.extend(["-outputdir", getTestOpts().outputdir])
1325
1326 cmd = "'" + config.compiler + "' " \
1327 + join(flags,' ') + ' ' \
1328 + srcname + ' ' \
1329 + join(config.way_flags(name)[way],' ') + ' ' \
1330 + extra_hc_opts + ' ' \
1331 + getTestOpts().extra_hc_opts + ' ' \
1332 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1333
1334 if getTestOpts().cmd_wrapper != None:
1335 cmd = getTestOpts().cmd_wrapper(cmd);
1336
1337 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1338
1339 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1340
1341 exit_code = result >> 8
1342 signal = result & 0xff
1343
1344 # split the stdout into compilation/program output
1345 split_file(in_testdir(outname), delimiter,
1346 qualify(name, 'comp.stdout'),
1347 qualify(name, 'run.stdout'))
1348 split_file(in_testdir(errname), delimiter,
1349 qualify(name, 'comp.stderr'),
1350 qualify(name, 'run.stderr'))
1351
1352 # check the exit code
1353 if exit_code != getTestOpts().exit_code:
1354 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1355 dump_stdout(name)
1356 dump_stderr(name)
1357 return failBecause('bad exit code')
1358
1359 # ToDo: if the sub-shell was killed by ^C, then exit
1360
1361 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1362 check_stdout_ok(name)):
1363 return passed()
1364 else:
1365 return failBecause('bad stdout or stderr')
1366
1367
1368 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1369 infile = open(in_fn)
1370 out1 = open(out1_fn, 'w')
1371 out2 = open(out2_fn, 'w')
1372
1373 line = infile.readline()
1374 line = re.sub('\r', '', line) # ignore Windows EOL
1375 while (re.sub('^\s*','',line) != delimiter and line != ''):
1376 out1.write(line)
1377 line = infile.readline()
1378 line = re.sub('\r', '', line)
1379 out1.close()
1380
1381 line = infile.readline()
1382 while (line != ''):
1383 out2.write(line)
1384 line = infile.readline()
1385 out2.close()
1386
1387 # -----------------------------------------------------------------------------
1388 # Generate External Core for the given program, then compile the resulting Core
1389 # and compare its output to the expected output
1390
1391 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1392
1393 depsfilename = qualify(name, 'deps')
1394 errname = add_suffix(name, 'comp.stderr')
1395 qerrname = qualify(errname,'')
1396
1397 hcname = qualify(name, 'hc')
1398 oname = qualify(name, 'o')
1399
1400 rm_no_fail( qerrname )
1401 rm_no_fail( qualify(name, '') )
1402
1403 if (top_mod == ''):
1404 srcname = add_hs_lhs_suffix(name)
1405 else:
1406 srcname = top_mod
1407
1408 qcorefilename = qualify(name, 'hcr')
1409 corefilename = add_suffix(name, 'hcr')
1410 rm_no_fail(qcorefilename)
1411
1412 # Generate External Core
1413
1414 if (top_mod == ''):
1415 to_do = ' ' + srcname + ' '
1416 else:
1417 to_do = ' --make ' + top_mod + ' '
1418
1419 flags = copy.copy(getTestOpts().compiler_always_flags)
1420 if getTestOpts().outputdir != None:
1421 flags.extend(["-outputdir", getTestOpts().outputdir])
1422 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1423 + config.compiler + "' " \
1424 + join(flags,' ') + ' ' \
1425 + join(config.way_flags(name)[way],' ') + ' ' \
1426 + extra_hc_opts + ' ' \
1427 + getTestOpts().extra_hc_opts \
1428 + to_do \
1429 + '>' + errname + ' 2>&1'
1430 result = runCmdFor(name, cmd)
1431
1432 exit_code = result >> 8
1433
1434 if exit_code != 0:
1435 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1436 if_verbose_dump(1,qerrname)
1437 return failBecause('ext core exit code non-0')
1438
1439 # Compile the resulting files -- if there's more than one module, we need to read the output
1440 # of the previous compilation in order to find the dependencies
1441 if (top_mod == ''):
1442 to_compile = corefilename
1443 else:
1444 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1445 deps = open(depsfilename).read()
1446 deplist = string.replace(deps, '\n',' ');
1447 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1448 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1449
1450 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1451 if getTestOpts().outputdir != None:
1452 flags.extend(["-outputdir", getTestOpts().outputdir])
1453
1454 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1455 + config.compiler + "' " \
1456 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1457 + to_compile + ' ' \
1458 + extra_hc_opts + ' ' \
1459 + getTestOpts().extra_hc_opts + ' ' \
1460 + flags \
1461 + ' -fglasgow-exts -o ' + name \
1462 + '>' + errname + ' 2>&1'
1463
1464 result = runCmdFor(name, cmd)
1465 exit_code = result >> 8
1466
1467 if exit_code != 0:
1468 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1469 if_verbose_dump(1,qerrname)
1470 return failBecause('ext core exit code non-0')
1471
1472 # Clean up
1473 rm_no_fail ( oname )
1474 rm_no_fail ( hcname )
1475 rm_no_fail ( qcorefilename )
1476 rm_no_fail ( depsfilename )
1477
1478 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1479
1480 # -----------------------------------------------------------------------------
1481 # Utils
1482
1483 def check_stdout_ok( name ):
1484 if getTestOpts().with_namebase == None:
1485 namebase = name
1486 else:
1487 namebase = getTestOpts().with_namebase
1488
1489 actual_stdout_file = qualify(name, 'run.stdout')
1490 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1491
1492 def norm(str):
1493 if platform_specific:
1494 return str
1495 else:
1496 return normalise_output(str)
1497
1498 two_norm = two_normalisers(norm, getTestOpts().extra_normaliser)
1499
1500 check_stdout = getTestOpts().check_stdout
1501 if check_stdout:
1502 return check_stdout(actual_stdout_file, two_norm)
1503
1504 return compare_outputs('stdout', \
1505 two_norm, \
1506 expected_stdout_file, actual_stdout_file)
1507
1508 def dump_stdout( name ):
1509 print 'Stdout:'
1510 print read_no_crs(qualify(name, 'run.stdout'))
1511
1512 def check_stderr_ok( name ):
1513 if getTestOpts().with_namebase == None:
1514 namebase = name
1515 else:
1516 namebase = getTestOpts().with_namebase
1517
1518 actual_stderr_file = qualify(name, 'run.stderr')
1519 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1520
1521 def norm(str):
1522 if platform_specific:
1523 return str
1524 else:
1525 return normalise_errmsg(str)
1526
1527 return compare_outputs('stderr', \
1528 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1529 expected_stderr_file, actual_stderr_file)
1530
1531 def dump_stderr( name ):
1532 print "Stderr:"
1533 print read_no_crs(qualify(name, 'run.stderr'))
1534
1535 def read_no_crs(file):
1536 str = ''
1537 try:
1538 h = open(file)
1539 str = h.read()
1540 h.close
1541 except:
1542 # On Windows, if the program fails very early, it seems the
1543 # files stdout/stderr are redirected to may not get created
1544 pass
1545 return re.sub('\r', '', str)
1546
1547 def write_file(file, str):
1548 h = open(file, 'w')
1549 h.write(str)
1550 h.close
1551
1552 def check_hp_ok(name):
1553
1554 # do not qualify for hp2ps because we should be in the right directory
1555 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1556
1557 hp2psResult = runCmdExitCode(hp2psCmd)
1558
1559 actual_ps_file = qualify(name, 'ps')
1560
1561 if(hp2psResult == 0):
1562 if (os.path.exists(actual_ps_file)):
1563 if gs_working:
1564 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1565 if (gsResult == 0):
1566 return (True)
1567 else:
1568 print "hp2ps output for " + name + "is not valid PostScript"
1569 else: return (True) # assume postscript is valid without ghostscript
1570 else:
1571 print "hp2ps did not generate PostScript for " + name
1572 return (False)
1573 else:
1574 print "hp2ps error when processing heap profile for " + name
1575 return(False)
1576
1577 def check_prof_ok(name):
1578
1579 prof_file = qualify(name,'prof')
1580
1581 if not os.path.exists(prof_file):
1582 print prof_file + " does not exist"
1583 return(False)
1584
1585 if os.path.getsize(qualify(name,'prof')) == 0:
1586 print prof_file + " is empty"
1587 return(False)
1588
1589 if getTestOpts().with_namebase == None:
1590 namebase = name
1591 else:
1592 namebase = getTestOpts().with_namebase
1593
1594 (platform_specific, expected_prof_file) = \
1595 platform_wordsize_qualify(namebase, 'prof.sample')
1596
1597 # sample prof file is not required
1598 if not os.path.exists(expected_prof_file):
1599 return True
1600 else:
1601 return compare_outputs('prof', \
1602 two_normalisers(normalise_whitespace,normalise_prof), \
1603 expected_prof_file, prof_file)
1604
1605 # Compare expected output to actual output, and optionally accept the
1606 # new output. Returns true if output matched or was accepted, false
1607 # otherwise.
1608 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1609 if os.path.exists(expected_file):
1610 expected_raw = read_no_crs(expected_file)
1611 # print "norm:", normaliser(expected_raw)
1612 expected_str = normaliser(expected_raw)
1613 expected_file_for_diff = expected_file
1614 else:
1615 expected_str = ''
1616 expected_file_for_diff = '/dev/null'
1617
1618 actual_raw = read_no_crs(actual_file)
1619 actual_str = normaliser(actual_raw)
1620
1621 if expected_str == actual_str:
1622 return 1
1623 else:
1624 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1625
1626 if expected_file_for_diff == '/dev/null':
1627 expected_normalised_file = '/dev/null'
1628 else:
1629 expected_normalised_file = expected_file + ".normalised"
1630 write_file(expected_normalised_file, expected_str)
1631
1632 actual_normalised_file = actual_file + ".normalised"
1633 write_file(actual_normalised_file, actual_str)
1634
1635 # Ignore whitespace when diffing. We should only get to this
1636 # point if there are non-whitespace differences
1637 #
1638 # Note we are diffing the *actual* output, not the normalised
1639 # output. The normalised output may have whitespace squashed
1640 # (including newlines) so the diff would be hard to read.
1641 # This does mean that the diff might contain changes that
1642 # would be normalised away.
1643 if (config.verbose >= 1):
1644 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1645 ' ' + actual_file )
1646
1647 # If for some reason there were no non-whitespace differences,
1648 # then do a full diff
1649 if r == 0:
1650 r = os.system( 'diff -u ' + expected_file_for_diff + \
1651 ' ' + actual_file )
1652
1653 if config.accept:
1654 if_verbose(1, 'Accepting new output.')
1655 write_file(expected_file, actual_raw)
1656 return 1
1657 else:
1658 return 0
1659
1660
1661 def normalise_whitespace( str ):
1662 # Merge contiguous whitespace characters into a single space.
1663 str = re.sub('[ \t\n]+', ' ', str)
1664 return str
1665
1666 def normalise_errmsg( str ):
1667 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1668 # the colon is there because it appears in error messages; this
1669 # hacky solution is used in place of more sophisticated filename
1670 # mangling
1671 str = re.sub('([^\\s])\\.exe', '\\1', str)
1672 # normalise slashes, minimise Windows/Unix filename differences
1673 str = re.sub('\\\\', '/', str)
1674 # The inplace ghc's are called ghc-stage[123] to avoid filename
1675 # collisions, so we need to normalise that to just "ghc"
1676 str = re.sub('ghc-stage[123]', 'ghc', str)
1677 # We sometimes see the name of the integer-gmp package on stderr,
1678 # but this can change (either the implementation name or the
1679 # version number), so we canonicalise it here
1680 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1681 return str
1682
1683 # normalise a .prof file, so that we can reasonably compare it against
1684 # a sample. This doesn't compare any of the actual profiling data,
1685 # only the shape of the profile and the number of entries.
1686 def normalise_prof (str):
1687 # strip everything up to the line beginning "COST CENTRE"
1688 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1689
1690 # strip results for CAFs, these tend to change unpredictably
1691 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1692
1693 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1694 # sometimes under MAIN.
1695 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1696
1697 # We have somthing like this:
1698
1699 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1700 # k Main 204 1 0.0 0.0 0.0 0.0
1701 # foo Main 205 1 0.0 0.0 0.0 0.0
1702 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1703
1704 # then we remove all the specific profiling data, leaving only the
1705 # cost centre name, module, and entries, to end up with this:
1706
1707 # MAIN MAIN 0
1708 # k Main 1
1709 # foo Main 1
1710 # foo.bar Main 1
1711
1712 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1713 return str
1714
1715 def normalise_slashes_( str ):
1716 str = re.sub('\\\\', '/', str)
1717 return str
1718
1719 def normalise_exe_( str ):
1720 str = re.sub('\.exe', '', str)
1721 return str
1722
1723 def normalise_output( str ):
1724 # Remove a .exe extension (for Windows)
1725 # This can occur in error messages generated by the program.
1726 str = re.sub('([^\\s])\\.exe', '\\1', str)
1727 return str
1728
1729 def normalise_asm( str ):
1730 lines = str.split('\n')
1731 # Only keep instructions and labels not starting with a dot.
1732 metadata = re.compile('^[ \t]*\\..*$')
1733 out = []
1734 for line in lines:
1735 # Drop metadata directives (e.g. ".type")
1736 if not metadata.match(line):
1737 line = re.sub('@plt', '', line)
1738 instr = line.lstrip().split()
1739 # Drop empty lines.
1740 if not instr:
1741 continue
1742 # Drop operands, except for call instructions.
1743 elif instr[0] == 'call':
1744 out.append(instr[0] + ' ' + instr[1])
1745 else:
1746 out.append(instr[0])
1747 out = '\n'.join(out)
1748 return out
1749
1750 def if_verbose( n, str ):
1751 if config.verbose >= n:
1752 print str
1753
1754 def if_verbose_dump( n, f ):
1755 if config.verbose >= n:
1756 try:
1757 print open(f).read()
1758 except:
1759 print ''
1760
1761 def rawSystem(cmd_and_args):
1762 # We prefer subprocess.call to os.spawnv as the latter
1763 # seems to send its arguments through a shell or something
1764 # with the Windows (non-cygwin) python. An argument "a b c"
1765 # turns into three arguments ["a", "b", "c"].
1766
1767 # However, subprocess is new in python 2.4, so fall back to
1768 # using spawnv if we don't have it
1769
1770 if have_subprocess:
1771 return subprocess.call(cmd_and_args)
1772 else:
1773 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1774
1775 # Note that this doesn't handle the timeout itself; it is just used for
1776 # commands that have timeout handling built-in.
1777 def rawSystemWithTimeout(cmd_and_args):
1778 r = rawSystem(cmd_and_args)
1779 if r == 98:
1780 # The python timeout program uses 98 to signal that ^C was pressed
1781 stopNow()
1782 return r
1783
1784 # cmd is a complex command in Bourne-shell syntax
1785 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1786 # Hence it must ultimately be run by a Bourne shell
1787 #
1788 # Mostly it invokes the command wrapped in 'timeout' thus
1789 # timeout 300 'cd . && ...blah blah'
1790 # so it's timeout's job to invoke the Bourne shell
1791 #
1792 # But watch out for the case when there is no timeout program!
1793 # Then, when using the native Python, os.system will invoke the cmd shell
1794
1795 def runCmd( cmd ):
1796 if_verbose( 3, cmd )
1797 r = 0
1798 if config.os == 'mingw32':
1799 # On MinGW, we will always have timeout
1800 assert config.timeout_prog!=''
1801
1802 if config.timeout_prog != '':
1803 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1804 else:
1805 r = os.system(cmd)
1806 return r << 8
1807
1808 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1809 if_verbose( 3, cmd )
1810 r = 0
1811 if config.os == 'mingw32':
1812 # On MinGW, we will always have timeout
1813 assert config.timeout_prog!=''
1814 timeout = int(ceil(config.timeout * timeout_multiplier))
1815
1816 if config.timeout_prog != '':
1817 if config.check_files_written:
1818 fn = name + ".strace"
1819 r = rawSystemWithTimeout(
1820 ["strace", "-o", fn, "-fF",
1821 "-e", "creat,open,chdir,clone,vfork",
1822 config.timeout_prog, str(timeout), cmd])
1823 addTestFilesWritten(name, fn)
1824 rm_no_fail(fn)
1825 else:
1826 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1827 else:
1828 r = os.system(cmd)
1829 return r << 8
1830
1831 def runCmdExitCode( cmd ):
1832 return (runCmd(cmd) >> 8);
1833
1834
1835 # -----------------------------------------------------------------------------
1836 # checking for files being written to by multiple tests
1837
1838 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1839 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1840 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1841 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1842 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1843 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1844 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1845 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1846 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1847 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1848 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1849 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1850 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1851 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1852 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1853 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1854 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1855
1856 # Files that are read or written but shouldn't be:
1857 # * ghci_history shouldn't be read or written by tests
1858 # * things under package.conf.d shouldn't be written by tests
1859 bad_file_usages = {}
1860
1861 # Mapping from tests to the list of files that they write
1862 files_written = {}
1863
1864 # Mapping from tests to the list of files that they write but don't clean
1865 files_written_not_removed = {}
1866
1867 def add_bad_file_usage(name, file):
1868 try:
1869 if not file in bad_file_usages[name]:
1870 bad_file_usages[name].append(file)
1871 except:
1872 bad_file_usages[name] = [file]
1873
1874 def mkPath(curdir, path):
1875 # Given the current full directory is 'curdir', what is the full
1876 # path to 'path'?
1877 return os.path.realpath(os.path.join(curdir, path))
1878
1879 def addTestFilesWritten(name, fn):
1880 if config.use_threads:
1881 with t.lockFilesWritten:
1882 addTestFilesWrittenHelper(name, fn)
1883 else:
1884 addTestFilesWrittenHelper(name, fn)
1885
1886 def addTestFilesWrittenHelper(name, fn):
1887 started = False
1888 working_directories = {}
1889
1890 with open(fn, 'r') as f:
1891 for line in f:
1892 m_pid = re_strace_pid.match(line)
1893 if m_pid:
1894 pid = m_pid.group(1)
1895 content = m_pid.group(2)
1896 elif re_strace_unavailable.match(line):
1897 next
1898 else:
1899 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1900
1901 m_open = re_strace_open.match(content)
1902 m_chdir = re_strace_chdir.match(content)
1903 m_clone = re_strace_clone.match(content)
1904 m_vfork = re_strace_vfork.match(content)
1905
1906 if not started:
1907 working_directories[pid] = os.getcwd()
1908 started = True
1909
1910 if m_open:
1911 file = m_open.group(1)
1912 file = mkPath(working_directories[pid], file)
1913 if file.endswith("ghci_history"):
1914 add_bad_file_usage(name, file)
1915 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1916 flags = m_open.group(2).split('|')
1917 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1918 if re.match('package\.conf\.d', file):
1919 add_bad_file_usage(name, file)
1920 else:
1921 try:
1922 if not file in files_written[name]:
1923 files_written[name].append(file)
1924 except:
1925 files_written[name] = [file]
1926 elif 'O_RDONLY' in flags:
1927 pass
1928 else:
1929 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1930 elif m_chdir:
1931 # We optimistically assume that unfinished chdir's are going to succeed
1932 dir = m_chdir.group(1)
1933 working_directories[pid] = mkPath(working_directories[pid], dir)
1934 elif m_clone:
1935 working_directories[m_clone.group(2)] = working_directories[pid]
1936 elif m_vfork:
1937 working_directories[m_vfork.group(2)] = working_directories[pid]
1938 elif re_strace_open_resumed.match(content):
1939 pass
1940 elif re_strace_chdir_resumed.match(content):
1941 pass
1942 elif re_strace_vfork_unfinished.match(content):
1943 pass
1944 elif re_strace_clone_unfinished.match(content):
1945 pass
1946 elif re_strace_ignore_sigchild.match(content):
1947 pass
1948 elif re_strace_ignore_sigvtalarm.match(content):
1949 pass
1950 elif re_strace_ignore_sigint.match(content):
1951 pass
1952 elif re_strace_ignore_sigfpe.match(content):
1953 pass
1954 elif re_strace_ignore_sigsegv.match(content):
1955 pass
1956 elif re_strace_ignore_sigpipe.match(content):
1957 pass
1958 else:
1959 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1960
1961 def checkForFilesWrittenProblems(file):
1962 foundProblem = False
1963
1964 files_written_inverted = {}
1965 for t in files_written.keys():
1966 for f in files_written[t]:
1967 try:
1968 files_written_inverted[f].append(t)
1969 except:
1970 files_written_inverted[f] = [t]
1971
1972 for f in files_written_inverted.keys():
1973 if len(files_written_inverted[f]) > 1:
1974 if not foundProblem:
1975 foundProblem = True
1976 file.write("\n")
1977 file.write("\nSome files are written by multiple tests:\n")
1978 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1979 if foundProblem:
1980 file.write("\n")
1981
1982 # -----
1983
1984 if len(files_written_not_removed) > 0:
1985 file.write("\n")
1986 file.write("\nSome files written but not removed:\n")
1987 tests = files_written_not_removed.keys()
1988 tests.sort()
1989 for t in tests:
1990 for f in files_written_not_removed[t]:
1991 file.write(" " + t + ": " + f + "\n")
1992 file.write("\n")
1993
1994 # -----
1995
1996 if len(bad_file_usages) > 0:
1997 file.write("\n")
1998 file.write("\nSome bad file usages:\n")
1999 tests = bad_file_usages.keys()
2000 tests.sort()
2001 for t in tests:
2002 for f in bad_file_usages[t]:
2003 file.write(" " + t + ": " + f + "\n")
2004 file.write("\n")
2005
2006 # -----------------------------------------------------------------------------
2007 # checking if ghostscript is available for checking the output of hp2ps
2008
2009 def genGSCmd(psfile):
2010 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2011
2012 def gsNotWorking():
2013 global gs_working
2014 print "GhostScript not available for hp2ps tests"
2015
2016 global gs_working
2017 gs_working = 0
2018 if config.have_profiling:
2019 if config.gs != '':
2020 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2021 if resultGood == 0:
2022 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2023 if resultBad != 0:
2024 print "GhostScript available for hp2ps tests"
2025 gs_working = 1;
2026 else:
2027 gsNotWorking();
2028 else:
2029 gsNotWorking();
2030 else:
2031 gsNotWorking();
2032
2033 def rm_no_fail( file ):
2034 try:
2035 os.remove( file )
2036 finally:
2037 return
2038
2039 def add_suffix( name, suffix ):
2040 if suffix == '':
2041 return name
2042 else:
2043 return name + '.' + suffix
2044
2045 def add_hs_lhs_suffix(name):
2046 if getTestOpts().c_src:
2047 return add_suffix(name, 'c')
2048 elif getTestOpts().cmm_src:
2049 return add_suffix(name, 'cmm')
2050 elif getTestOpts().objc_src:
2051 return add_suffix(name, 'm')
2052 elif getTestOpts().objcpp_src:
2053 return add_suffix(name, 'mm')
2054 elif getTestOpts().literate:
2055 return add_suffix(name, 'lhs')
2056 else:
2057 return add_suffix(name, 'hs')
2058
2059 def replace_suffix( name, suffix ):
2060 base, suf = os.path.splitext(name)
2061 return base + '.' + suffix
2062
2063 def in_testdir( name ):
2064 return (getTestOpts().testdir + '/' + name)
2065
2066 def qualify( name, suff ):
2067 return in_testdir(add_suffix(name, suff))
2068
2069
2070 # Finding the sample output. The filename is of the form
2071 #
2072 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2073 #
2074 # and we pick the most specific version available. The <version> is
2075 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2076 # more fine-grained control use if_compiler_lt().
2077 #
2078 def platform_wordsize_qualify( name, suff ):
2079
2080 basepath = qualify(name, suff)
2081
2082 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2083 for (platformSpecific, plat) in [(1, '-' + config.platform),
2084 (1, '-' + config.os),
2085 (0, '')]
2086 for ws in ['-ws-' + config.wordsize, '']
2087 for comp in ['-' + config.compiler_type, '']
2088 for vers in ['-' + config.compiler_maj_version, '']]
2089
2090 dir = glob.glob(basepath + '*')
2091 dir = map (lambda d: normalise_slashes_(d), dir)
2092
2093 for (platformSpecific, f) in paths:
2094 if f in dir:
2095 return (platformSpecific,f)
2096
2097 return (0, basepath)
2098
2099 # Clean up prior to the test, so that we can't spuriously conclude
2100 # that it passed on the basis of old run outputs.
2101 def pretest_cleanup(name):
2102 if getTestOpts().outputdir != None:
2103 odir = in_testdir(getTestOpts().outputdir)
2104 try:
2105 shutil.rmtree(odir)
2106 except:
2107 pass
2108 os.mkdir(odir)
2109
2110 rm_no_fail(qualify(name,'interp.stderr'))
2111 rm_no_fail(qualify(name,'interp.stdout'))
2112 rm_no_fail(qualify(name,'comp.stderr'))
2113 rm_no_fail(qualify(name,'comp.stdout'))
2114 rm_no_fail(qualify(name,'run.stderr'))
2115 rm_no_fail(qualify(name,'run.stdout'))
2116 rm_no_fail(qualify(name,'tix'))
2117 rm_no_fail(qualify(name,'exe.tix'))
2118 # simple_build zaps the following:
2119 # rm_nofail(qualify("o"))
2120 # rm_nofail(qualify(""))
2121 # not interested in the return code
2122
2123 # -----------------------------------------------------------------------------
2124 # Return a list of all the files ending in '.T' below the directory dir.
2125
2126 def findTFiles(roots):
2127 return concat(map(findTFiles_,roots))
2128
2129 def findTFiles_(path):
2130 if os.path.isdir(path):
2131 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2132 return findTFiles(paths)
2133 elif path[-2:] == '.T':
2134 return [path]
2135 else:
2136 return []
2137
2138 # -----------------------------------------------------------------------------
2139 # Output a test summary to the specified file object
2140
2141 def summary(t, file):
2142
2143 file.write('\n')
2144 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2145 file.write('OVERALL SUMMARY for test run started at '
2146 + time.strftime("%c %Z", t.start_time) + '\n'
2147 + string.rjust(str(datetime.timedelta(seconds=
2148 round(time.time() - time.mktime(t.start_time)))), 8)
2149 + ' spent to go through\n'
2150 + string.rjust(`t.total_tests`, 8)
2151 + ' total tests, which gave rise to\n'
2152 + string.rjust(`t.total_test_cases`, 8)
2153 + ' test cases, of which\n'
2154 + string.rjust(`t.n_tests_skipped`, 8)
2155 + ' were skipped\n'
2156 + '\n'
2157 + string.rjust(`t.n_missing_libs`, 8)
2158 + ' had missing libraries\n'
2159 + string.rjust(`t.n_expected_passes`, 8)
2160 + ' expected passes\n'
2161 + string.rjust(`t.n_expected_failures`, 8)
2162 + ' expected failures\n'
2163 + '\n'
2164 + string.rjust(`t.n_framework_failures`, 8)
2165 + ' caused framework failures\n'
2166 + string.rjust(`t.n_unexpected_passes`, 8)
2167 + ' unexpected passes\n'
2168 + string.rjust(`t.n_unexpected_failures`, 8)
2169 + ' unexpected failures\n'
2170 + '\n')
2171
2172 if t.n_unexpected_passes > 0:
2173 file.write('Unexpected passes:\n')
2174 printPassingTestInfosSummary(file, t.unexpected_passes)
2175
2176 if t.n_unexpected_failures > 0:
2177 file.write('Unexpected failures:\n')
2178 printFailingTestInfosSummary(file, t.unexpected_failures)
2179
2180 if config.check_files_written:
2181 checkForFilesWrittenProblems(file)
2182
2183 if stopping():
2184 file.write('WARNING: Testsuite run was terminated early\n')
2185
2186 def printUnexpectedTests(file, testInfoss):
2187 unexpected = []
2188 for testInfos in testInfoss:
2189 directories = testInfos.keys()
2190 for directory in directories:
2191 tests = testInfos[directory].keys()
2192 unexpected += tests
2193 if unexpected != []:
2194 file.write('Unexpected results from:\n')
2195 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2196 file.write('\n')
2197
2198 def printPassingTestInfosSummary(file, testInfos):
2199 directories = testInfos.keys()
2200 directories.sort()
2201 maxDirLen = max(map ((lambda x : len(x)), directories))
2202 for directory in directories:
2203 tests = testInfos[directory].keys()
2204 tests.sort()
2205 for test in tests:
2206 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2207 ' (' + join(testInfos[directory][test],',') + ')\n')
2208 file.write('\n')
2209
2210 def printFailingTestInfosSummary(file, testInfos):
2211 directories = testInfos.keys()
2212 directories.sort()
2213 maxDirLen = max(map ((lambda x : len(x)), directories))
2214 for directory in directories:
2215 tests = testInfos[directory].keys()
2216 tests.sort()
2217 for test in tests:
2218 reasons = testInfos[directory][test].keys()
2219 for reason in reasons:
2220 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2221 ' [' + reason + ']' + \
2222 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2223 file.write('\n')
2224
2225 def getStdout(cmd):
2226 if have_subprocess:
2227 p = subprocess.Popen(cmd,
2228 stdout=subprocess.PIPE,
2229 stderr=subprocess.PIPE)
2230 (stdout, stderr) = p.communicate()
2231 r = p.wait()
2232 if r != 0:
2233 raise Exception("Command failed: " + str(cmd))
2234 if stderr != '':
2235 raise Exception("stderr from command: " + str(cmd))
2236 return stdout
2237 else:
2238 raise Exception("Need subprocess to get stdout, but don't have it")