Fold testsuite.git into ghc.git (re #8545)
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 import types
20 from math import ceil, trunc
21
22 have_subprocess = False
23 try:
24 import subprocess
25 have_subprocess = True
26 except:
27 print "Warning: subprocess not found, will fall back to spawnv"
28
29 from string import join
30 from testglobals import *
31 from testutil import *
32
33 if config.use_threads:
34 import threading
35 import thread
36
37 global wantToStop
38 wantToStop = False
39 def stopNow():
40 global wantToStop
41 wantToStop = True
42 def stopping():
43 return wantToStop
44
45 # Options valid for the current test only (these get reset to
46 # testdir_testopts after each test).
47
48 global testopts_local
49 if config.use_threads:
50 testopts_local = threading.local()
51 else:
52 class TestOpts_Local:
53 pass
54 testopts_local = TestOpts_Local()
55
56 def getTestOpts():
57 return testopts_local.x
58
59 def setLocalTestOpts(opts):
60 global testopts_local
61 testopts_local.x=opts
62
63 def isStatsTest():
64 opts = getTestOpts()
65 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
66
67
68 # This can be called at the top of a file of tests, to set default test options
69 # for the following tests.
70 def setTestOpts( f ):
71 global thisdir_settings
72 thisdir_settings = [thisdir_settings, f]
73
74 # -----------------------------------------------------------------------------
75 # Canned setup functions for common cases. eg. for a test you might say
76 #
77 # test('test001', normal, compile, [''])
78 #
79 # to run it without any options, but change it to
80 #
81 # test('test001', expect_fail, compile, [''])
82 #
83 # to expect failure for this test.
84
85 def normal( name, opts ):
86 return;
87
88 def skip( name, opts ):
89 opts.skip = 1
90
91 def expect_fail( name, opts ):
92 opts.expect = 'fail';
93
94 def reqlib( lib ):
95 return lambda name, opts, l=lib: _reqlib (name, opts, l )
96
97 # Cache the results of looking to see if we have a library or not.
98 # This makes quite a difference, especially on Windows.
99 have_lib = {}
100
101 def _reqlib( name, opts, lib ):
102 if have_lib.has_key(lib):
103 got_it = have_lib[lib]
104 else:
105 if have_subprocess:
106 # By preference we use subprocess, as the alternative uses
107 # /dev/null which mingw doesn't have.
108 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 stdout=subprocess.PIPE,
110 stderr=subprocess.PIPE)
111 # read from stdout and stderr to avoid blocking due to
112 # buffers filling
113 p.communicate()
114 r = p.wait()
115 else:
116 r = os.system(config.ghc_pkg + ' describe ' + lib
117 + ' > /dev/null 2> /dev/null')
118 got_it = r == 0
119 have_lib[lib] = got_it
120
121 if not got_it:
122 opts.expect = 'missing-lib'
123
124 def req_profiling( name, opts ):
125 if not config.have_profiling:
126 opts.expect = 'fail'
127
128 def req_shared_libs( name, opts ):
129 if not config.have_shared_libs:
130 opts.expect = 'fail'
131
132 def req_interp( name, opts ):
133 if not config.have_interp:
134 opts.expect = 'fail'
135
136 def req_smp( name, opts ):
137 if not config.have_smp:
138 opts.expect = 'fail'
139
140 def ignore_output( name, opts ):
141 opts.ignore_output = 1
142
143 def no_stdin( name, opts ):
144 opts.no_stdin = 1
145
146 def combined_output( name, opts ):
147 opts.combined_output = True
148
149 # -----
150
151 def expect_fail_for( ways ):
152 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153
154 def _expect_fail_for( name, opts, ways ):
155 opts.expect_fail_for = ways
156
157 def expect_broken( bug ):
158 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
159
160 def _expect_broken( name, opts, bug ):
161 record_broken(name, opts, bug)
162 opts.expect = 'fail';
163
164 def expect_broken_for( bug, ways ):
165 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
166
167 def _expect_broken_for( name, opts, bug, ways ):
168 record_broken(name, opts, bug)
169 opts.expect_fail_for = ways
170
171 def record_broken(name, opts, bug):
172 global brokens
173 me = (bug, opts.testdir, name)
174 if not me in brokens:
175 brokens.append(me)
176
177 # -----
178
179 def omit_ways( ways ):
180 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181
182 def _omit_ways( name, opts, ways ):
183 opts.omit_ways = ways
184
185 # -----
186
187 def only_ways( ways ):
188 return lambda name, opts, w=ways: _only_ways( name, opts, w )
189
190 def _only_ways( name, opts, ways ):
191 opts.only_ways = ways
192
193 # -----
194
195 def extra_ways( ways ):
196 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197
198 def _extra_ways( name, opts, ways ):
199 opts.extra_ways = ways
200
201 # -----
202
203 def omit_compiler_types( compiler_types ):
204 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
205
206 def _omit_compiler_types( name, opts, compiler_types ):
207 if config.compiler_type in compiler_types:
208 opts.skip = 1
209
210 # -----
211
212 def only_compiler_types( compiler_types ):
213 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
214
215 def _only_compiler_types( name, opts, compiler_types ):
216 if config.compiler_type not in compiler_types:
217 opts.skip = 1
218
219 # -----
220
221 def set_stdin( file ):
222 return lambda name, opts, f=file: _set_stdin(name, opts, f);
223
224 def _set_stdin( name, opts, f ):
225 opts.stdin = f
226
227 # -----
228
229 def exit_code( val ):
230 return lambda name, opts, v=val: _exit_code(name, opts, v);
231
232 def _exit_code( name, opts, v ):
233 opts.exit_code = v
234
235 # -----
236
237 def timeout_multiplier( val ):
238 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239
240 def _timeout_multiplier( name, opts, v ):
241 opts.timeout_multiplier = v
242
243 # -----
244
245 def extra_run_opts( val ):
246 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247
248 def _extra_run_opts( name, opts, v ):
249 opts.extra_run_opts = v
250
251 # -----
252
253 def extra_hc_opts( val ):
254 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
255
256 def _extra_hc_opts( name, opts, v ):
257 opts.extra_hc_opts = v
258
259 # -----
260
261 def extra_clean( files ):
262 return lambda name, opts, v=files: _extra_clean(name, opts, v);
263
264 def _extra_clean( name, opts, v ):
265 opts.clean_files = v
266
267 # -----
268
269 def stats_num_field( field, expecteds ):
270 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
271
272 def _stats_num_field( name, opts, field, expecteds ):
273 if field in opts.stats_range_fields:
274 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
275
276 if type(expecteds) is types.ListType:
277 for (b, expected, dev) in expecteds:
278 if b:
279 opts.stats_range_fields[field] = (expected, dev)
280 return
281 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282
283 else:
284 (expected, dev) = expecteds
285 opts.stats_range_fields[field] = (expected, dev)
286
287 def compiler_stats_num_field( field, expecteds ):
288 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289
290 def _compiler_stats_num_field( name, opts, field, expecteds ):
291 if field in opts.compiler_stats_range_fields:
292 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
293
294 # Compiler performance numbers change when debugging is on, making the results
295 # useless and confusing. Therefore, skip if debugging is on.
296 if compiler_debugged():
297 skip(name, opts)
298
299 for (b, expected, dev) in expecteds:
300 if b:
301 opts.compiler_stats_range_fields[field] = (expected, dev)
302 return
303
304 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
305
306 # -----
307
308 def when(b, f):
309 # When list_brokens is on, we want to see all expect_broken calls,
310 # so we always do f
311 if b or config.list_broken:
312 return f
313 else:
314 return normal
315
316 def unless(b, f):
317 return when(not b, f)
318
319 def doing_ghci():
320 return 'ghci' in config.run_ways
321
322 def ghci_dynamic( ):
323 return config.ghc_dynamic
324
325 def fast():
326 return config.fast
327
328 def platform( plat ):
329 return config.platform == plat
330
331 def opsys( os ):
332 return config.os == os
333
334 def arch( arch ):
335 return config.arch == arch
336
337 def wordsize( ws ):
338 return config.wordsize == str(ws)
339
340 def msys( ):
341 return config.msys
342
343 def cygwin( ):
344 return config.cygwin
345
346 def have_vanilla( ):
347 return config.have_vanilla
348
349 def have_dynamic( ):
350 return config.have_dynamic
351
352 def have_profiling( ):
353 return config.have_profiling
354
355 def in_tree_compiler( ):
356 return config.in_tree_compiler
357
358 def compiler_type( compiler ):
359 return config.compiler_type == compiler
360
361 def compiler_lt( compiler, version ):
362 return config.compiler_type == compiler and \
363 version_lt(config.compiler_version, version)
364
365 def compiler_le( compiler, version ):
366 return config.compiler_type == compiler and \
367 version_le(config.compiler_version, version)
368
369 def compiler_gt( compiler, version ):
370 return config.compiler_type == compiler and \
371 version_gt(config.compiler_version, version)
372
373 def compiler_ge( compiler, version ):
374 return config.compiler_type == compiler and \
375 version_ge(config.compiler_version, version)
376
377 def unregisterised( ):
378 return config.unregisterised
379
380 def compiler_profiled( ):
381 return config.compiler_profiled
382
383 def compiler_debugged( ):
384 return config.compiler_debugged
385
386 def tag( t ):
387 return t in config.compiler_tags
388
389 # ---
390
391 def namebase( nb ):
392 return lambda opts, nb=nb: _namebase(opts, nb)
393
394 def _namebase( opts, nb ):
395 opts.with_namebase = nb
396
397 # ---
398
399 def high_memory_usage(name, opts):
400 opts.alone = True
401
402 # If a test is for a multi-CPU race, then running the test alone
403 # increases the chance that we'll actually see it.
404 def multi_cpu_race(name, opts):
405 opts.alone = True
406
407 # ---
408 def literate( name, opts ):
409 opts.literate = 1;
410
411 def c_src( name, opts ):
412 opts.c_src = 1;
413
414 def objc_src( name, opts ):
415 opts.objc_src = 1;
416
417 def objcpp_src( name, opts ):
418 opts.objcpp_src = 1;
419
420 def cmm_src( name, opts ):
421 opts.cmm_src = 1;
422
423 def outputdir( odir ):
424 return lambda name, opts, d=odir: _outputdir(name, opts, d)
425
426 def _outputdir( name, opts, odir ):
427 opts.outputdir = odir;
428
429 # ----
430
431 def pre_cmd( cmd ):
432 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
433
434 def _pre_cmd( name, opts, cmd ):
435 opts.pre_cmd = cmd
436
437 # ----
438
439 def clean_cmd( cmd ):
440 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
441
442 def _clean_cmd( name, opts, cmd ):
443 opts.clean_cmd = cmd
444
445 # ----
446
447 def cmd_prefix( prefix ):
448 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
449
450 def _cmd_prefix( name, opts, prefix ):
451 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
452
453 # ----
454
455 def cmd_wrapper( fun ):
456 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
457
458 def _cmd_wrapper( name, opts, fun ):
459 opts.cmd_wrapper = fun
460
461 # ----
462
463 def compile_cmd_prefix( prefix ):
464 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
465
466 def _compile_cmd_prefix( name, opts, prefix ):
467 opts.compile_cmd_prefix = prefix
468
469 # ----
470
471 def normalise_slashes( name, opts ):
472 opts.extra_normaliser = normalise_slashes_
473
474 def normalise_exe( name, opts ):
475 opts.extra_normaliser = normalise_exe_
476
477 def normalise_fun( fun ):
478 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
479
480 def _normalise_fun( name, opts, f ):
481 opts.extra_normaliser = f
482
483 def normalise_errmsg_fun( fun ):
484 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
485
486 def _normalise_errmsg_fun( name, opts, f ):
487 opts.extra_errmsg_normaliser = f
488
489 def two_normalisers(f, g):
490 return lambda x, f=f, g=g: f(g(x))
491
492 # ----
493 # Function for composing two opt-fns together
494
495 def executeSetups(fs, name, opts):
496 if type(fs) is types.ListType:
497 # If we have a list of setups, then execute each one
498 map (lambda f : executeSetups(f, name, opts), fs)
499 else:
500 # fs is a single function, so just apply it
501 fs(name, opts)
502
503 # -----------------------------------------------------------------------------
504 # The current directory of tests
505
506 def newTestDir( dir ):
507 global thisdir_settings
508 # reset the options for this test directory
509 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
510
511 def _newTestDir( name, opts, dir ):
512 opts.testdir = dir
513 opts.compiler_always_flags = config.compiler_always_flags
514
515 # -----------------------------------------------------------------------------
516 # Actually doing tests
517
518 parallelTests = []
519 aloneTests = []
520 allTestNames = set([])
521
522 def runTest (opts, name, func, args):
523 ok = 0
524
525 if config.use_threads:
526 t.thread_pool.acquire()
527 try:
528 while config.threads<(t.running_threads+1):
529 t.thread_pool.wait()
530 t.running_threads = t.running_threads+1
531 ok=1
532 t.thread_pool.release()
533 thread.start_new_thread(test_common_thread, (name, opts, func, args))
534 except:
535 if not ok:
536 t.thread_pool.release()
537 else:
538 test_common_work (name, opts, func, args)
539
540 # name :: String
541 # setup :: TestOpts -> IO ()
542 def test (name, setup, func, args):
543 global aloneTests
544 global parallelTests
545 global allTestNames
546 global thisdir_settings
547 if name in allTestNames:
548 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
549 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
550 framework_fail(name, 'bad_name', 'This test has an invalid name')
551
552 # Make a deep copy of the default_testopts, as we need our own copy
553 # of any dictionaries etc inside it. Otherwise, if one test modifies
554 # them, all tests will see the modified version!
555 myTestOpts = copy.deepcopy(default_testopts)
556
557 executeSetups([thisdir_settings, setup], name, myTestOpts)
558
559 thisTest = lambda : runTest(myTestOpts, name, func, args)
560 if myTestOpts.alone:
561 aloneTests.append(thisTest)
562 else:
563 parallelTests.append(thisTest)
564 allTestNames.add(name)
565
566 if config.use_threads:
567 def test_common_thread(name, opts, func, args):
568 t.lock.acquire()
569 try:
570 test_common_work(name,opts,func,args)
571 finally:
572 t.lock.release()
573 t.thread_pool.acquire()
574 t.running_threads = t.running_threads - 1
575 t.thread_pool.notify()
576 t.thread_pool.release()
577
578 def get_package_cache_timestamp():
579 if config.package_conf_cache_file == '':
580 return 0.0
581 else:
582 try:
583 return os.stat(config.package_conf_cache_file).st_mtime
584 except:
585 return 0.0
586
587
588 def test_common_work (name, opts, func, args):
589 try:
590 t.total_tests = t.total_tests+1
591 setLocalTestOpts(opts)
592
593 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
594
595 # All the ways we might run this test
596 if func == compile or func == multimod_compile:
597 all_ways = config.compile_ways
598 elif func == compile_and_run or func == multimod_compile_and_run:
599 all_ways = config.run_ways
600 elif func == ghci_script:
601 if 'ghci' in config.run_ways:
602 all_ways = ['ghci']
603 else:
604 all_ways = []
605 else:
606 all_ways = ['normal']
607
608 # A test itself can request extra ways by setting opts.extra_ways
609 all_ways = all_ways + filter(lambda way: way not in all_ways,
610 opts.extra_ways)
611
612 t.total_test_cases = t.total_test_cases + len(all_ways)
613
614 ok_way = lambda way: \
615 not getTestOpts().skip \
616 and (config.only == [] or name in config.only) \
617 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
618 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
619 and (not (config.skip_perf_tests and isStatsTest())) \
620 and way not in getTestOpts().omit_ways
621
622 # Which ways we are asked to skip
623 do_ways = filter (ok_way,all_ways)
624
625 # In fast mode, we skip all but one way
626 if config.fast and len(do_ways) > 0:
627 do_ways = [do_ways[0]]
628
629 if not config.clean_only:
630 # Run the required tests...
631 for way in do_ways:
632 if stopping():
633 break
634 do_test (name, way, func, args)
635
636 for way in all_ways:
637 if way not in do_ways:
638 skiptest (name,way)
639
640 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
641 pretest_cleanup(name)
642 clean(map (lambda suff: name + suff,
643 ['', '.exe', '.exe.manifest', '.genscript',
644 '.stderr.normalised', '.stdout.normalised',
645 '.run.stderr.normalised', '.run.stdout.normalised',
646 '.comp.stderr.normalised', '.comp.stdout.normalised',
647 '.interp.stderr.normalised', '.interp.stdout.normalised',
648 '.stats', '.comp.stats',
649 '.hi', '.o', '.prof', '.exe.prof', '.hc',
650 '_stub.h', '_stub.c', '_stub.o',
651 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
652
653 if func == multi_compile or func == multi_compile_fail:
654 extra_mods = args[1]
655 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
656 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
657
658 clean(getTestOpts().clean_files)
659
660 if getTestOpts().outputdir != None:
661 odir = in_testdir(getTestOpts().outputdir)
662 try:
663 shutil.rmtree(odir)
664 except:
665 pass
666
667 try:
668 shutil.rmtree(in_testdir('.hpc.' + name))
669 except:
670 pass
671
672 try:
673 cleanCmd = getTestOpts().clean_cmd
674 if cleanCmd != None:
675 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
676 if result != 0:
677 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
678 except e:
679 framework_fail(name, 'cleaning', 'clean-command exception')
680
681 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
682
683 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
684 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
685
686 try:
687 for f in files_written[name]:
688 if os.path.exists(f):
689 try:
690 if not f in files_written_not_removed[name]:
691 files_written_not_removed[name].append(f)
692 except:
693 files_written_not_removed[name] = [f]
694 except:
695 pass
696 except Exception, e:
697 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
698
699 def clean(strs):
700 for str in strs:
701 for name in glob.glob(in_testdir(str)):
702 clean_full_path(name)
703
704 def clean_full_path(name):
705 try:
706 # Remove files...
707 os.remove(name)
708 except OSError, e1:
709 try:
710 # ... and empty directories
711 os.rmdir(name)
712 except OSError, e2:
713 # We don't want to fail here, but we do want to know
714 # what went wrong, so print out the exceptions.
715 # ENOENT isn't a problem, though, as we clean files
716 # that don't necessarily exist.
717 if e1.errno != errno.ENOENT:
718 print e1
719 if e2.errno != errno.ENOENT:
720 print e2
721
722 def do_test(name, way, func, args):
723 full_name = name + '(' + way + ')'
724
725 try:
726 if_verbose(2, "=====> %s %d of %d %s " % \
727 (full_name, t.total_tests, len(allTestNames), \
728 [t.n_unexpected_passes, \
729 t.n_unexpected_failures, \
730 t.n_framework_failures]))
731
732 if config.use_threads:
733 t.lock.release()
734
735 try:
736 preCmd = getTestOpts().pre_cmd
737 if preCmd != None:
738 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
739 if result != 0:
740 framework_fail(name, way, 'pre-command failed: ' + str(result))
741 except e:
742 framework_fail(name, way, 'pre-command exception')
743
744 try:
745 result = apply(func, [name,way] + args)
746 finally:
747 if config.use_threads:
748 t.lock.acquire()
749
750 if getTestOpts().expect != 'pass' and \
751 getTestOpts().expect != 'fail' and \
752 getTestOpts().expect != 'missing-lib':
753 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
754
755 try:
756 passFail = result['passFail']
757 except:
758 passFail = 'No passFail found'
759
760 if passFail == 'pass':
761 if getTestOpts().expect == 'pass' \
762 and way not in getTestOpts().expect_fail_for:
763 t.n_expected_passes = t.n_expected_passes + 1
764 if name in t.expected_passes:
765 t.expected_passes[name].append(way)
766 else:
767 t.expected_passes[name] = [way]
768 else:
769 if_verbose(1, '*** unexpected pass for %s' % full_name)
770 t.n_unexpected_passes = t.n_unexpected_passes + 1
771 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
772 elif passFail == 'fail':
773 if getTestOpts().expect == 'pass' \
774 and way not in getTestOpts().expect_fail_for:
775 if_verbose(1, '*** unexpected failure for %s' % full_name)
776 t.n_unexpected_failures = t.n_unexpected_failures + 1
777 reason = result['reason']
778 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
779 else:
780 if getTestOpts().expect == 'missing-lib':
781 t.n_missing_libs = t.n_missing_libs + 1
782 if name in t.missing_libs:
783 t.missing_libs[name].append(way)
784 else:
785 t.missing_libs[name] = [way]
786 else:
787 t.n_expected_failures = t.n_expected_failures + 1
788 if name in t.expected_failures:
789 t.expected_failures[name].append(way)
790 else:
791 t.expected_failures[name] = [way]
792 else:
793 framework_fail(name, way, 'bad result ' + passFail)
794 except KeyboardInterrupt:
795 stopNow()
796 except:
797 framework_fail(name, way, 'do_test exception')
798 traceback.print_exc()
799
800 def addPassingTestInfo (testInfos, directory, name, way):
801 directory = re.sub('^\\.[/\\\\]', '', directory)
802
803 if not directory in testInfos:
804 testInfos[directory] = {}
805
806 if not name in testInfos[directory]:
807 testInfos[directory][name] = []
808
809 testInfos[directory][name].append(way)
810
811 def addFailingTestInfo (testInfos, directory, name, reason, way):
812 directory = re.sub('^\\.[/\\\\]', '', directory)
813
814 if not directory in testInfos:
815 testInfos[directory] = {}
816
817 if not name in testInfos[directory]:
818 testInfos[directory][name] = {}
819
820 if not reason in testInfos[directory][name]:
821 testInfos[directory][name][reason] = []
822
823 testInfos[directory][name][reason].append(way)
824
825 def skiptest (name, way):
826 # print 'Skipping test \"', name, '\"'
827 t.n_tests_skipped = t.n_tests_skipped + 1
828 if name in t.tests_skipped:
829 t.tests_skipped[name].append(way)
830 else:
831 t.tests_skipped[name] = [way]
832
833 def framework_fail( name, way, reason ):
834 full_name = name + '(' + way + ')'
835 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
836 t.n_framework_failures = t.n_framework_failures + 1
837 if name in t.framework_failures:
838 t.framework_failures[name].append(way)
839 else:
840 t.framework_failures[name] = [way]
841
842 def badResult(result):
843 try:
844 if result['passFail'] == 'pass':
845 return False
846 return True
847 except:
848 return True
849
850 def passed():
851 return {'passFail': 'pass'}
852
853 def failBecause(reason):
854 return {'passFail': 'fail', 'reason': reason}
855
856 # -----------------------------------------------------------------------------
857 # Generic command tests
858
859 # A generic command test is expected to run and exit successfully.
860 #
861 # The expected exit code can be changed via exit_code() as normal, and
862 # the expected stdout/stderr are stored in <testname>.stdout and
863 # <testname>.stderr. The output of the command can be ignored
864 # altogether by using run_command_ignore_output instead of
865 # run_command.
866
867 def run_command( name, way, cmd ):
868 return simple_run( name, '', cmd, '' )
869
870 # -----------------------------------------------------------------------------
871 # GHCi tests
872
873 def ghci_script( name, way, script ):
874 # filter out -fforce-recomp from compiler_always_flags, because we're
875 # actually testing the recompilation behaviour in the GHCi tests.
876 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
877 flags.append(getTestOpts().extra_hc_opts)
878 if getTestOpts().outputdir != None:
879 flags.extend(["-outputdir", getTestOpts().outputdir])
880
881 # We pass HC and HC_OPTS as environment variables, so that the
882 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
883 cmd = "HC='" + config.compiler + "' " + \
884 "HC_OPTS='" + join(flags,' ') + "' " + \
885 "'" + config.compiler + "'" + \
886 ' --interactive -v0 -ignore-dot-ghci ' + \
887 join(flags,' ')
888
889 getTestOpts().stdin = script
890 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
891
892 # -----------------------------------------------------------------------------
893 # Compile-only tests
894
895 def compile( name, way, extra_hc_opts ):
896 return do_compile( name, way, 0, '', [], extra_hc_opts )
897
898 def compile_fail( name, way, extra_hc_opts ):
899 return do_compile( name, way, 1, '', [], extra_hc_opts )
900
901 def multimod_compile( name, way, top_mod, extra_hc_opts ):
902 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
903
904 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
905 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
906
907 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
908 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
909
910 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
911 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
912
913 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
914 # print 'Compile only, extra args = ', extra_hc_opts
915 pretest_cleanup(name)
916
917 result = extras_build( way, extra_mods, extra_hc_opts )
918 if badResult(result):
919 return result
920 extra_hc_opts = result['hc_opts']
921
922 force = 0
923 if extra_mods:
924 force = 1
925 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
926
927 if badResult(result):
928 return result
929
930 # the actual stderr should always match the expected, regardless
931 # of whether we expected the compilation to fail or not (successful
932 # compilations may generate warnings).
933
934 if getTestOpts().with_namebase == None:
935 namebase = name
936 else:
937 namebase = getTestOpts().with_namebase
938
939 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
940 actual_stderr_file = qualify(name, 'comp.stderr')
941
942 if not compare_outputs('stderr', \
943 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
944 expected_stderr_file, actual_stderr_file):
945 return failBecause('stderr mismatch')
946
947 # no problems found, this test passed
948 return passed()
949
950 def compile_cmp_asm( name, way, extra_hc_opts ):
951 print 'Compile only, extra args = ', extra_hc_opts
952 pretest_cleanup(name)
953 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
954
955 if badResult(result):
956 return result
957
958 # the actual stderr should always match the expected, regardless
959 # of whether we expected the compilation to fail or not (successful
960 # compilations may generate warnings).
961
962 if getTestOpts().with_namebase == None:
963 namebase = name
964 else:
965 namebase = getTestOpts().with_namebase
966
967 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
968 actual_asm_file = qualify(name, 's')
969
970 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
971 expected_asm_file, actual_asm_file):
972 return failBecause('asm mismatch')
973
974 # no problems found, this test passed
975 return passed()
976
977 # -----------------------------------------------------------------------------
978 # Compile-and-run tests
979
980 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
981 # print 'Compile and run, extra args = ', extra_hc_opts
982 pretest_cleanup(name)
983
984 result = extras_build( way, extra_mods, extra_hc_opts )
985 if badResult(result):
986 return result
987 extra_hc_opts = result['hc_opts']
988
989 if way == 'ghci': # interpreted...
990 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
991 elif way == 'extcore' or way == 'optextcore' :
992 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
993 else: # compiled...
994 force = 0
995 if extra_mods:
996 force = 1
997
998 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
999 if badResult(result):
1000 return result
1001
1002 cmd = './' + name;
1003
1004 # we don't check the compiler's stderr for a compile-and-run test
1005 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1006
1007 def compile_and_run( name, way, extra_hc_opts ):
1008 return compile_and_run__( name, way, '', [], extra_hc_opts)
1009
1010 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1011 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1012
1013 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1014 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1015
1016 def stats( name, way, stats_file ):
1017 opts = getTestOpts()
1018 return checkStats(stats_file, opts.stats_range_fields)
1019
1020 # -----------------------------------------------------------------------------
1021 # Check -t stats info
1022
1023 def checkStats(stats_file, range_fields):
1024 result = passed()
1025 if len(range_fields) > 0:
1026 f = open(in_testdir(stats_file))
1027 contents = f.read()
1028 f.close()
1029
1030 for (field, (expected, dev)) in range_fields.items():
1031 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1032 if m == None:
1033 print 'Failed to find field: ', field
1034 result = failBecause('no such stats field')
1035 val = int(m.group(1))
1036
1037 lowerBound = trunc( expected * ((100 - float(dev))/100));
1038 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1039
1040 if val < lowerBound:
1041 print field, 'value is too low:'
1042 print '(If this is because you have improved GHC, please'
1043 print 'update the test so that GHC doesn\'t regress again)'
1044 result = failBecause('stat too good')
1045 if val > upperBound:
1046 print field, 'value is too high:'
1047 result = failBecause('stat not good enough')
1048
1049 if val < lowerBound or val > upperBound:
1050 valStr = str(val)
1051 valLen = len(valStr)
1052 expectedStr = str(expected)
1053 expectedLen = len(expectedStr)
1054 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1055 def display(descr, val, extra):
1056 print descr, string.rjust(str(val), length), extra
1057 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1058 display(' Lower bound ' + field + ':', lowerBound, '')
1059 display(' Upper bound ' + field + ':', upperBound, '')
1060 display(' Actual ' + field + ':', val, '')
1061
1062 return result
1063
1064 # -----------------------------------------------------------------------------
1065 # Build a single-module program
1066
1067 def extras_build( way, extra_mods, extra_hc_opts ):
1068 for modopts in extra_mods:
1069 mod, opts = modopts
1070 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1071 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1072 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1073 if badResult(result):
1074 return result
1075
1076 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1077
1078
1079 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1080 opts = getTestOpts()
1081 errname = add_suffix(name, 'comp.stderr')
1082 rm_no_fail( qualify(errname, '') )
1083
1084 if top_mod != '':
1085 srcname = top_mod
1086 rm_no_fail( qualify(name, '') )
1087 base, suf = os.path.splitext(top_mod)
1088 rm_no_fail( qualify(base, '') )
1089 rm_no_fail( qualify(base, 'exe') )
1090 elif addsuf:
1091 srcname = add_hs_lhs_suffix(name)
1092 rm_no_fail( qualify(name, '') )
1093 else:
1094 srcname = name
1095 rm_no_fail( qualify(name, 'o') )
1096
1097 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1098
1099 to_do = ''
1100 if top_mod != '':
1101 to_do = '--make '
1102 if link:
1103 to_do = to_do + '-o ' + name
1104 elif link:
1105 to_do = '-o ' + name
1106 elif opts.compile_to_hc:
1107 to_do = '-C'
1108 else:
1109 to_do = '-c' # just compile
1110
1111 stats_file = name + '.comp.stats'
1112 if len(opts.compiler_stats_range_fields) > 0:
1113 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1114
1115 # Required by GHC 7.3+, harmless for earlier versions:
1116 if (getTestOpts().c_src or
1117 getTestOpts().objc_src or
1118 getTestOpts().objcpp_src or
1119 getTestOpts().cmm_src):
1120 extra_hc_opts += ' -no-hs-main '
1121
1122 if getTestOpts().compile_cmd_prefix == '':
1123 cmd_prefix = ''
1124 else:
1125 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1126
1127 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1128 if noforce:
1129 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1130 if getTestOpts().outputdir != None:
1131 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1132
1133 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1134 + config.compiler + "' " \
1135 + join(comp_flags,' ') + ' ' \
1136 + to_do + ' ' + srcname + ' ' \
1137 + join(config.way_flags(name)[way],' ') + ' ' \
1138 + extra_hc_opts + ' ' \
1139 + opts.extra_hc_opts + ' ' \
1140 + '>' + errname + ' 2>&1'
1141
1142 result = runCmdFor(name, cmd)
1143
1144 if result != 0 and not should_fail:
1145 actual_stderr = qualify(name, 'comp.stderr')
1146 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1147 if_verbose_dump(1,actual_stderr)
1148
1149 # ToDo: if the sub-shell was killed by ^C, then exit
1150
1151 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1152
1153 if badResult(statsResult):
1154 return statsResult
1155
1156 if should_fail:
1157 if result == 0:
1158 return failBecause('exit code 0')
1159 else:
1160 if result != 0:
1161 return failBecause('exit code non-0')
1162
1163 return passed()
1164
1165 # -----------------------------------------------------------------------------
1166 # Run a program and check its output
1167 #
1168 # If testname.stdin exists, route input from that, else
1169 # from /dev/null. Route output to testname.run.stdout and
1170 # testname.run.stderr. Returns the exit code of the run.
1171
1172 def simple_run( name, way, prog, args ):
1173 opts = getTestOpts()
1174
1175 # figure out what to use for stdin
1176 if opts.stdin != '':
1177 use_stdin = opts.stdin
1178 else:
1179 stdin_file = add_suffix(name, 'stdin')
1180 if os.path.exists(in_testdir(stdin_file)):
1181 use_stdin = stdin_file
1182 else:
1183 use_stdin = '/dev/null'
1184
1185 run_stdout = add_suffix(name,'run.stdout')
1186 run_stderr = add_suffix(name,'run.stderr')
1187
1188 rm_no_fail(qualify(name,'run.stdout'))
1189 rm_no_fail(qualify(name,'run.stderr'))
1190 rm_no_fail(qualify(name, 'hp'))
1191 rm_no_fail(qualify(name,'ps'))
1192 rm_no_fail(qualify(name, 'prof'))
1193
1194 my_rts_flags = rts_flags(way)
1195
1196 stats_file = name + '.stats'
1197 if len(opts.stats_range_fields) > 0:
1198 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1199
1200 if opts.no_stdin:
1201 stdin_comes_from = ''
1202 else:
1203 stdin_comes_from = ' <' + use_stdin
1204
1205 if opts.combined_output:
1206 redirection = ' >' + run_stdout \
1207 + ' 2>&1'
1208 else:
1209 redirection = ' >' + run_stdout \
1210 + ' 2>' + run_stderr
1211
1212 cmd = prog + ' ' + args + ' ' \
1213 + my_rts_flags + ' ' \
1214 + stdin_comes_from \
1215 + redirection
1216
1217 if opts.cmd_wrapper != None:
1218 cmd = opts.cmd_wrapper(cmd);
1219
1220 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1221
1222 # run the command
1223 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1224
1225 exit_code = result >> 8
1226 signal = result & 0xff
1227
1228 # check the exit code
1229 if exit_code != opts.exit_code:
1230 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1231 dump_stdout(name)
1232 dump_stderr(name)
1233 return failBecause('bad exit code')
1234
1235 check_hp = my_rts_flags.find("-h") != -1
1236 check_prof = my_rts_flags.find("-p") != -1
1237
1238 if not opts.ignore_output:
1239 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1240 bad_stdout = not check_stdout_ok(name)
1241 if bad_stderr:
1242 return failBecause('bad stderr')
1243 if bad_stdout:
1244 return failBecause('bad stdout')
1245 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1246 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1247 return failBecause('bad heap profile')
1248 if check_prof and not check_prof_ok(name):
1249 return failBecause('bad profile')
1250
1251 return checkStats(stats_file, opts.stats_range_fields)
1252
1253 def rts_flags(way):
1254 if (way == ''):
1255 return ''
1256 else:
1257 args = config.way_rts_flags[way]
1258
1259 if args == []:
1260 return ''
1261 else:
1262 return '+RTS ' + join(args,' ') + ' -RTS'
1263
1264 # -----------------------------------------------------------------------------
1265 # Run a program in the interpreter and check its output
1266
1267 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1268 outname = add_suffix(name, 'interp.stdout')
1269 errname = add_suffix(name, 'interp.stderr')
1270 rm_no_fail(outname)
1271 rm_no_fail(errname)
1272 rm_no_fail(name)
1273
1274 if (top_mod == ''):
1275 srcname = add_hs_lhs_suffix(name)
1276 else:
1277 srcname = top_mod
1278
1279 scriptname = add_suffix(name, 'genscript')
1280 qscriptname = in_testdir(scriptname)
1281 rm_no_fail(qscriptname)
1282
1283 delimiter = '===== program output begins here\n'
1284
1285 script = open(qscriptname, 'w')
1286 if not compile_only:
1287 # set the prog name and command-line args to match the compiled
1288 # environment.
1289 script.write(':set prog ' + name + '\n')
1290 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1291 # Add marker lines to the stdout and stderr output files, so we
1292 # can separate GHCi's output from the program's.
1293 script.write(':! echo ' + delimiter)
1294 script.write(':! echo 1>&2 ' + delimiter)
1295 # Set stdout to be line-buffered to match the compiled environment.
1296 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1297 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1298 # in the event of an exception as for the compiled program.
1299 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1300 script.close()
1301
1302 # figure out what to use for stdin
1303 if getTestOpts().stdin != '':
1304 stdin_file = in_testdir(getTestOpts().stdin)
1305 else:
1306 stdin_file = qualify(name, 'stdin')
1307
1308 if os.path.exists(stdin_file):
1309 stdin = open(stdin_file, 'r')
1310 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1311
1312 script.close()
1313
1314 flags = copy.copy(getTestOpts().compiler_always_flags)
1315 if getTestOpts().outputdir != None:
1316 flags.extend(["-outputdir", getTestOpts().outputdir])
1317
1318 cmd = "'" + config.compiler + "' " \
1319 + join(flags,' ') + ' ' \
1320 + srcname + ' ' \
1321 + join(config.way_flags(name)[way],' ') + ' ' \
1322 + extra_hc_opts + ' ' \
1323 + getTestOpts().extra_hc_opts + ' ' \
1324 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1325
1326 if getTestOpts().cmd_wrapper != None:
1327 cmd = getTestOpts().cmd_wrapper(cmd);
1328
1329 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1330
1331 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1332
1333 exit_code = result >> 8
1334 signal = result & 0xff
1335
1336 # split the stdout into compilation/program output
1337 split_file(in_testdir(outname), delimiter,
1338 qualify(name, 'comp.stdout'),
1339 qualify(name, 'run.stdout'))
1340 split_file(in_testdir(errname), delimiter,
1341 qualify(name, 'comp.stderr'),
1342 qualify(name, 'run.stderr'))
1343
1344 # check the exit code
1345 if exit_code != getTestOpts().exit_code:
1346 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1347 dump_stdout(name)
1348 dump_stderr(name)
1349 return failBecause('bad exit code')
1350
1351 # ToDo: if the sub-shell was killed by ^C, then exit
1352
1353 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1354 check_stdout_ok(name)):
1355 return passed()
1356 else:
1357 return failBecause('bad stdout or stderr')
1358
1359
1360 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1361 infile = open(in_fn)
1362 out1 = open(out1_fn, 'w')
1363 out2 = open(out2_fn, 'w')
1364
1365 line = infile.readline()
1366 line = re.sub('\r', '', line) # ignore Windows EOL
1367 while (re.sub('^\s*','',line) != delimiter and line != ''):
1368 out1.write(line)
1369 line = infile.readline()
1370 line = re.sub('\r', '', line)
1371 out1.close()
1372
1373 line = infile.readline()
1374 while (line != ''):
1375 out2.write(line)
1376 line = infile.readline()
1377 out2.close()
1378
1379 # -----------------------------------------------------------------------------
1380 # Generate External Core for the given program, then compile the resulting Core
1381 # and compare its output to the expected output
1382
1383 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1384
1385 depsfilename = qualify(name, 'deps')
1386 errname = add_suffix(name, 'comp.stderr')
1387 qerrname = qualify(errname,'')
1388
1389 hcname = qualify(name, 'hc')
1390 oname = qualify(name, 'o')
1391
1392 rm_no_fail( qerrname )
1393 rm_no_fail( qualify(name, '') )
1394
1395 if (top_mod == ''):
1396 srcname = add_hs_lhs_suffix(name)
1397 else:
1398 srcname = top_mod
1399
1400 qcorefilename = qualify(name, 'hcr')
1401 corefilename = add_suffix(name, 'hcr')
1402 rm_no_fail(qcorefilename)
1403
1404 # Generate External Core
1405
1406 if (top_mod == ''):
1407 to_do = ' ' + srcname + ' '
1408 else:
1409 to_do = ' --make ' + top_mod + ' '
1410
1411 flags = copy.copy(getTestOpts().compiler_always_flags)
1412 if getTestOpts().outputdir != None:
1413 flags.extend(["-outputdir", getTestOpts().outputdir])
1414 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1415 + config.compiler + "' " \
1416 + join(flags,' ') + ' ' \
1417 + join(config.way_flags(name)[way],' ') + ' ' \
1418 + extra_hc_opts + ' ' \
1419 + getTestOpts().extra_hc_opts \
1420 + to_do \
1421 + '>' + errname + ' 2>&1'
1422 result = runCmdFor(name, cmd)
1423
1424 exit_code = result >> 8
1425
1426 if exit_code != 0:
1427 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1428 if_verbose_dump(1,qerrname)
1429 return failBecause('ext core exit code non-0')
1430
1431 # Compile the resulting files -- if there's more than one module, we need to read the output
1432 # of the previous compilation in order to find the dependencies
1433 if (top_mod == ''):
1434 to_compile = corefilename
1435 else:
1436 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1437 deps = open(depsfilename).read()
1438 deplist = string.replace(deps, '\n',' ');
1439 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1440 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1441
1442 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1443 if getTestOpts().outputdir != None:
1444 flags.extend(["-outputdir", getTestOpts().outputdir])
1445
1446 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1447 + config.compiler + "' " \
1448 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1449 + to_compile + ' ' \
1450 + extra_hc_opts + ' ' \
1451 + getTestOpts().extra_hc_opts + ' ' \
1452 + flags \
1453 + ' -fglasgow-exts -o ' + name \
1454 + '>' + errname + ' 2>&1'
1455
1456 result = runCmdFor(name, cmd)
1457 exit_code = result >> 8
1458
1459 if exit_code != 0:
1460 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1461 if_verbose_dump(1,qerrname)
1462 return failBecause('ext core exit code non-0')
1463
1464 # Clean up
1465 rm_no_fail ( oname )
1466 rm_no_fail ( hcname )
1467 rm_no_fail ( qcorefilename )
1468 rm_no_fail ( depsfilename )
1469
1470 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1471
1472 # -----------------------------------------------------------------------------
1473 # Utils
1474
1475 def check_stdout_ok( name ):
1476 if getTestOpts().with_namebase == None:
1477 namebase = name
1478 else:
1479 namebase = getTestOpts().with_namebase
1480
1481 actual_stdout_file = qualify(name, 'run.stdout')
1482 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1483
1484 def norm(str):
1485 if platform_specific:
1486 return str
1487 else:
1488 return normalise_output(str)
1489
1490 return compare_outputs('stdout', \
1491 two_normalisers(norm, getTestOpts().extra_normaliser), \
1492 expected_stdout_file, actual_stdout_file)
1493
1494 def dump_stdout( name ):
1495 print 'Stdout:'
1496 print read_no_crs(qualify(name, 'run.stdout'))
1497
1498 def check_stderr_ok( name ):
1499 if getTestOpts().with_namebase == None:
1500 namebase = name
1501 else:
1502 namebase = getTestOpts().with_namebase
1503
1504 actual_stderr_file = qualify(name, 'run.stderr')
1505 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1506
1507 def norm(str):
1508 if platform_specific:
1509 return str
1510 else:
1511 return normalise_errmsg(str)
1512
1513 return compare_outputs('stderr', \
1514 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1515 expected_stderr_file, actual_stderr_file)
1516
1517 def dump_stderr( name ):
1518 print "Stderr:"
1519 print read_no_crs(qualify(name, 'run.stderr'))
1520
1521 def read_no_crs(file):
1522 str = ''
1523 try:
1524 h = open(file)
1525 str = h.read()
1526 h.close
1527 except:
1528 # On Windows, if the program fails very early, it seems the
1529 # files stdout/stderr are redirected to may not get created
1530 pass
1531 return re.sub('\r', '', str)
1532
1533 def write_file(file, str):
1534 h = open(file, 'w')
1535 h.write(str)
1536 h.close
1537
1538 def check_hp_ok(name):
1539
1540 # do not qualify for hp2ps because we should be in the right directory
1541 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1542
1543 hp2psResult = runCmdExitCode(hp2psCmd)
1544
1545 actual_ps_file = qualify(name, 'ps')
1546
1547 if(hp2psResult == 0):
1548 if (os.path.exists(actual_ps_file)):
1549 if gs_working:
1550 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1551 if (gsResult == 0):
1552 return (True)
1553 else:
1554 print "hp2ps output for " + name + "is not valid PostScript"
1555 else: return (True) # assume postscript is valid without ghostscript
1556 else:
1557 print "hp2ps did not generate PostScript for " + name
1558 return (False)
1559 else:
1560 print "hp2ps error when processing heap profile for " + name
1561 return(False)
1562
1563 def check_prof_ok(name):
1564
1565 prof_file = qualify(name,'prof')
1566
1567 if not os.path.exists(prof_file):
1568 print prof_file + " does not exist"
1569 return(False)
1570
1571 if os.path.getsize(qualify(name,'prof')) == 0:
1572 print prof_file + " is empty"
1573 return(False)
1574
1575 if getTestOpts().with_namebase == None:
1576 namebase = name
1577 else:
1578 namebase = getTestOpts().with_namebase
1579
1580 (platform_specific, expected_prof_file) = \
1581 platform_wordsize_qualify(namebase, 'prof.sample')
1582
1583 # sample prof file is not required
1584 if not os.path.exists(expected_prof_file):
1585 return True
1586 else:
1587 return compare_outputs('prof', \
1588 two_normalisers(normalise_whitespace,normalise_prof), \
1589 expected_prof_file, prof_file)
1590
1591 # Compare expected output to actual output, and optionally accept the
1592 # new output. Returns true if output matched or was accepted, false
1593 # otherwise.
1594 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1595 if os.path.exists(expected_file):
1596 expected_raw = read_no_crs(expected_file)
1597 # print "norm:", normaliser(expected_raw)
1598 expected_str = normaliser(expected_raw)
1599 expected_file_for_diff = expected_file
1600 else:
1601 expected_str = ''
1602 expected_file_for_diff = '/dev/null'
1603
1604 actual_raw = read_no_crs(actual_file)
1605 actual_str = normaliser(actual_raw)
1606
1607 if expected_str == actual_str:
1608 return 1
1609 else:
1610 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1611
1612 if expected_file_for_diff == '/dev/null':
1613 expected_normalised_file = '/dev/null'
1614 else:
1615 expected_normalised_file = expected_file + ".normalised"
1616 write_file(expected_normalised_file, expected_str)
1617
1618 actual_normalised_file = actual_file + ".normalised"
1619 write_file(actual_normalised_file, actual_str)
1620
1621 # Ignore whitespace when diffing. We should only get to this
1622 # point if there are non-whitespace differences
1623 #
1624 # Note we are diffing the *actual* output, not the normalised
1625 # output. The normalised output may have whitespace squashed
1626 # (including newlines) so the diff would be hard to read.
1627 # This does mean that the diff might contain changes that
1628 # would be normalised away.
1629 if (config.verbose >= 1):
1630 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1631 ' ' + actual_file )
1632
1633 # If for some reason there were no non-whitespace differences,
1634 # then do a full diff
1635 if r == 0:
1636 r = os.system( 'diff -u ' + expected_file_for_diff + \
1637 ' ' + actual_file )
1638
1639 if config.accept:
1640 if_verbose(1, 'Accepting new output.')
1641 write_file(expected_file, actual_raw)
1642 return 1
1643 else:
1644 return 0
1645
1646
1647 def normalise_whitespace( str ):
1648 # Merge contiguous whitespace characters into a single space.
1649 str = re.sub('[ \t\n]+', ' ', str)
1650 return str
1651
1652 def normalise_errmsg( str ):
1653 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1654 # the colon is there because it appears in error messages; this
1655 # hacky solution is used in place of more sophisticated filename
1656 # mangling
1657 str = re.sub('([^\\s])\\.exe', '\\1', str)
1658 # normalise slashes, minimise Windows/Unix filename differences
1659 str = re.sub('\\\\', '/', str)
1660 # The inplace ghc's are called ghc-stage[123] to avoid filename
1661 # collisions, so we need to normalise that to just "ghc"
1662 str = re.sub('ghc-stage[123]', 'ghc', str)
1663 # We sometimes see the name of the integer-gmp package on stderr,
1664 # but this can change (either the implementation name or the
1665 # version number), so we canonicalise it here
1666 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1667 return str
1668
1669 # normalise a .prof file, so that we can reasonably compare it against
1670 # a sample. This doesn't compare any of the actual profiling data,
1671 # only the shape of the profile and the number of entries.
1672 def normalise_prof (str):
1673 # strip everything up to the line beginning "COST CENTRE"
1674 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1675
1676 # strip results for CAFs, these tend to change unpredictably
1677 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1678
1679 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1680 # sometimes under MAIN.
1681 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1682
1683 # We have somthing like this:
1684
1685 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1686 # k Main 204 1 0.0 0.0 0.0 0.0
1687 # foo Main 205 1 0.0 0.0 0.0 0.0
1688 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1689
1690 # then we remove all the specific profiling data, leaving only the
1691 # cost centre name, module, and entries, to end up with this:
1692
1693 # MAIN MAIN 0
1694 # k Main 1
1695 # foo Main 1
1696 # foo.bar Main 1
1697
1698 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1699 return str
1700
1701 def normalise_slashes_( str ):
1702 str = re.sub('\\\\', '/', str)
1703 return str
1704
1705 def normalise_exe_( str ):
1706 str = re.sub('\.exe', '', str)
1707 return str
1708
1709 def normalise_output( str ):
1710 # Remove a .exe extension (for Windows)
1711 # This can occur in error messages generated by the program.
1712 str = re.sub('([^\\s])\\.exe', '\\1', str)
1713 return str
1714
1715 def normalise_asm( str ):
1716 lines = str.split('\n')
1717 # Only keep instructions and labels not starting with a dot.
1718 metadata = re.compile('^[ \t]*\\..*$')
1719 out = []
1720 for line in lines:
1721 # Drop metadata directives (e.g. ".type")
1722 if not metadata.match(line):
1723 line = re.sub('@plt', '', line)
1724 instr = line.lstrip().split()
1725 # Drop empty lines.
1726 if not instr:
1727 continue
1728 # Drop operands, except for call instructions.
1729 elif instr[0] == 'call':
1730 out.append(instr[0] + ' ' + instr[1])
1731 else:
1732 out.append(instr[0])
1733 out = '\n'.join(out)
1734 return out
1735
1736 def if_verbose( n, str ):
1737 if config.verbose >= n:
1738 print str
1739
1740 def if_verbose_dump( n, f ):
1741 if config.verbose >= n:
1742 try:
1743 print open(f).read()
1744 except:
1745 print ''
1746
1747 def rawSystem(cmd_and_args):
1748 # We prefer subprocess.call to os.spawnv as the latter
1749 # seems to send its arguments through a shell or something
1750 # with the Windows (non-cygwin) python. An argument "a b c"
1751 # turns into three arguments ["a", "b", "c"].
1752
1753 # However, subprocess is new in python 2.4, so fall back to
1754 # using spawnv if we don't have it
1755
1756 if have_subprocess:
1757 return subprocess.call(cmd_and_args)
1758 else:
1759 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1760
1761 # Note that this doesn't handle the timeout itself; it is just used for
1762 # commands that have timeout handling built-in.
1763 def rawSystemWithTimeout(cmd_and_args):
1764 r = rawSystem(cmd_and_args)
1765 if r == 98:
1766 # The python timeout program uses 98 to signal that ^C was pressed
1767 stopNow()
1768 return r
1769
1770 # cmd is a complex command in Bourne-shell syntax
1771 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1772 # Hence it must ultimately be run by a Bourne shell
1773 #
1774 # Mostly it invokes the command wrapped in 'timeout' thus
1775 # timeout 300 'cd . && ...blah blah'
1776 # so it's timeout's job to invoke the Bourne shell
1777 #
1778 # But watch out for the case when there is no timeout program!
1779 # Then, when using the native Python, os.system will invoke the cmd shell
1780
1781 def runCmd( cmd ):
1782 if_verbose( 3, cmd )
1783 r = 0
1784 if config.os == 'mingw32':
1785 # On MinGW, we will always have timeout
1786 assert config.timeout_prog!=''
1787
1788 if config.timeout_prog != '':
1789 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1790 else:
1791 r = os.system(cmd)
1792 return r << 8
1793
1794 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1795 if_verbose( 3, cmd )
1796 r = 0
1797 if config.os == 'mingw32':
1798 # On MinGW, we will always have timeout
1799 assert config.timeout_prog!=''
1800 timeout = int(ceil(config.timeout * timeout_multiplier))
1801
1802 if config.timeout_prog != '':
1803 if config.check_files_written:
1804 fn = name + ".strace"
1805 r = rawSystemWithTimeout(
1806 ["strace", "-o", fn, "-fF",
1807 "-e", "creat,open,chdir,clone,vfork",
1808 config.timeout_prog, str(timeout), cmd])
1809 addTestFilesWritten(name, fn)
1810 rm_no_fail(fn)
1811 else:
1812 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1813 else:
1814 r = os.system(cmd)
1815 return r << 8
1816
1817 def runCmdExitCode( cmd ):
1818 return (runCmd(cmd) >> 8);
1819
1820
1821 # -----------------------------------------------------------------------------
1822 # checking for files being written to by multiple tests
1823
1824 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1825 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1826 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1827 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1828 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1829 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1830 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1831 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1832 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1833 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1834 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1835 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1836 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1837 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1838 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1839 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1840 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1841
1842 # Files that are read or written but shouldn't be:
1843 # * ghci_history shouldn't be read or written by tests
1844 # * things under package.conf.d shouldn't be written by tests
1845 bad_file_usages = {}
1846
1847 # Mapping from tests to the list of files that they write
1848 files_written = {}
1849
1850 # Mapping from tests to the list of files that they write but don't clean
1851 files_written_not_removed = {}
1852
1853 def add_bad_file_usage(name, file):
1854 try:
1855 if not file in bad_file_usages[name]:
1856 bad_file_usages[name].append(file)
1857 except:
1858 bad_file_usages[name] = [file]
1859
1860 def mkPath(curdir, path):
1861 # Given the current full directory is 'curdir', what is the full
1862 # path to 'path'?
1863 return os.path.realpath(os.path.join(curdir, path))
1864
1865 def addTestFilesWritten(name, fn):
1866 if config.use_threads:
1867 with t.lockFilesWritten:
1868 addTestFilesWrittenHelper(name, fn)
1869 else:
1870 addTestFilesWrittenHelper(name, fn)
1871
1872 def addTestFilesWrittenHelper(name, fn):
1873 started = False
1874 working_directories = {}
1875
1876 with open(fn, 'r') as f:
1877 for line in f:
1878 m_pid = re_strace_pid.match(line)
1879 if m_pid:
1880 pid = m_pid.group(1)
1881 content = m_pid.group(2)
1882 elif re_strace_unavailable.match(line):
1883 next
1884 else:
1885 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1886
1887 m_open = re_strace_open.match(content)
1888 m_chdir = re_strace_chdir.match(content)
1889 m_clone = re_strace_clone.match(content)
1890 m_vfork = re_strace_vfork.match(content)
1891
1892 if not started:
1893 working_directories[pid] = os.getcwd()
1894 started = True
1895
1896 if m_open:
1897 file = m_open.group(1)
1898 file = mkPath(working_directories[pid], file)
1899 if file.endswith("ghci_history"):
1900 add_bad_file_usage(name, file)
1901 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1902 flags = m_open.group(2).split('|')
1903 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1904 if re.match('package\.conf\.d', file):
1905 add_bad_file_usage(name, file)
1906 else:
1907 try:
1908 if not file in files_written[name]:
1909 files_written[name].append(file)
1910 except:
1911 files_written[name] = [file]
1912 elif 'O_RDONLY' in flags:
1913 pass
1914 else:
1915 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1916 elif m_chdir:
1917 # We optimistically assume that unfinished chdir's are going to succeed
1918 dir = m_chdir.group(1)
1919 working_directories[pid] = mkPath(working_directories[pid], dir)
1920 elif m_clone:
1921 working_directories[m_clone.group(2)] = working_directories[pid]
1922 elif m_vfork:
1923 working_directories[m_vfork.group(2)] = working_directories[pid]
1924 elif re_strace_open_resumed.match(content):
1925 pass
1926 elif re_strace_chdir_resumed.match(content):
1927 pass
1928 elif re_strace_vfork_unfinished.match(content):
1929 pass
1930 elif re_strace_clone_unfinished.match(content):
1931 pass
1932 elif re_strace_ignore_sigchild.match(content):
1933 pass
1934 elif re_strace_ignore_sigvtalarm.match(content):
1935 pass
1936 elif re_strace_ignore_sigint.match(content):
1937 pass
1938 elif re_strace_ignore_sigfpe.match(content):
1939 pass
1940 elif re_strace_ignore_sigsegv.match(content):
1941 pass
1942 elif re_strace_ignore_sigpipe.match(content):
1943 pass
1944 else:
1945 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1946
1947 def checkForFilesWrittenProblems(file):
1948 foundProblem = False
1949
1950 files_written_inverted = {}
1951 for t in files_written.keys():
1952 for f in files_written[t]:
1953 try:
1954 files_written_inverted[f].append(t)
1955 except:
1956 files_written_inverted[f] = [t]
1957
1958 for f in files_written_inverted.keys():
1959 if len(files_written_inverted[f]) > 1:
1960 if not foundProblem:
1961 foundProblem = True
1962 file.write("\n")
1963 file.write("\nSome files are written by multiple tests:\n")
1964 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1965 if foundProblem:
1966 file.write("\n")
1967
1968 # -----
1969
1970 if len(files_written_not_removed) > 0:
1971 file.write("\n")
1972 file.write("\nSome files written but not removed:\n")
1973 tests = files_written_not_removed.keys()
1974 tests.sort()
1975 for t in tests:
1976 for f in files_written_not_removed[t]:
1977 file.write(" " + t + ": " + f + "\n")
1978 file.write("\n")
1979
1980 # -----
1981
1982 if len(bad_file_usages) > 0:
1983 file.write("\n")
1984 file.write("\nSome bad file usages:\n")
1985 tests = bad_file_usages.keys()
1986 tests.sort()
1987 for t in tests:
1988 for f in bad_file_usages[t]:
1989 file.write(" " + t + ": " + f + "\n")
1990 file.write("\n")
1991
1992 # -----------------------------------------------------------------------------
1993 # checking if ghostscript is available for checking the output of hp2ps
1994
1995 def genGSCmd(psfile):
1996 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1997
1998 def gsNotWorking():
1999 global gs_working
2000 print "GhostScript not available for hp2ps tests"
2001
2002 global gs_working
2003 gs_working = 0
2004 if config.have_profiling:
2005 if config.gs != '':
2006 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2007 if resultGood == 0:
2008 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2009 if resultBad != 0:
2010 print "GhostScript available for hp2ps tests"
2011 gs_working = 1;
2012 else:
2013 gsNotWorking();
2014 else:
2015 gsNotWorking();
2016 else:
2017 gsNotWorking();
2018
2019 def rm_no_fail( file ):
2020 try:
2021 os.remove( file )
2022 finally:
2023 return
2024
2025 def add_suffix( name, suffix ):
2026 if suffix == '':
2027 return name
2028 else:
2029 return name + '.' + suffix
2030
2031 def add_hs_lhs_suffix(name):
2032 if getTestOpts().c_src:
2033 return add_suffix(name, 'c')
2034 elif getTestOpts().cmm_src:
2035 return add_suffix(name, 'cmm')
2036 elif getTestOpts().objc_src:
2037 return add_suffix(name, 'm')
2038 elif getTestOpts().objcpp_src:
2039 return add_suffix(name, 'mm')
2040 elif getTestOpts().literate:
2041 return add_suffix(name, 'lhs')
2042 else:
2043 return add_suffix(name, 'hs')
2044
2045 def replace_suffix( name, suffix ):
2046 base, suf = os.path.splitext(name)
2047 return base + '.' + suffix
2048
2049 def in_testdir( name ):
2050 return (getTestOpts().testdir + '/' + name)
2051
2052 def qualify( name, suff ):
2053 return in_testdir(add_suffix(name, suff))
2054
2055
2056 # Finding the sample output. The filename is of the form
2057 #
2058 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2059 #
2060 # and we pick the most specific version available. The <version> is
2061 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2062 # more fine-grained control use if_compiler_lt().
2063 #
2064 def platform_wordsize_qualify( name, suff ):
2065
2066 basepath = qualify(name, suff)
2067
2068 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2069 for (platformSpecific, plat) in [(1, '-' + config.platform),
2070 (1, '-' + config.os),
2071 (0, '')]
2072 for ws in ['-ws-' + config.wordsize, '']
2073 for comp in ['-' + config.compiler_type, '']
2074 for vers in ['-' + config.compiler_maj_version, '']]
2075
2076 dir = glob.glob(basepath + '*')
2077 dir = map (lambda d: normalise_slashes_(d), dir)
2078
2079 for (platformSpecific, f) in paths:
2080 if f in dir:
2081 return (platformSpecific,f)
2082
2083 return (0, basepath)
2084
2085 # Clean up prior to the test, so that we can't spuriously conclude
2086 # that it passed on the basis of old run outputs.
2087 def pretest_cleanup(name):
2088 if getTestOpts().outputdir != None:
2089 odir = in_testdir(getTestOpts().outputdir)
2090 try:
2091 shutil.rmtree(odir)
2092 except:
2093 pass
2094 os.mkdir(odir)
2095
2096 rm_no_fail(qualify(name,'interp.stderr'))
2097 rm_no_fail(qualify(name,'interp.stdout'))
2098 rm_no_fail(qualify(name,'comp.stderr'))
2099 rm_no_fail(qualify(name,'comp.stdout'))
2100 rm_no_fail(qualify(name,'run.stderr'))
2101 rm_no_fail(qualify(name,'run.stdout'))
2102 rm_no_fail(qualify(name,'tix'))
2103 rm_no_fail(qualify(name,'exe.tix'))
2104 # simple_build zaps the following:
2105 # rm_nofail(qualify("o"))
2106 # rm_nofail(qualify(""))
2107 # not interested in the return code
2108
2109 # -----------------------------------------------------------------------------
2110 # Return a list of all the files ending in '.T' below the directory dir.
2111
2112 def findTFiles(roots):
2113 return concat(map(findTFiles_,roots))
2114
2115 def findTFiles_(path):
2116 if os.path.isdir(path):
2117 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2118 return findTFiles(paths)
2119 elif path[-2:] == '.T':
2120 return [path]
2121 else:
2122 return []
2123
2124 # -----------------------------------------------------------------------------
2125 # Output a test summary to the specified file object
2126
2127 def summary(t, file):
2128
2129 file.write('\n')
2130 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2131 file.write('OVERALL SUMMARY for test run started at '
2132 + time.strftime("%c %Z", t.start_time) + '\n'
2133 + string.rjust(str(datetime.timedelta(seconds=
2134 round(time.time() - time.mktime(t.start_time)))), 8)
2135 + ' spent to go through\n'
2136 + string.rjust(`t.total_tests`, 8)
2137 + ' total tests, which gave rise to\n'
2138 + string.rjust(`t.total_test_cases`, 8)
2139 + ' test cases, of which\n'
2140 + string.rjust(`t.n_tests_skipped`, 8)
2141 + ' were skipped\n'
2142 + '\n'
2143 + string.rjust(`t.n_missing_libs`, 8)
2144 + ' had missing libraries\n'
2145 + string.rjust(`t.n_expected_passes`, 8)
2146 + ' expected passes\n'
2147 + string.rjust(`t.n_expected_failures`, 8)
2148 + ' expected failures\n'
2149 + '\n'
2150 + string.rjust(`t.n_framework_failures`, 8)
2151 + ' caused framework failures\n'
2152 + string.rjust(`t.n_unexpected_passes`, 8)
2153 + ' unexpected passes\n'
2154 + string.rjust(`t.n_unexpected_failures`, 8)
2155 + ' unexpected failures\n'
2156 + '\n')
2157
2158 if t.n_unexpected_passes > 0:
2159 file.write('Unexpected passes:\n')
2160 printPassingTestInfosSummary(file, t.unexpected_passes)
2161
2162 if t.n_unexpected_failures > 0:
2163 file.write('Unexpected failures:\n')
2164 printFailingTestInfosSummary(file, t.unexpected_failures)
2165
2166 if config.check_files_written:
2167 checkForFilesWrittenProblems(file)
2168
2169 if stopping():
2170 file.write('WARNING: Testsuite run was terminated early\n')
2171
2172 def printUnexpectedTests(file, testInfoss):
2173 unexpected = []
2174 for testInfos in testInfoss:
2175 directories = testInfos.keys()
2176 for directory in directories:
2177 tests = testInfos[directory].keys()
2178 unexpected += tests
2179 if unexpected != []:
2180 file.write('Unexpected results from:\n')
2181 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2182 file.write('\n')
2183
2184 def printPassingTestInfosSummary(file, testInfos):
2185 directories = testInfos.keys()
2186 directories.sort()
2187 maxDirLen = max(map ((lambda x : len(x)), directories))
2188 for directory in directories:
2189 tests = testInfos[directory].keys()
2190 tests.sort()
2191 for test in tests:
2192 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2193 ' (' + join(testInfos[directory][test],',') + ')\n')
2194 file.write('\n')
2195
2196 def printFailingTestInfosSummary(file, testInfos):
2197 directories = testInfos.keys()
2198 directories.sort()
2199 maxDirLen = max(map ((lambda x : len(x)), directories))
2200 for directory in directories:
2201 tests = testInfos[directory].keys()
2202 tests.sort()
2203 for test in tests:
2204 reasons = testInfos[directory][test].keys()
2205 for reason in reasons:
2206 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2207 ' [' + reason + ']' + \
2208 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2209 file.write('\n')
2210
2211 def getStdout(cmd):
2212 if have_subprocess:
2213 p = subprocess.Popen(cmd,
2214 stdout=subprocess.PIPE,
2215 stderr=subprocess.PIPE)
2216 (stdout, stderr) = p.communicate()
2217 r = p.wait()
2218 if r != 0:
2219 raise Exception("Command failed: " + str(cmd))
2220 if stderr != '':
2221 raise Exception("stderr from command: " + str(cmd))
2222 return stdout
2223 else:
2224 raise Exception("Need subprocess to get stdout, but don't have it")