7fcdaefe8e49872a39f92c73a875110697b2ad2e
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 record_broken(name, bug)
155 opts.expect = 'fail';
156
157 def expect_broken_for( bug, ways ):
158 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
159
160 def _expect_broken_for( name, opts, bug, ways ):
161 record_broken(name, bug)
162 opts.expect_fail_for = ways
163
164 def record_broken(name, bug):
165 global brokens
166 me = (bug, name)
167 if not me in brokens:
168 brokens.append(me)
169
170 # -----
171
172 def omit_ways( ways ):
173 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174
175 def _omit_ways( name, opts, ways ):
176 opts.omit_ways = ways
177
178 # -----
179
180 def only_ways( ways ):
181 return lambda name, opts, w=ways: _only_ways( name, opts, w )
182
183 def _only_ways( name, opts, ways ):
184 opts.only_ways = ways
185
186 # -----
187
188 def extra_ways( ways ):
189 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190
191 def _extra_ways( name, opts, ways ):
192 opts.extra_ways = ways
193
194 # -----
195
196 def omit_compiler_types( compiler_types ):
197 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
198
199 def _omit_compiler_types( name, opts, compiler_types ):
200 if config.compiler_type in compiler_types:
201 opts.skip = 1
202
203 # -----
204
205 def only_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
207
208 def _only_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type not in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def set_stdin( file ):
215 return lambda name, opts, f=file: _set_stdin(name, opts, f);
216
217 def _set_stdin( name, opts, f ):
218 opts.stdin = f
219
220 # -----
221
222 def exit_code( val ):
223 return lambda name, opts, v=val: _exit_code(name, opts, v);
224
225 def _exit_code( name, opts, v ):
226 opts.exit_code = v
227
228 # -----
229
230 def timeout_multiplier( val ):
231 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232
233 def _timeout_multiplier( name, opts, v ):
234 opts.timeout_multiplier = v
235
236 # -----
237
238 def extra_run_opts( val ):
239 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240
241 def _extra_run_opts( name, opts, v ):
242 opts.extra_run_opts = v
243
244 # -----
245
246 def extra_hc_opts( val ):
247 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
248
249 def _extra_hc_opts( name, opts, v ):
250 opts.extra_hc_opts = v
251
252 # -----
253
254 def extra_clean( files ):
255 return lambda name, opts, v=files: _extra_clean(name, opts, v);
256
257 def _extra_clean( name, opts, v ):
258 opts.clean_files = v
259
260 # -----
261
262 def stats_num_field( field, expecteds ):
263 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
264
265 def _stats_num_field( name, opts, field, expecteds ):
266 if field in opts.stats_range_fields:
267 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
268
269 if type(expecteds) is types.ListType:
270 for (b, expected, dev) in expecteds:
271 if b:
272 opts.stats_range_fields[field] = (expected, dev)
273 return
274 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275
276 else:
277 (expected, dev) = expecteds
278 opts.stats_range_fields[field] = (expected, dev)
279
280 def compiler_stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282
283 def _compiler_stats_num_field( name, opts, field, expecteds ):
284 if field in opts.compiler_stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 for (b, expected, dev) in expecteds:
288 if b:
289 opts.compiler_stats_range_fields[field] = (expected, dev)
290 return
291
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 # -----
295
296 def skip_if_no_ghci(name, opts):
297 if not ('ghci' in config.run_ways):
298 opts.skip = 1
299
300 # ----
301
302 def skip_if_fast(name, opts):
303 if config.fast:
304 opts.skip = 1
305
306 # -----
307
308 def when(b, f):
309 # When list_brokens is on, we want to see all expect_broken calls,
310 # so we always do f
311 if b or config.list_broken:
312 return f
313 else:
314 return normal
315
316 def unless(b, f):
317 return when(not b, f)
318
319 def platform( plat ):
320 return config.platform == plat
321
322 def opsys( os ):
323 return config.os == os
324
325 def arch( arch ):
326 return config.arch == arch
327
328 def wordsize( ws ):
329 return config.wordsize == str(ws)
330
331 def unregisterised( ):
332 return config.unregisterised
333
334 def msys( ):
335 return config.msys
336
337 def cygwin( ):
338 return config.cygwin
339
340 def have_vanilla( ):
341 return config.have_vanilla
342
343 def have_dynamic( ):
344 return config.have_dynamic
345
346 def have_profiling( ):
347 return config.have_profiling
348
349 # ---
350
351 def ghci_dynamic( ):
352 return config.ghc_dynamic_by_default
353
354 def in_tree_compiler( ):
355 return config.in_tree_compiler
356
357 def compiler_type( compiler ):
358 return config.compiler_type == compiler
359
360 def compiler_profiled( ):
361 return config.compiler_profiled
362
363 def compiler_lt( compiler, version ):
364 return config.compiler_type == compiler and \
365 version_lt(config.compiler_version, version)
366
367 def compiler_le( compiler, version ):
368 return config.compiler_type == compiler and \
369 version_le(config.compiler_version, version)
370
371 def compiler_gt( compiler, version ):
372 return config.compiler_type == compiler and \
373 version_gt(config.compiler_version, version)
374
375 def compiler_ge( compiler, version ):
376 return config.compiler_type == compiler and \
377 version_ge(config.compiler_version, version)
378
379 def compiler_debugged( ):
380 return config.compiler_debugged
381
382 def namebase( nb ):
383 return lambda opts, nb=nb: _namebase(opts, nb)
384
385 def _namebase( opts, nb ):
386 opts.with_namebase = nb
387
388 # ---
389
390 def tag( t ):
391 return t in config.compiler_tags
392
393 # ---
394 def high_memory_usage(name, opts):
395 opts.alone = True
396
397 # ---
398 def literate( name, opts ):
399 opts.literate = 1;
400
401 def c_src( name, opts ):
402 opts.c_src = 1;
403
404 def objc_src( name, opts ):
405 opts.objc_src = 1;
406
407 def objcpp_src( name, opts ):
408 opts.objcpp_src = 1;
409
410 def cmm_src( name, opts ):
411 opts.cmm_src = 1;
412
413 def outputdir( odir ):
414 return lambda name, opts, d=odir: _outputdir(name, opts, d)
415
416 def _outputdir( name, opts, odir ):
417 opts.outputdir = odir;
418
419 # ----
420
421 def pre_cmd( cmd ):
422 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
423
424 def _pre_cmd( name, opts, cmd ):
425 opts.pre_cmd = cmd
426
427 # ----
428
429 def clean_cmd( cmd ):
430 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
431
432 def _clean_cmd( name, opts, cmd ):
433 opts.clean_cmd = cmd
434
435 # ----
436
437 def cmd_prefix( prefix ):
438 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
439
440 def _cmd_prefix( name, opts, prefix ):
441 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
442
443 # ----
444
445 def cmd_wrapper( fun ):
446 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
447
448 def _cmd_wrapper( name, opts, fun ):
449 opts.cmd_wrapper = fun
450
451 # ----
452
453 def compile_cmd_prefix( prefix ):
454 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
455
456 def _compile_cmd_prefix( name, opts, prefix ):
457 opts.compile_cmd_prefix = prefix
458
459 # ----
460
461 def normalise_slashes( name, opts ):
462 opts.extra_normaliser = normalise_slashes_
463
464 def normalise_exe( name, opts ):
465 opts.extra_normaliser = normalise_exe_
466
467 def normalise_fun( fun ):
468 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
469
470 def _normalise_fun( name, opts, f ):
471 opts.extra_normaliser = f
472
473 def normalise_errmsg_fun( fun ):
474 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
475
476 def _normalise_errmsg_fun( name, opts, f ):
477 opts.extra_errmsg_normaliser = f
478
479 def two_normalisers(f, g):
480 return lambda x, f=f, g=g: f(g(x))
481
482 # ----
483 # Function for composing two opt-fns together
484
485 def composes( fs ):
486 return reduce(lambda f, g: compose(f, g), fs)
487
488 def compose( f, g ):
489 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
490
491 def _compose( name, opts, f, g ):
492 f(name, opts)
493 g(name, opts)
494
495 # -----------------------------------------------------------------------------
496 # The current directory of tests
497
498 def newTestDir( dir ):
499 global thisdir_settings
500 # reset the options for this test directory
501 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
502
503 def _newTestDir( name, opts, dir ):
504 opts.testdir = dir
505 opts.compiler_always_flags = config.compiler_always_flags
506
507 # -----------------------------------------------------------------------------
508 # Actually doing tests
509
510 parallelTests = []
511 aloneTests = []
512 allTestNames = set([])
513
514 def runTest (opts, name, func, args):
515 ok = 0
516
517 if config.use_threads:
518 t.thread_pool.acquire()
519 try:
520 while config.threads<(t.running_threads+1):
521 t.thread_pool.wait()
522 t.running_threads = t.running_threads+1
523 ok=1
524 t.thread_pool.release()
525 thread.start_new_thread(test_common_thread, (name, opts, func, args))
526 except:
527 if not ok:
528 t.thread_pool.release()
529 else:
530 test_common_work (name, opts, func, args)
531
532 # name :: String
533 # setup :: TestOpts -> IO ()
534 def test (name, setup, func, args):
535 global aloneTests
536 global parallelTests
537 global allTestNames
538 global thisdir_settings
539 if name in allTestNames:
540 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
541 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
542 framework_fail(name, 'bad_name', 'This test has an invalid name')
543
544 # Make a deep copy of the default_testopts, as we need our own copy
545 # of any dictionaries etc inside it. Otherwise, if one test modifies
546 # them, all tests will see the modified version!
547 myTestOpts = copy.deepcopy(default_testopts)
548
549 if type(setup) is types.ListType:
550 setup = composes(setup)
551
552 setup = compose(thisdir_settings, setup)
553 setup(name, myTestOpts)
554
555 thisTest = lambda : runTest(myTestOpts, name, func, args)
556 if myTestOpts.alone:
557 aloneTests.append(thisTest)
558 else:
559 parallelTests.append(thisTest)
560 allTestNames.add(name)
561
562 if config.use_threads:
563 def test_common_thread(name, opts, func, args):
564 t.lock.acquire()
565 try:
566 test_common_work(name,opts,func,args)
567 finally:
568 t.lock.release()
569 t.thread_pool.acquire()
570 t.running_threads = t.running_threads - 1
571 t.thread_pool.notify()
572 t.thread_pool.release()
573
574 def get_package_cache_timestamp():
575 if config.package_conf_cache_file == '':
576 return 0.0
577 else:
578 try:
579 return os.stat(config.package_conf_cache_file).st_mtime
580 except:
581 return 0.0
582
583
584 def test_common_work (name, opts, func, args):
585 try:
586 t.total_tests = t.total_tests+1
587 setLocalTestOpts(opts)
588
589 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
590
591 # All the ways we might run this test
592 if func == compile or func == multimod_compile:
593 all_ways = config.compile_ways
594 elif func == compile_and_run or func == multimod_compile_and_run:
595 all_ways = config.run_ways
596 elif func == ghci_script:
597 if 'ghci' in config.run_ways:
598 all_ways = ['ghci']
599 else:
600 all_ways = []
601 else:
602 all_ways = ['normal']
603
604 # A test itself can request extra ways by setting opts.extra_ways
605 all_ways = all_ways + filter(lambda way: way not in all_ways,
606 opts.extra_ways)
607
608 t.total_test_cases = t.total_test_cases + len(all_ways)
609
610 ok_way = lambda way: \
611 not getTestOpts().skip \
612 and (config.only == [] or name in config.only) \
613 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
614 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
615 and way not in getTestOpts().omit_ways
616
617 # Which ways we are asked to skip
618 do_ways = filter (ok_way,all_ways)
619
620 # In fast mode, we skip all but one way
621 if config.fast and len(do_ways) > 0:
622 do_ways = [do_ways[0]]
623
624 if not config.clean_only:
625 # Run the required tests...
626 for way in do_ways:
627 if stopping():
628 break
629 do_test (name, way, func, args)
630
631 for way in all_ways:
632 if way not in do_ways:
633 skiptest (name,way)
634
635 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
636 clean(map (lambda suff: name + suff,
637 ['', '.exe', '.exe.manifest', '.genscript',
638 '.stderr.normalised', '.stdout.normalised',
639 '.run.stderr', '.run.stdout',
640 '.run.stderr.normalised', '.run.stdout.normalised',
641 '.comp.stderr', '.comp.stdout',
642 '.comp.stderr.normalised', '.comp.stdout.normalised',
643 '.interp.stderr', '.interp.stdout',
644 '.interp.stderr.normalised', '.interp.stdout.normalised',
645 '.stats', '.comp.stats',
646 '.hi', '.o', '.prof', '.exe.prof', '.hc',
647 '_stub.h', '_stub.c', '_stub.o',
648 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
649
650 if func == multi_compile or func == multi_compile_fail:
651 extra_mods = args[1]
652 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
653 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
654
655 clean(getTestOpts().clean_files)
656
657 if getTestOpts().outputdir != None:
658 odir = in_testdir(getTestOpts().outputdir)
659 try:
660 shutil.rmtree(odir)
661 except:
662 pass
663
664 try:
665 shutil.rmtree(in_testdir('.hpc.' + name))
666 except:
667 pass
668
669 try:
670 cleanCmd = getTestOpts().clean_cmd
671 if cleanCmd != None:
672 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
673 if result != 0:
674 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
675 except e:
676 framework_fail(name, 'cleaning', 'clean-command exception')
677
678 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
679
680 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
681 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
682
683 try:
684 for f in files_written[name]:
685 if os.path.exists(f):
686 try:
687 if not f in files_written_not_removed[name]:
688 files_written_not_removed[name].append(f)
689 except:
690 files_written_not_removed[name] = [f]
691 except:
692 pass
693 except Exception, e:
694 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
695
696 def clean(strs):
697 for str in strs:
698 for name in glob.glob(in_testdir(str)):
699 clean_full_path(name)
700
701 def clean_full_path(name):
702 try:
703 # Remove files...
704 os.remove(name)
705 except OSError, e1:
706 try:
707 # ... and empty directories
708 os.rmdir(name)
709 except OSError, e2:
710 # We don't want to fail here, but we do want to know
711 # what went wrong, so print out the exceptions.
712 # ENOENT isn't a problem, though, as we clean files
713 # that don't necessarily exist.
714 if e1.errno != errno.ENOENT:
715 print e1
716 if e2.errno != errno.ENOENT:
717 print e2
718
719 def do_test(name, way, func, args):
720 full_name = name + '(' + way + ')'
721
722 try:
723 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
724 str([t.n_unexpected_passes, \
725 t.n_unexpected_failures, \
726 t.n_framework_failures])
727
728 if config.use_threads:
729 t.lock.release()
730
731 try:
732 preCmd = getTestOpts().pre_cmd
733 if preCmd != None:
734 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
735 if result != 0:
736 framework_fail(name, way, 'pre-command failed: ' + str(result))
737 except e:
738 framework_fail(name, way, 'pre-command exception')
739
740 try:
741 result = apply(func, [name,way] + args)
742 finally:
743 if config.use_threads:
744 t.lock.acquire()
745
746 if getTestOpts().expect != 'pass' and \
747 getTestOpts().expect != 'fail' and \
748 getTestOpts().expect != 'missing-lib':
749 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
750
751 try:
752 passFail = result['passFail']
753 except:
754 passFail = 'No passFail found'
755
756 if passFail == 'pass':
757 if getTestOpts().expect == 'pass' \
758 and way not in getTestOpts().expect_fail_for:
759 t.n_expected_passes = t.n_expected_passes + 1
760 if name in t.expected_passes:
761 t.expected_passes[name].append(way)
762 else:
763 t.expected_passes[name] = [way]
764 else:
765 print '*** unexpected pass for', full_name
766 t.n_unexpected_passes = t.n_unexpected_passes + 1
767 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
768 elif passFail == 'fail':
769 if getTestOpts().expect == 'pass' \
770 and way not in getTestOpts().expect_fail_for:
771 print '*** unexpected failure for', full_name
772 t.n_unexpected_failures = t.n_unexpected_failures + 1
773 reason = result['reason']
774 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
775 else:
776 if getTestOpts().expect == 'missing-lib':
777 t.n_missing_libs = t.n_missing_libs + 1
778 if name in t.missing_libs:
779 t.missing_libs[name].append(way)
780 else:
781 t.missing_libs[name] = [way]
782 else:
783 t.n_expected_failures = t.n_expected_failures + 1
784 if name in t.expected_failures:
785 t.expected_failures[name].append(way)
786 else:
787 t.expected_failures[name] = [way]
788 else:
789 framework_fail(name, way, 'bad result ' + passFail)
790 except KeyboardInterrupt:
791 stopNow()
792 except:
793 framework_fail(name, way, 'do_test exception')
794 traceback.print_exc()
795
796 def addPassingTestInfo (testInfos, directory, name, way):
797 directory = re.sub('^\\.[/\\\\]', '', directory)
798
799 if not directory in testInfos:
800 testInfos[directory] = {}
801
802 if not name in testInfos[directory]:
803 testInfos[directory][name] = []
804
805 testInfos[directory][name].append(way)
806
807 def addFailingTestInfo (testInfos, directory, name, reason, way):
808 directory = re.sub('^\\.[/\\\\]', '', directory)
809
810 if not directory in testInfos:
811 testInfos[directory] = {}
812
813 if not name in testInfos[directory]:
814 testInfos[directory][name] = {}
815
816 if not reason in testInfos[directory][name]:
817 testInfos[directory][name][reason] = []
818
819 testInfos[directory][name][reason].append(way)
820
821 def skiptest (name, way):
822 # print 'Skipping test \"', name, '\"'
823 t.n_tests_skipped = t.n_tests_skipped + 1
824 if name in t.tests_skipped:
825 t.tests_skipped[name].append(way)
826 else:
827 t.tests_skipped[name] = [way]
828
829 def framework_fail( name, way, reason ):
830 full_name = name + '(' + way + ')'
831 print '*** framework failure for', full_name, reason
832 t.n_framework_failures = t.n_framework_failures + 1
833 if name in t.framework_failures:
834 t.framework_failures[name].append(way)
835 else:
836 t.framework_failures[name] = [way]
837
838 def badResult(result):
839 try:
840 if result['passFail'] == 'pass':
841 return False
842 return True
843 except:
844 return True
845
846 def passed():
847 return {'passFail': 'pass'}
848
849 def failBecause(reason):
850 return {'passFail': 'fail', 'reason': reason}
851
852 # -----------------------------------------------------------------------------
853 # Generic command tests
854
855 # A generic command test is expected to run and exit successfully.
856 #
857 # The expected exit code can be changed via exit_code() as normal, and
858 # the expected stdout/stderr are stored in <testname>.stdout and
859 # <testname>.stderr. The output of the command can be ignored
860 # altogether by using run_command_ignore_output instead of
861 # run_command.
862
863 def run_command( name, way, cmd ):
864 return simple_run( name, '', cmd, '' )
865
866 # -----------------------------------------------------------------------------
867 # GHCi tests
868
869 def ghci_script( name, way, script ):
870 # filter out -fforce-recomp from compiler_always_flags, because we're
871 # actually testing the recompilation behaviour in the GHCi tests.
872 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
873 flags.append(getTestOpts().extra_hc_opts)
874 if getTestOpts().outputdir != None:
875 flags.extend(["-outputdir", getTestOpts().outputdir])
876
877 # We pass HC and HC_OPTS as environment variables, so that the
878 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
879 cmd = "HC='" + config.compiler + "' " + \
880 "HC_OPTS='" + join(flags,' ') + "' " + \
881 "'" + config.compiler + "'" + \
882 ' --interactive -v0 -ignore-dot-ghci ' + \
883 join(flags,' ')
884
885 getTestOpts().stdin = script
886 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
887
888 # -----------------------------------------------------------------------------
889 # Compile-only tests
890
891 def compile( name, way, extra_hc_opts ):
892 return do_compile( name, way, 0, '', [], extra_hc_opts )
893
894 def compile_fail( name, way, extra_hc_opts ):
895 return do_compile( name, way, 1, '', [], extra_hc_opts )
896
897 def multimod_compile( name, way, top_mod, extra_hc_opts ):
898 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
899
900 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
901 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
902
903 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
904 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
905
906 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
907 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
908
909 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
910 # print 'Compile only, extra args = ', extra_hc_opts
911 pretest_cleanup(name)
912
913 result = extras_build( way, extra_mods, extra_hc_opts )
914 if badResult(result):
915 return result
916 extra_hc_opts = result['hc_opts']
917
918 force = 0
919 if extra_mods:
920 force = 1
921 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
922
923 if badResult(result):
924 return result
925
926 # the actual stderr should always match the expected, regardless
927 # of whether we expected the compilation to fail or not (successful
928 # compilations may generate warnings).
929
930 if getTestOpts().with_namebase == None:
931 namebase = name
932 else:
933 namebase = getTestOpts().with_namebase
934
935 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
936 actual_stderr_file = qualify(name, 'comp.stderr')
937
938 if not compare_outputs('stderr', \
939 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
940 expected_stderr_file, actual_stderr_file):
941 return failBecause('stderr mismatch')
942
943 # no problems found, this test passed
944 return passed()
945
946 def compile_cmp_asm( name, way, extra_hc_opts ):
947 print 'Compile only, extra args = ', extra_hc_opts
948 pretest_cleanup(name)
949 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
950
951 if badResult(result):
952 return result
953
954 # the actual stderr should always match the expected, regardless
955 # of whether we expected the compilation to fail or not (successful
956 # compilations may generate warnings).
957
958 if getTestOpts().with_namebase == None:
959 namebase = name
960 else:
961 namebase = getTestOpts().with_namebase
962
963 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
964 actual_asm_file = qualify(name, 's')
965
966 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
967 expected_asm_file, actual_asm_file):
968 return failBecause('asm mismatch')
969
970 # no problems found, this test passed
971 return passed()
972
973 # -----------------------------------------------------------------------------
974 # Compile-and-run tests
975
976 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
977 # print 'Compile and run, extra args = ', extra_hc_opts
978 pretest_cleanup(name)
979
980 result = extras_build( way, extra_mods, extra_hc_opts )
981 if badResult(result):
982 return result
983 extra_hc_opts = result['hc_opts']
984
985 if way == 'ghci': # interpreted...
986 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
987 elif way == 'extcore' or way == 'optextcore' :
988 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
989 else: # compiled...
990 force = 0
991 if extra_mods:
992 force = 1
993
994 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
995 if badResult(result):
996 return result
997
998 cmd = './' + name;
999
1000 # we don't check the compiler's stderr for a compile-and-run test
1001 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1002
1003 def compile_and_run( name, way, extra_hc_opts ):
1004 return compile_and_run__( name, way, '', [], extra_hc_opts)
1005
1006 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1007 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1008
1009 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1010 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1011
1012 def stats( name, way, stats_file ):
1013 opts = getTestOpts()
1014 return checkStats(stats_file, opts.stats_range_fields)
1015
1016 # -----------------------------------------------------------------------------
1017 # Check -t stats info
1018
1019 def checkStats(stats_file, range_fields):
1020 result = passed()
1021 if len(range_fields) > 0:
1022 f = open(in_testdir(stats_file))
1023 contents = f.read()
1024 f.close()
1025
1026 for (field, (expected, dev)) in range_fields.items():
1027 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1028 if m == None:
1029 print 'Failed to find field: ', field
1030 result = failBecause('no such stats field')
1031 val = int(m.group(1))
1032
1033 lowerBound = trunc( expected * ((100 - float(dev))/100));
1034 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1035
1036 if val < lowerBound:
1037 print field, 'value is too low:'
1038 print '(If this is because you have improved GHC, please'
1039 print 'update the test so that GHC doesn\'t regress again)'
1040 result = failBecause('stat too good')
1041 if val > upperBound:
1042 print field, 'value is too high:'
1043 result = failBecause('stat not good enough')
1044
1045 if val < lowerBound or val > upperBound:
1046 valStr = str(val)
1047 valLen = len(valStr)
1048 expectedStr = str(expected)
1049 expectedLen = len(expectedStr)
1050 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1051 def display(descr, val, extra):
1052 print descr, string.rjust(str(val), length), extra
1053 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1054 display(' Lower bound ' + field + ':', lowerBound, '')
1055 display(' Upper bound ' + field + ':', upperBound, '')
1056 display(' Actual ' + field + ':', val, '')
1057
1058 return result
1059
1060 # -----------------------------------------------------------------------------
1061 # Build a single-module program
1062
1063 def extras_build( way, extra_mods, extra_hc_opts ):
1064 for modopts in extra_mods:
1065 mod, opts = modopts
1066 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1067 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1068 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1069 if badResult(result):
1070 return result
1071
1072 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1073
1074
1075 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1076 opts = getTestOpts()
1077 errname = add_suffix(name, 'comp.stderr')
1078 rm_no_fail( qualify(errname, '') )
1079
1080 if top_mod != '':
1081 srcname = top_mod
1082 rm_no_fail( qualify(name, '') )
1083 base, suf = os.path.splitext(top_mod)
1084 rm_no_fail( qualify(base, '') )
1085 rm_no_fail( qualify(base, 'exe') )
1086 elif addsuf:
1087 srcname = add_hs_lhs_suffix(name)
1088 rm_no_fail( qualify(name, '') )
1089 else:
1090 srcname = name
1091 rm_no_fail( qualify(name, 'o') )
1092
1093 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1094
1095 to_do = ''
1096 if top_mod != '':
1097 to_do = '--make '
1098 if link:
1099 to_do = to_do + '-o ' + name
1100 elif link:
1101 to_do = '-o ' + name
1102 elif opts.compile_to_hc:
1103 to_do = '-C'
1104 else:
1105 to_do = '-c' # just compile
1106
1107 stats_file = name + '.comp.stats'
1108 if len(opts.compiler_stats_range_fields) > 0:
1109 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1110
1111 # Required by GHC 7.3+, harmless for earlier versions:
1112 if (getTestOpts().c_src or
1113 getTestOpts().objc_src or
1114 getTestOpts().objcpp_src or
1115 getTestOpts().cmm_src):
1116 extra_hc_opts += ' -no-hs-main '
1117
1118 if getTestOpts().compile_cmd_prefix == '':
1119 cmd_prefix = ''
1120 else:
1121 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1122
1123 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1124 if noforce:
1125 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1126 if getTestOpts().outputdir != None:
1127 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1128
1129 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1130 + config.compiler + "' " \
1131 + join(comp_flags,' ') + ' ' \
1132 + to_do + ' ' + srcname + ' ' \
1133 + join(config.way_flags(name)[way],' ') + ' ' \
1134 + extra_hc_opts + ' ' \
1135 + opts.extra_hc_opts + ' ' \
1136 + '>' + errname + ' 2>&1'
1137
1138 result = runCmdFor(name, cmd)
1139
1140 if result != 0 and not should_fail:
1141 actual_stderr = qualify(name, 'comp.stderr')
1142 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1143 if_verbose_dump(1,actual_stderr)
1144
1145 # ToDo: if the sub-shell was killed by ^C, then exit
1146
1147 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1148
1149 if badResult(statsResult):
1150 return statsResult
1151
1152 if should_fail:
1153 if result == 0:
1154 return failBecause('exit code 0')
1155 else:
1156 if result != 0:
1157 return failBecause('exit code non-0')
1158
1159 return passed()
1160
1161 # -----------------------------------------------------------------------------
1162 # Run a program and check its output
1163 #
1164 # If testname.stdin exists, route input from that, else
1165 # from /dev/null. Route output to testname.run.stdout and
1166 # testname.run.stderr. Returns the exit code of the run.
1167
1168 def simple_run( name, way, prog, args ):
1169 opts = getTestOpts()
1170
1171 # figure out what to use for stdin
1172 if opts.stdin != '':
1173 use_stdin = opts.stdin
1174 else:
1175 stdin_file = add_suffix(name, 'stdin')
1176 if os.path.exists(in_testdir(stdin_file)):
1177 use_stdin = stdin_file
1178 else:
1179 use_stdin = '/dev/null'
1180
1181 run_stdout = add_suffix(name,'run.stdout')
1182 run_stderr = add_suffix(name,'run.stderr')
1183
1184 rm_no_fail(qualify(name,'run.stdout'))
1185 rm_no_fail(qualify(name,'run.stderr'))
1186 rm_no_fail(qualify(name, 'hp'))
1187 rm_no_fail(qualify(name,'ps'))
1188 rm_no_fail(qualify(name, 'prof'))
1189
1190 my_rts_flags = rts_flags(way)
1191
1192 stats_file = name + '.stats'
1193 if len(opts.stats_range_fields) > 0:
1194 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1195
1196 if opts.no_stdin:
1197 stdin_comes_from = ''
1198 else:
1199 stdin_comes_from = ' <' + use_stdin
1200
1201 if opts.combined_output:
1202 redirection = ' >' + run_stdout \
1203 + ' 2>&1'
1204 else:
1205 redirection = ' >' + run_stdout \
1206 + ' 2>' + run_stderr
1207
1208 cmd = prog + ' ' + args + ' ' \
1209 + my_rts_flags + ' ' \
1210 + stdin_comes_from \
1211 + redirection
1212
1213 if opts.cmd_wrapper != None:
1214 cmd = opts.cmd_wrapper(cmd);
1215
1216 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1217
1218 # run the command
1219 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1220
1221 exit_code = result >> 8
1222 signal = result & 0xff
1223
1224 # check the exit code
1225 if exit_code != opts.exit_code:
1226 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1227 dump_stdout(name)
1228 dump_stderr(name)
1229 return failBecause('bad exit code')
1230
1231 check_hp = my_rts_flags.find("-h") != -1
1232 check_prof = my_rts_flags.find("-p") != -1
1233
1234 if not opts.ignore_output:
1235 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1236 bad_stdout = not check_stdout_ok(name)
1237 if bad_stderr:
1238 return failBecause('bad stderr')
1239 if bad_stdout:
1240 return failBecause('bad stdout')
1241 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1242 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1243 return failBecause('bad heap profile')
1244 if check_prof and not check_prof_ok(name):
1245 return failBecause('bad profile')
1246
1247 return checkStats(stats_file, opts.stats_range_fields)
1248
1249 def rts_flags(way):
1250 if (way == ''):
1251 return ''
1252 else:
1253 args = config.way_rts_flags[way]
1254
1255 if args == []:
1256 return ''
1257 else:
1258 return '+RTS ' + join(args,' ') + ' -RTS'
1259
1260 # -----------------------------------------------------------------------------
1261 # Run a program in the interpreter and check its output
1262
1263 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1264 outname = add_suffix(name, 'interp.stdout')
1265 errname = add_suffix(name, 'interp.stderr')
1266 rm_no_fail(outname)
1267 rm_no_fail(errname)
1268 rm_no_fail(name)
1269
1270 if (top_mod == ''):
1271 srcname = add_hs_lhs_suffix(name)
1272 else:
1273 srcname = top_mod
1274
1275 scriptname = add_suffix(name, 'genscript')
1276 qscriptname = in_testdir(scriptname)
1277 rm_no_fail(qscriptname)
1278
1279 delimiter = '===== program output begins here\n'
1280
1281 script = open(qscriptname, 'w')
1282 if not compile_only:
1283 # set the prog name and command-line args to match the compiled
1284 # environment.
1285 script.write(':set prog ' + name + '\n')
1286 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1287 # Add marker lines to the stdout and stderr output files, so we
1288 # can separate GHCi's output from the program's.
1289 script.write(':! echo ' + delimiter)
1290 script.write(':! echo 1>&2 ' + delimiter)
1291 # Set stdout to be line-buffered to match the compiled environment.
1292 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1293 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1294 # in the event of an exception as for the compiled program.
1295 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1296 script.close()
1297
1298 # figure out what to use for stdin
1299 if getTestOpts().stdin != '':
1300 stdin_file = in_testdir(getTestOpts().stdin)
1301 else:
1302 stdin_file = qualify(name, 'stdin')
1303
1304 if os.path.exists(stdin_file):
1305 stdin = open(stdin_file, 'r')
1306 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1307
1308 script.close()
1309
1310 flags = copy.copy(getTestOpts().compiler_always_flags)
1311 if getTestOpts().outputdir != None:
1312 flags.extend(["-outputdir", getTestOpts().outputdir])
1313
1314 cmd = "'" + config.compiler + "' " \
1315 + join(flags,' ') + ' ' \
1316 + srcname + ' ' \
1317 + join(config.way_flags(name)[way],' ') + ' ' \
1318 + extra_hc_opts + ' ' \
1319 + getTestOpts().extra_hc_opts + ' ' \
1320 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1321
1322 if getTestOpts().cmd_wrapper != None:
1323 cmd = getTestOpts().cmd_wrapper(cmd);
1324
1325 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1326
1327 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1328
1329 exit_code = result >> 8
1330 signal = result & 0xff
1331
1332 # split the stdout into compilation/program output
1333 split_file(in_testdir(outname), delimiter,
1334 qualify(name, 'comp.stdout'),
1335 qualify(name, 'run.stdout'))
1336 split_file(in_testdir(errname), delimiter,
1337 qualify(name, 'comp.stderr'),
1338 qualify(name, 'run.stderr'))
1339
1340 # check the exit code
1341 if exit_code != getTestOpts().exit_code:
1342 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1343 dump_stdout(name)
1344 dump_stderr(name)
1345 return failBecause('bad exit code')
1346
1347 # ToDo: if the sub-shell was killed by ^C, then exit
1348
1349 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1350 check_stdout_ok(name)):
1351 return passed()
1352 else:
1353 return failBecause('bad stdout or stderr')
1354
1355
1356 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1357 infile = open(in_fn)
1358 out1 = open(out1_fn, 'w')
1359 out2 = open(out2_fn, 'w')
1360
1361 line = infile.readline()
1362 line = re.sub('\r', '', line) # ignore Windows EOL
1363 while (re.sub('^\s*','',line) != delimiter and line != ''):
1364 out1.write(line)
1365 line = infile.readline()
1366 line = re.sub('\r', '', line)
1367 out1.close()
1368
1369 line = infile.readline()
1370 while (line != ''):
1371 out2.write(line)
1372 line = infile.readline()
1373 out2.close()
1374
1375 # -----------------------------------------------------------------------------
1376 # Generate External Core for the given program, then compile the resulting Core
1377 # and compare its output to the expected output
1378
1379 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1380
1381 depsfilename = qualify(name, 'deps')
1382 errname = add_suffix(name, 'comp.stderr')
1383 qerrname = qualify(errname,'')
1384
1385 hcname = qualify(name, 'hc')
1386 oname = qualify(name, 'o')
1387
1388 rm_no_fail( qerrname )
1389 rm_no_fail( qualify(name, '') )
1390
1391 if (top_mod == ''):
1392 srcname = add_hs_lhs_suffix(name)
1393 else:
1394 srcname = top_mod
1395
1396 qcorefilename = qualify(name, 'hcr')
1397 corefilename = add_suffix(name, 'hcr')
1398 rm_no_fail(qcorefilename)
1399
1400 # Generate External Core
1401
1402 if (top_mod == ''):
1403 to_do = ' ' + srcname + ' '
1404 else:
1405 to_do = ' --make ' + top_mod + ' '
1406
1407 flags = copy.copy(getTestOpts().compiler_always_flags)
1408 if getTestOpts().outputdir != None:
1409 flags.extend(["-outputdir", getTestOpts().outputdir])
1410 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1411 + config.compiler + "' " \
1412 + join(flags,' ') + ' ' \
1413 + join(config.way_flags(name)[way],' ') + ' ' \
1414 + extra_hc_opts + ' ' \
1415 + getTestOpts().extra_hc_opts \
1416 + to_do \
1417 + '>' + errname + ' 2>&1'
1418 result = runCmdFor(name, cmd)
1419
1420 exit_code = result >> 8
1421
1422 if exit_code != 0:
1423 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1424 if_verbose_dump(1,qerrname)
1425 return failBecause('ext core exit code non-0')
1426
1427 # Compile the resulting files -- if there's more than one module, we need to read the output
1428 # of the previous compilation in order to find the dependencies
1429 if (top_mod == ''):
1430 to_compile = corefilename
1431 else:
1432 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1433 deps = open(depsfilename).read()
1434 deplist = string.replace(deps, '\n',' ');
1435 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1436 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1437
1438 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1439 if getTestOpts().outputdir != None:
1440 flags.extend(["-outputdir", getTestOpts().outputdir])
1441
1442 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1443 + config.compiler + "' " \
1444 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1445 + to_compile + ' ' \
1446 + extra_hc_opts + ' ' \
1447 + getTestOpts().extra_hc_opts + ' ' \
1448 + flags \
1449 + ' -fglasgow-exts -o ' + name \
1450 + '>' + errname + ' 2>&1'
1451
1452 result = runCmdFor(name, cmd)
1453 exit_code = result >> 8
1454
1455 if exit_code != 0:
1456 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1457 if_verbose_dump(1,qerrname)
1458 return failBecause('ext core exit code non-0')
1459
1460 # Clean up
1461 rm_no_fail ( oname )
1462 rm_no_fail ( hcname )
1463 rm_no_fail ( qcorefilename )
1464 rm_no_fail ( depsfilename )
1465
1466 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1467
1468 # -----------------------------------------------------------------------------
1469 # Utils
1470
1471 def check_stdout_ok( name ):
1472 if getTestOpts().with_namebase == None:
1473 namebase = name
1474 else:
1475 namebase = getTestOpts().with_namebase
1476
1477 actual_stdout_file = qualify(name, 'run.stdout')
1478 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1479
1480 def norm(str):
1481 if platform_specific:
1482 return str
1483 else:
1484 return normalise_output(str)
1485
1486 return compare_outputs('stdout', \
1487 two_normalisers(norm, getTestOpts().extra_normaliser), \
1488 expected_stdout_file, actual_stdout_file)
1489
1490 def dump_stdout( name ):
1491 print 'Stdout:'
1492 print read_no_crs(qualify(name, 'run.stdout'))
1493
1494 def check_stderr_ok( name ):
1495 if getTestOpts().with_namebase == None:
1496 namebase = name
1497 else:
1498 namebase = getTestOpts().with_namebase
1499
1500 actual_stderr_file = qualify(name, 'run.stderr')
1501 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1502
1503 def norm(str):
1504 if platform_specific:
1505 return str
1506 else:
1507 return normalise_errmsg(str)
1508
1509 return compare_outputs('stderr', \
1510 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1511 expected_stderr_file, actual_stderr_file)
1512
1513 def dump_stderr( name ):
1514 print "Stderr:"
1515 print read_no_crs(qualify(name, 'run.stderr'))
1516
1517 def read_no_crs(file):
1518 str = ''
1519 try:
1520 h = open(file)
1521 str = h.read()
1522 h.close
1523 except:
1524 # On Windows, if the program fails very early, it seems the
1525 # files stdout/stderr are redirected to may not get created
1526 pass
1527 return re.sub('\r', '', str)
1528
1529 def write_file(file, str):
1530 h = open(file, 'w')
1531 h.write(str)
1532 h.close
1533
1534 def check_hp_ok(name):
1535
1536 # do not qualify for hp2ps because we should be in the right directory
1537 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1538
1539 hp2psResult = runCmdExitCode(hp2psCmd)
1540
1541 actual_ps_file = qualify(name, 'ps')
1542
1543 if(hp2psResult == 0):
1544 if (os.path.exists(actual_ps_file)):
1545 if gs_working:
1546 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1547 if (gsResult == 0):
1548 return (True)
1549 else:
1550 print "hp2ps output for " + name + "is not valid PostScript"
1551 else: return (True) # assume postscript is valid without ghostscript
1552 else:
1553 print "hp2ps did not generate PostScript for " + name
1554 return (False)
1555 else:
1556 print "hp2ps error when processing heap profile for " + name
1557 return(False)
1558
1559 def check_prof_ok(name):
1560
1561 prof_file = qualify(name,'prof')
1562
1563 if not os.path.exists(prof_file):
1564 print prof_file + " does not exist"
1565 return(False)
1566
1567 if os.path.getsize(qualify(name,'prof')) == 0:
1568 print prof_file + " is empty"
1569 return(False)
1570
1571 if getTestOpts().with_namebase == None:
1572 namebase = name
1573 else:
1574 namebase = getTestOpts().with_namebase
1575
1576 (platform_specific, expected_prof_file) = \
1577 platform_wordsize_qualify(namebase, 'prof.sample')
1578
1579 # sample prof file is not required
1580 if not os.path.exists(expected_prof_file):
1581 return True
1582 else:
1583 return compare_outputs('prof', \
1584 two_normalisers(normalise_whitespace,normalise_prof), \
1585 expected_prof_file, prof_file)
1586
1587 # Compare expected output to actual output, and optionally accept the
1588 # new output. Returns true if output matched or was accepted, false
1589 # otherwise.
1590 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1591 if os.path.exists(expected_file):
1592 expected_raw = read_no_crs(expected_file)
1593 # print "norm:", normaliser(expected_raw)
1594 expected_str = normaliser(expected_raw)
1595 expected_file_for_diff = expected_file
1596 else:
1597 expected_str = ''
1598 expected_file_for_diff = '/dev/null'
1599
1600 actual_raw = read_no_crs(actual_file)
1601 actual_str = normaliser(actual_raw)
1602
1603 if expected_str == actual_str:
1604 return 1
1605 else:
1606 print 'Actual ' + kind + ' output differs from expected:'
1607
1608 if expected_file_for_diff == '/dev/null':
1609 expected_normalised_file = '/dev/null'
1610 else:
1611 expected_normalised_file = expected_file + ".normalised"
1612 write_file(expected_normalised_file, expected_str)
1613
1614 actual_normalised_file = actual_file + ".normalised"
1615 write_file(actual_normalised_file, actual_str)
1616
1617 # Ignore whitespace when diffing. We should only get to this
1618 # point if there are non-whitespace differences
1619 #
1620 # Note we are diffing the *actual* output, not the normalised
1621 # output. The normalised output may have whitespace squashed
1622 # (including newlines) so the diff would be hard to read.
1623 # This does mean that the diff might contain changes that
1624 # would be normalised away.
1625 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1626 ' ' + actual_file )
1627
1628 # If for some reason there were no non-whitespace differences,
1629 # then do a full diff
1630 if r == 0:
1631 r = os.system( 'diff -u ' + expected_file_for_diff + \
1632 ' ' + actual_file )
1633
1634 if config.accept:
1635 print 'Accepting new output.'
1636 write_file(expected_file, actual_raw)
1637 return 1
1638 else:
1639 return 0
1640
1641
1642 def normalise_whitespace( str ):
1643 # Merge contiguous whitespace characters into a single space.
1644 str = re.sub('[ \t\n]+', ' ', str)
1645 return str
1646
1647 def normalise_errmsg( str ):
1648 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1649 # the colon is there because it appears in error messages; this
1650 # hacky solution is used in place of more sophisticated filename
1651 # mangling
1652 str = re.sub('([^\\s])\\.exe', '\\1', str)
1653 # normalise slashes, minimise Windows/Unix filename differences
1654 str = re.sub('\\\\', '/', str)
1655 # The inplace ghc's are called ghc-stage[123] to avoid filename
1656 # collisions, so we need to normalise that to just "ghc"
1657 str = re.sub('ghc-stage[123]', 'ghc', str)
1658 # We sometimes see the name of the integer-gmp package on stderr,
1659 # but this can change (either the implementation name or the
1660 # version number), so we canonicalise it here
1661 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1662 return str
1663
1664 # normalise a .prof file, so that we can reasonably compare it against
1665 # a sample. This doesn't compare any of the actual profiling data,
1666 # only the shape of the profile and the number of entries.
1667 def normalise_prof (str):
1668 # strip everything up to the line beginning "COST CENTRE"
1669 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1670
1671 # strip results for CAFs, these tend to change unpredictably
1672 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1673
1674 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1675 # sometimes under MAIN.
1676 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1677
1678 # We have somthing like this:
1679
1680 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1681 # k Main 204 1 0.0 0.0 0.0 0.0
1682 # foo Main 205 1 0.0 0.0 0.0 0.0
1683 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1684
1685 # then we remove all the specific profiling data, leaving only the
1686 # cost centre name, module, and entries, to end up with this:
1687
1688 # MAIN MAIN 0
1689 # k Main 1
1690 # foo Main 1
1691 # foo.bar Main 1
1692
1693 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1694 return str
1695
1696 def normalise_slashes_( str ):
1697 str = re.sub('\\\\', '/', str)
1698 return str
1699
1700 def normalise_exe_( str ):
1701 str = re.sub('\.exe', '', str)
1702 return str
1703
1704 def normalise_output( str ):
1705 # Remove a .exe extension (for Windows)
1706 # This can occur in error messages generated by the program.
1707 str = re.sub('([^\\s])\\.exe', '\\1', str)
1708 return str
1709
1710 def normalise_asm( str ):
1711 lines = str.split('\n')
1712 # Only keep instructions and labels not starting with a dot.
1713 metadata = re.compile('^[ \t]*\\..*$')
1714 out = []
1715 for line in lines:
1716 # Drop metadata directives (e.g. ".type")
1717 if not metadata.match(line):
1718 line = re.sub('@plt', '', line)
1719 instr = line.lstrip().split()
1720 # Drop empty lines.
1721 if not instr:
1722 continue
1723 # Drop operands, except for call instructions.
1724 elif instr[0] == 'call':
1725 out.append(instr[0] + ' ' + instr[1])
1726 else:
1727 out.append(instr[0])
1728 out = '\n'.join(out)
1729 return out
1730
1731 def if_verbose( n, str ):
1732 if config.verbose >= n:
1733 print str
1734
1735 def if_verbose_dump( n, f ):
1736 if config.verbose >= n:
1737 try:
1738 print open(f).read()
1739 except:
1740 print ''
1741
1742 def rawSystem(cmd_and_args):
1743 # We prefer subprocess.call to os.spawnv as the latter
1744 # seems to send its arguments through a shell or something
1745 # with the Windows (non-cygwin) python. An argument "a b c"
1746 # turns into three arguments ["a", "b", "c"].
1747
1748 # However, subprocess is new in python 2.4, so fall back to
1749 # using spawnv if we don't have it
1750
1751 if have_subprocess:
1752 return subprocess.call(cmd_and_args)
1753 else:
1754 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1755
1756 # Note that this doesn't handle the timeout itself; it is just used for
1757 # commands that have timeout handling built-in.
1758 def rawSystemWithTimeout(cmd_and_args):
1759 r = rawSystem(cmd_and_args)
1760 if r == 98:
1761 # The python timeout program uses 98 to signal that ^C was pressed
1762 stopNow()
1763 return r
1764
1765 # cmd is a complex command in Bourne-shell syntax
1766 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1767 # Hence it must ultimately be run by a Bourne shell
1768 #
1769 # Mostly it invokes the command wrapped in 'timeout' thus
1770 # timeout 300 'cd . && ...blah blah'
1771 # so it's timeout's job to invoke the Bourne shell
1772 #
1773 # But watch out for the case when there is no timeout program!
1774 # Then, when using the native Python, os.system will invoke the cmd shell
1775
1776 def runCmd( cmd ):
1777 if_verbose( 1, cmd )
1778 r = 0
1779 if config.os == 'mingw32':
1780 # On MinGW, we will always have timeout
1781 assert config.timeout_prog!=''
1782
1783 if config.timeout_prog != '':
1784 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1785 else:
1786 r = os.system(cmd)
1787 return r << 8
1788
1789 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1790 if_verbose( 1, cmd )
1791 r = 0
1792 if config.os == 'mingw32':
1793 # On MinGW, we will always have timeout
1794 assert config.timeout_prog!=''
1795 timeout = int(ceil(config.timeout * timeout_multiplier))
1796
1797 if config.timeout_prog != '':
1798 if config.check_files_written:
1799 fn = name + ".strace"
1800 r = rawSystemWithTimeout(
1801 ["strace", "-o", fn, "-fF",
1802 "-e", "creat,open,chdir,clone,vfork",
1803 config.timeout_prog, str(timeout), cmd])
1804 addTestFilesWritten(name, fn)
1805 rm_no_fail(fn)
1806 else:
1807 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1808 else:
1809 r = os.system(cmd)
1810 return r << 8
1811
1812 def runCmdExitCode( cmd ):
1813 return (runCmd(cmd) >> 8);
1814
1815
1816 # -----------------------------------------------------------------------------
1817 # checking for files being written to by multiple tests
1818
1819 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1820 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1821 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1822 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1823 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1824 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1825 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1826 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1827 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1828 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1829 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1830 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1831 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1832 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1833 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1834 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1835 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1836
1837 # Files that are read or written but shouldn't be:
1838 # * ghci_history shouldn't be read or written by tests
1839 # * things under package.conf.d shouldn't be written by tests
1840 bad_file_usages = {}
1841
1842 # Mapping from tests to the list of files that they write
1843 files_written = {}
1844
1845 # Mapping from tests to the list of files that they write but don't clean
1846 files_written_not_removed = {}
1847
1848 def add_bad_file_usage(name, file):
1849 try:
1850 if not file in bad_file_usages[name]:
1851 bad_file_usages[name].append(file)
1852 except:
1853 bad_file_usages[name] = [file]
1854
1855 def mkPath(curdir, path):
1856 # Given the current full directory is 'curdir', what is the full
1857 # path to 'path'?
1858 return os.path.realpath(os.path.join(curdir, path))
1859
1860 def addTestFilesWritten(name, fn):
1861 if config.use_threads:
1862 with t.lockFilesWritten:
1863 addTestFilesWrittenHelper(name, fn)
1864 else:
1865 addTestFilesWrittenHelper(name, fn)
1866
1867 def addTestFilesWrittenHelper(name, fn):
1868 started = False
1869 working_directories = {}
1870
1871 with open(fn, 'r') as f:
1872 for line in f:
1873 m_pid = re_strace_pid.match(line)
1874 if m_pid:
1875 pid = m_pid.group(1)
1876 content = m_pid.group(2)
1877 elif re_strace_unavailable.match(line):
1878 next
1879 else:
1880 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1881
1882 m_open = re_strace_open.match(content)
1883 m_chdir = re_strace_chdir.match(content)
1884 m_clone = re_strace_clone.match(content)
1885 m_vfork = re_strace_vfork.match(content)
1886
1887 if not started:
1888 working_directories[pid] = os.getcwd()
1889 started = True
1890
1891 if m_open:
1892 file = m_open.group(1)
1893 file = mkPath(working_directories[pid], file)
1894 if file.endswith("ghci_history"):
1895 add_bad_file_usage(name, file)
1896 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1897 flags = m_open.group(2).split('|')
1898 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1899 if re.match('package\.conf\.d', file):
1900 add_bad_file_usage(name, file)
1901 else:
1902 try:
1903 if not file in files_written[name]:
1904 files_written[name].append(file)
1905 except:
1906 files_written[name] = [file]
1907 elif 'O_RDONLY' in flags:
1908 pass
1909 else:
1910 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1911 elif m_chdir:
1912 # We optimistically assume that unfinished chdir's are going to succeed
1913 dir = m_chdir.group(1)
1914 working_directories[pid] = mkPath(working_directories[pid], dir)
1915 elif m_clone:
1916 working_directories[m_clone.group(2)] = working_directories[pid]
1917 elif m_vfork:
1918 working_directories[m_vfork.group(2)] = working_directories[pid]
1919 elif re_strace_open_resumed.match(content):
1920 pass
1921 elif re_strace_chdir_resumed.match(content):
1922 pass
1923 elif re_strace_vfork_unfinished.match(content):
1924 pass
1925 elif re_strace_clone_unfinished.match(content):
1926 pass
1927 elif re_strace_ignore_sigchild.match(content):
1928 pass
1929 elif re_strace_ignore_sigvtalarm.match(content):
1930 pass
1931 elif re_strace_ignore_sigint.match(content):
1932 pass
1933 elif re_strace_ignore_sigfpe.match(content):
1934 pass
1935 elif re_strace_ignore_sigsegv.match(content):
1936 pass
1937 elif re_strace_ignore_sigpipe.match(content):
1938 pass
1939 else:
1940 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1941
1942 def checkForFilesWrittenProblems(file):
1943 foundProblem = False
1944
1945 files_written_inverted = {}
1946 for t in files_written.keys():
1947 for f in files_written[t]:
1948 try:
1949 files_written_inverted[f].append(t)
1950 except:
1951 files_written_inverted[f] = [t]
1952
1953 for f in files_written_inverted.keys():
1954 if len(files_written_inverted[f]) > 1:
1955 if not foundProblem:
1956 foundProblem = True
1957 file.write("\n")
1958 file.write("\nSome files are written by multiple tests:\n")
1959 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1960 if foundProblem:
1961 file.write("\n")
1962
1963 # -----
1964
1965 if len(files_written_not_removed) > 0:
1966 file.write("\n")
1967 file.write("\nSome files written but not removed:\n")
1968 tests = files_written_not_removed.keys()
1969 tests.sort()
1970 for t in tests:
1971 for f in files_written_not_removed[t]:
1972 file.write(" " + t + ": " + f + "\n")
1973 file.write("\n")
1974
1975 # -----
1976
1977 if len(bad_file_usages) > 0:
1978 file.write("\n")
1979 file.write("\nSome bad file usages:\n")
1980 tests = bad_file_usages.keys()
1981 tests.sort()
1982 for t in tests:
1983 for f in bad_file_usages[t]:
1984 file.write(" " + t + ": " + f + "\n")
1985 file.write("\n")
1986
1987 # -----------------------------------------------------------------------------
1988 # checking if ghostscript is available for checking the output of hp2ps
1989
1990 def genGSCmd(psfile):
1991 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1992
1993 def gsNotWorking():
1994 global gs_working
1995 print "GhostScript not available for hp2ps tests"
1996
1997 global gs_working
1998 gs_working = 0
1999 if config.have_profiling:
2000 if config.gs != '':
2001 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2002 if resultGood == 0:
2003 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2004 if resultBad != 0:
2005 print "GhostScript available for hp2ps tests"
2006 gs_working = 1;
2007 else:
2008 gsNotWorking();
2009 else:
2010 gsNotWorking();
2011 else:
2012 gsNotWorking();
2013
2014 def rm_no_fail( file ):
2015 try:
2016 os.remove( file )
2017 finally:
2018 return
2019
2020 def add_suffix( name, suffix ):
2021 if suffix == '':
2022 return name
2023 else:
2024 return name + '.' + suffix
2025
2026 def add_hs_lhs_suffix(name):
2027 if getTestOpts().c_src:
2028 return add_suffix(name, 'c')
2029 elif getTestOpts().cmm_src:
2030 return add_suffix(name, 'cmm')
2031 elif getTestOpts().objc_src:
2032 return add_suffix(name, 'm')
2033 elif getTestOpts().objcpp_src:
2034 return add_suffix(name, 'mm')
2035 elif getTestOpts().literate:
2036 return add_suffix(name, 'lhs')
2037 else:
2038 return add_suffix(name, 'hs')
2039
2040 def replace_suffix( name, suffix ):
2041 base, suf = os.path.splitext(name)
2042 return base + '.' + suffix
2043
2044 def in_testdir( name ):
2045 return (getTestOpts().testdir + '/' + name)
2046
2047 def qualify( name, suff ):
2048 return in_testdir(add_suffix(name, suff))
2049
2050
2051 # Finding the sample output. The filename is of the form
2052 #
2053 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2054 #
2055 # and we pick the most specific version available. The <version> is
2056 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2057 # more fine-grained control use if_compiler_lt().
2058 #
2059 def platform_wordsize_qualify( name, suff ):
2060
2061 basepath = qualify(name, suff)
2062
2063 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2064 for (platformSpecific, plat) in [(1, '-' + config.platform),
2065 (1, '-' + config.os),
2066 (0, '')]
2067 for ws in ['-ws-' + config.wordsize, '']
2068 for comp in ['-' + config.compiler_type, '']
2069 for vers in ['-' + config.compiler_maj_version, '']]
2070
2071 dir = glob.glob(basepath + '*')
2072 dir = map (lambda d: normalise_slashes_(d), dir)
2073
2074 for (platformSpecific, f) in paths:
2075 if f in dir:
2076 return (platformSpecific,f)
2077
2078 return (0, basepath)
2079
2080 # Clean up prior to the test, so that we can't spuriously conclude
2081 # that it passed on the basis of old run outputs.
2082 def pretest_cleanup(name):
2083 if getTestOpts().outputdir != None:
2084 odir = in_testdir(getTestOpts().outputdir)
2085 try:
2086 shutil.rmtree(odir)
2087 except:
2088 pass
2089 os.mkdir(odir)
2090
2091 rm_no_fail(qualify(name,'comp.stderr'))
2092 rm_no_fail(qualify(name,'run.stderr'))
2093 rm_no_fail(qualify(name,'run.stdout'))
2094 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2095 # simple_build zaps the following:
2096 # rm_nofail(qualify("o"))
2097 # rm_nofail(qualify(""))
2098 # not interested in the return code
2099
2100 # -----------------------------------------------------------------------------
2101 # Return a list of all the files ending in '.T' below the directory dir.
2102
2103 def findTFiles(roots):
2104 return concat(map(findTFiles_,roots))
2105
2106 def findTFiles_(path):
2107 if os.path.isdir(path):
2108 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2109 return findTFiles(paths)
2110 elif path[-2:] == '.T':
2111 return [path]
2112 else:
2113 return []
2114
2115 # -----------------------------------------------------------------------------
2116 # Output a test summary to the specified file object
2117
2118 def summary(t, file):
2119
2120 file.write('\n')
2121 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2122 file.write('OVERALL SUMMARY for test run started at '
2123 + t.start_time + '\n'
2124 + string.rjust(`t.total_tests`, 8)
2125 + ' total tests, which gave rise to\n'
2126 + string.rjust(`t.total_test_cases`, 8)
2127 + ' test cases, of which\n'
2128 + string.rjust(`t.n_tests_skipped`, 8)
2129 + ' were skipped\n'
2130 + '\n'
2131 + string.rjust(`t.n_missing_libs`, 8)
2132 + ' had missing libraries\n'
2133 + string.rjust(`t.n_expected_passes`, 8)
2134 + ' expected passes\n'
2135 + string.rjust(`t.n_expected_failures`, 8)
2136 + ' expected failures\n'
2137 + '\n'
2138 + string.rjust(`t.n_framework_failures`, 8)
2139 + ' caused framework failures\n'
2140 + string.rjust(`t.n_unexpected_passes`, 8)
2141 + ' unexpected passes\n'
2142 + string.rjust(`t.n_unexpected_failures`, 8)
2143 + ' unexpected failures\n'
2144 + '\n')
2145
2146 if t.n_unexpected_passes > 0:
2147 file.write('Unexpected passes:\n')
2148 printPassingTestInfosSummary(file, t.unexpected_passes)
2149
2150 if t.n_unexpected_failures > 0:
2151 file.write('Unexpected failures:\n')
2152 printFailingTestInfosSummary(file, t.unexpected_failures)
2153
2154 if config.check_files_written:
2155 checkForFilesWrittenProblems(file)
2156
2157 if stopping():
2158 file.write('WARNING: Testsuite run was terminated early\n')
2159
2160 def printUnexpectedTests(file, testInfoss):
2161 unexpected = []
2162 for testInfos in testInfoss:
2163 directories = testInfos.keys()
2164 for directory in directories:
2165 tests = testInfos[directory].keys()
2166 unexpected += tests
2167 if unexpected != []:
2168 file.write('Unexpected results from:\n')
2169 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2170 file.write('\n')
2171
2172 def printPassingTestInfosSummary(file, testInfos):
2173 directories = testInfos.keys()
2174 directories.sort()
2175 maxDirLen = max(map ((lambda x : len(x)), directories))
2176 for directory in directories:
2177 tests = testInfos[directory].keys()
2178 tests.sort()
2179 for test in tests:
2180 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2181 ' (' + join(testInfos[directory][test],',') + ')\n')
2182 file.write('\n')
2183
2184 def printFailingTestInfosSummary(file, testInfos):
2185 directories = testInfos.keys()
2186 directories.sort()
2187 maxDirLen = max(map ((lambda x : len(x)), directories))
2188 for directory in directories:
2189 tests = testInfos[directory].keys()
2190 tests.sort()
2191 for test in tests:
2192 reasons = testInfos[directory][test].keys()
2193 for reason in reasons:
2194 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2195 ' [' + reason + ']' + \
2196 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2197 file.write('\n')
2198
2199 def getStdout(cmd):
2200 if have_subprocess:
2201 p = subprocess.Popen(cmd,
2202 stdout=subprocess.PIPE,
2203 stderr=subprocess.PIPE)
2204 (stdout, stderr) = p.communicate()
2205 r = p.wait()
2206 if r != 0:
2207 raise Exception("Command failed: " + str(cmd))
2208 if stderr != '':
2209 raise Exception("stderr from command: " + str(cmd))
2210 return stdout
2211 else:
2212 raise Exception("Need subprocess to get stdout, but don't have it")