More helper conversions
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def ignore_output( name, opts ):
134 opts.ignore_output = 1
135
136 def no_stdin( name, opts ):
137 opts.no_stdin = 1
138
139 def combined_output( name, opts ):
140 opts.combined_output = True
141
142 # -----
143
144 def expect_fail_for( ways ):
145 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146
147 def _expect_fail_for( name, opts, ways ):
148 opts.expect_fail_for = ways
149
150 def expect_broken( bug ):
151 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
152
153 def _expect_broken( name, opts, bug ):
154 record_broken(name, bug)
155 opts.expect = 'fail';
156
157 def expect_broken_for( bug, ways ):
158 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
159
160 def _expect_broken_for( name, opts, bug, ways ):
161 record_broken(name, bug)
162 opts.expect_fail_for = ways
163
164 def record_broken(name, bug):
165 global brokens
166 me = (bug, name)
167 if not me in brokens:
168 brokens.append(me)
169
170 # -----
171
172 def omit_ways( ways ):
173 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174
175 def _omit_ways( name, opts, ways ):
176 opts.omit_ways = ways
177
178 # -----
179
180 def only_ways( ways ):
181 return lambda name, opts, w=ways: _only_ways( name, opts, w )
182
183 def _only_ways( name, opts, ways ):
184 opts.only_ways = ways
185
186 # -----
187
188 def extra_ways( ways ):
189 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190
191 def _extra_ways( name, opts, ways ):
192 opts.extra_ways = ways
193
194 # -----
195
196 def omit_compiler_types( compiler_types ):
197 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
198
199 def _omit_compiler_types( name, opts, compiler_types ):
200 if config.compiler_type in compiler_types:
201 opts.skip = 1
202
203 # -----
204
205 def only_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
207
208 def _only_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type not in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def set_stdin( file ):
215 return lambda name, opts, f=file: _set_stdin(name, opts, f);
216
217 def _set_stdin( name, opts, f ):
218 opts.stdin = f
219
220 # -----
221
222 def exit_code( val ):
223 return lambda name, opts, v=val: _exit_code(name, opts, v);
224
225 def _exit_code( name, opts, v ):
226 opts.exit_code = v
227
228 # -----
229
230 def timeout_multiplier( val ):
231 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232
233 def _timeout_multiplier( name, opts, v ):
234 opts.timeout_multiplier = v
235
236 # -----
237
238 def extra_run_opts( val ):
239 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240
241 def _extra_run_opts( name, opts, v ):
242 opts.extra_run_opts = v
243
244 # -----
245
246 def extra_hc_opts( val ):
247 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
248
249 def _extra_hc_opts( name, opts, v ):
250 opts.extra_hc_opts = v
251
252 # -----
253
254 def extra_clean( files ):
255 return lambda name, opts, v=files: _extra_clean(name, opts, v);
256
257 def _extra_clean( name, opts, v ):
258 opts.clean_files = v
259
260 # -----
261
262 def stats_num_field( field, expecteds ):
263 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
264
265 def _stats_num_field( name, opts, field, expecteds ):
266 if field in opts.stats_range_fields:
267 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
268
269 if type(expecteds) is types.ListType:
270 for (b, expected, dev) in expecteds:
271 if b:
272 opts.stats_range_fields[field] = (expected, dev)
273 return
274 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275
276 else:
277 (expected, dev) = expecteds
278 opts.stats_range_fields[field] = (expected, dev)
279
280 def compiler_stats_num_field( field, expecteds ):
281 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282
283 def _compiler_stats_num_field( name, opts, field, expecteds ):
284 if field in opts.compiler_stats_range_fields:
285 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
286
287 for (b, expected, dev) in expecteds:
288 if b:
289 opts.compiler_stats_range_fields[field] = (expected, dev)
290 return
291
292 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293
294 # -----
295
296 def skip_if_no_ghci(name, opts):
297 if not ('ghci' in config.run_ways):
298 opts.skip = 1
299
300 # ----
301
302 def when(b, f):
303 # When list_brokens is on, we want to see all expect_broken calls,
304 # so we always do f
305 if b or config.list_broken:
306 return f
307 else:
308 return normal
309
310 def unless(b, f):
311 return when(not b, f)
312
313 def fast():
314 return config.fast
315
316 def platform( plat ):
317 return config.platform == plat
318
319 def opsys( os ):
320 return config.os == os
321
322 def arch( arch ):
323 return config.arch == arch
324
325 def wordsize( ws ):
326 return config.wordsize == str(ws)
327
328 def unregisterised( ):
329 return config.unregisterised
330
331 def msys( ):
332 return config.msys
333
334 def cygwin( ):
335 return config.cygwin
336
337 def have_vanilla( ):
338 return config.have_vanilla
339
340 def have_dynamic( ):
341 return config.have_dynamic
342
343 def have_profiling( ):
344 return config.have_profiling
345
346 # ---
347
348 def ghci_dynamic( ):
349 return config.ghc_dynamic_by_default
350
351 def in_tree_compiler( ):
352 return config.in_tree_compiler
353
354 def compiler_type( compiler ):
355 return config.compiler_type == compiler
356
357 def compiler_profiled( ):
358 return config.compiler_profiled
359
360 def compiler_lt( compiler, version ):
361 return config.compiler_type == compiler and \
362 version_lt(config.compiler_version, version)
363
364 def compiler_le( compiler, version ):
365 return config.compiler_type == compiler and \
366 version_le(config.compiler_version, version)
367
368 def compiler_gt( compiler, version ):
369 return config.compiler_type == compiler and \
370 version_gt(config.compiler_version, version)
371
372 def compiler_ge( compiler, version ):
373 return config.compiler_type == compiler and \
374 version_ge(config.compiler_version, version)
375
376 def compiler_debugged( ):
377 return config.compiler_debugged
378
379 def namebase( nb ):
380 return lambda opts, nb=nb: _namebase(opts, nb)
381
382 def _namebase( opts, nb ):
383 opts.with_namebase = nb
384
385 # ---
386
387 def tag( t ):
388 return t in config.compiler_tags
389
390 # ---
391 def high_memory_usage(name, opts):
392 opts.alone = True
393
394 # ---
395 def literate( name, opts ):
396 opts.literate = 1;
397
398 def c_src( name, opts ):
399 opts.c_src = 1;
400
401 def objc_src( name, opts ):
402 opts.objc_src = 1;
403
404 def objcpp_src( name, opts ):
405 opts.objcpp_src = 1;
406
407 def cmm_src( name, opts ):
408 opts.cmm_src = 1;
409
410 def outputdir( odir ):
411 return lambda name, opts, d=odir: _outputdir(name, opts, d)
412
413 def _outputdir( name, opts, odir ):
414 opts.outputdir = odir;
415
416 # ----
417
418 def pre_cmd( cmd ):
419 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
420
421 def _pre_cmd( name, opts, cmd ):
422 opts.pre_cmd = cmd
423
424 # ----
425
426 def clean_cmd( cmd ):
427 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
428
429 def _clean_cmd( name, opts, cmd ):
430 opts.clean_cmd = cmd
431
432 # ----
433
434 def cmd_prefix( prefix ):
435 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
436
437 def _cmd_prefix( name, opts, prefix ):
438 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
439
440 # ----
441
442 def cmd_wrapper( fun ):
443 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
444
445 def _cmd_wrapper( name, opts, fun ):
446 opts.cmd_wrapper = fun
447
448 # ----
449
450 def compile_cmd_prefix( prefix ):
451 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
452
453 def _compile_cmd_prefix( name, opts, prefix ):
454 opts.compile_cmd_prefix = prefix
455
456 # ----
457
458 def normalise_slashes( name, opts ):
459 opts.extra_normaliser = normalise_slashes_
460
461 def normalise_exe( name, opts ):
462 opts.extra_normaliser = normalise_exe_
463
464 def normalise_fun( fun ):
465 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
466
467 def _normalise_fun( name, opts, f ):
468 opts.extra_normaliser = f
469
470 def normalise_errmsg_fun( fun ):
471 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
472
473 def _normalise_errmsg_fun( name, opts, f ):
474 opts.extra_errmsg_normaliser = f
475
476 def two_normalisers(f, g):
477 return lambda x, f=f, g=g: f(g(x))
478
479 # ----
480 # Function for composing two opt-fns together
481
482 def composes( fs ):
483 return reduce(lambda f, g: compose(f, g), fs)
484
485 def compose( f, g ):
486 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
487
488 def _compose( name, opts, f, g ):
489 f(name, opts)
490 g(name, opts)
491
492 # -----------------------------------------------------------------------------
493 # The current directory of tests
494
495 def newTestDir( dir ):
496 global thisdir_settings
497 # reset the options for this test directory
498 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
499
500 def _newTestDir( name, opts, dir ):
501 opts.testdir = dir
502 opts.compiler_always_flags = config.compiler_always_flags
503
504 # -----------------------------------------------------------------------------
505 # Actually doing tests
506
507 parallelTests = []
508 aloneTests = []
509 allTestNames = set([])
510
511 def runTest (opts, name, func, args):
512 ok = 0
513
514 if config.use_threads:
515 t.thread_pool.acquire()
516 try:
517 while config.threads<(t.running_threads+1):
518 t.thread_pool.wait()
519 t.running_threads = t.running_threads+1
520 ok=1
521 t.thread_pool.release()
522 thread.start_new_thread(test_common_thread, (name, opts, func, args))
523 except:
524 if not ok:
525 t.thread_pool.release()
526 else:
527 test_common_work (name, opts, func, args)
528
529 # name :: String
530 # setup :: TestOpts -> IO ()
531 def test (name, setup, func, args):
532 global aloneTests
533 global parallelTests
534 global allTestNames
535 global thisdir_settings
536 if name in allTestNames:
537 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
538 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
539 framework_fail(name, 'bad_name', 'This test has an invalid name')
540
541 # Make a deep copy of the default_testopts, as we need our own copy
542 # of any dictionaries etc inside it. Otherwise, if one test modifies
543 # them, all tests will see the modified version!
544 myTestOpts = copy.deepcopy(default_testopts)
545
546 if type(setup) is types.ListType:
547 setup = composes(setup)
548
549 setup = compose(thisdir_settings, setup)
550 setup(name, myTestOpts)
551
552 thisTest = lambda : runTest(myTestOpts, name, func, args)
553 if myTestOpts.alone:
554 aloneTests.append(thisTest)
555 else:
556 parallelTests.append(thisTest)
557 allTestNames.add(name)
558
559 if config.use_threads:
560 def test_common_thread(name, opts, func, args):
561 t.lock.acquire()
562 try:
563 test_common_work(name,opts,func,args)
564 finally:
565 t.lock.release()
566 t.thread_pool.acquire()
567 t.running_threads = t.running_threads - 1
568 t.thread_pool.notify()
569 t.thread_pool.release()
570
571 def get_package_cache_timestamp():
572 if config.package_conf_cache_file == '':
573 return 0.0
574 else:
575 try:
576 return os.stat(config.package_conf_cache_file).st_mtime
577 except:
578 return 0.0
579
580
581 def test_common_work (name, opts, func, args):
582 try:
583 t.total_tests = t.total_tests+1
584 setLocalTestOpts(opts)
585
586 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
587
588 # All the ways we might run this test
589 if func == compile or func == multimod_compile:
590 all_ways = config.compile_ways
591 elif func == compile_and_run or func == multimod_compile_and_run:
592 all_ways = config.run_ways
593 elif func == ghci_script:
594 if 'ghci' in config.run_ways:
595 all_ways = ['ghci']
596 else:
597 all_ways = []
598 else:
599 all_ways = ['normal']
600
601 # A test itself can request extra ways by setting opts.extra_ways
602 all_ways = all_ways + filter(lambda way: way not in all_ways,
603 opts.extra_ways)
604
605 t.total_test_cases = t.total_test_cases + len(all_ways)
606
607 ok_way = lambda way: \
608 not getTestOpts().skip \
609 and (config.only == [] or name in config.only) \
610 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
611 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
612 and way not in getTestOpts().omit_ways
613
614 # Which ways we are asked to skip
615 do_ways = filter (ok_way,all_ways)
616
617 # In fast mode, we skip all but one way
618 if config.fast and len(do_ways) > 0:
619 do_ways = [do_ways[0]]
620
621 if not config.clean_only:
622 # Run the required tests...
623 for way in do_ways:
624 if stopping():
625 break
626 do_test (name, way, func, args)
627
628 for way in all_ways:
629 if way not in do_ways:
630 skiptest (name,way)
631
632 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
633 clean(map (lambda suff: name + suff,
634 ['', '.exe', '.exe.manifest', '.genscript',
635 '.stderr.normalised', '.stdout.normalised',
636 '.run.stderr', '.run.stdout',
637 '.run.stderr.normalised', '.run.stdout.normalised',
638 '.comp.stderr', '.comp.stdout',
639 '.comp.stderr.normalised', '.comp.stdout.normalised',
640 '.interp.stderr', '.interp.stdout',
641 '.interp.stderr.normalised', '.interp.stdout.normalised',
642 '.stats', '.comp.stats',
643 '.hi', '.o', '.prof', '.exe.prof', '.hc',
644 '_stub.h', '_stub.c', '_stub.o',
645 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
646
647 if func == multi_compile or func == multi_compile_fail:
648 extra_mods = args[1]
649 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
650 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
651
652 clean(getTestOpts().clean_files)
653
654 if getTestOpts().outputdir != None:
655 odir = in_testdir(getTestOpts().outputdir)
656 try:
657 shutil.rmtree(odir)
658 except:
659 pass
660
661 try:
662 shutil.rmtree(in_testdir('.hpc.' + name))
663 except:
664 pass
665
666 try:
667 cleanCmd = getTestOpts().clean_cmd
668 if cleanCmd != None:
669 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
670 if result != 0:
671 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
672 except e:
673 framework_fail(name, 'cleaning', 'clean-command exception')
674
675 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
676
677 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
678 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
679
680 try:
681 for f in files_written[name]:
682 if os.path.exists(f):
683 try:
684 if not f in files_written_not_removed[name]:
685 files_written_not_removed[name].append(f)
686 except:
687 files_written_not_removed[name] = [f]
688 except:
689 pass
690 except Exception, e:
691 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
692
693 def clean(strs):
694 for str in strs:
695 for name in glob.glob(in_testdir(str)):
696 clean_full_path(name)
697
698 def clean_full_path(name):
699 try:
700 # Remove files...
701 os.remove(name)
702 except OSError, e1:
703 try:
704 # ... and empty directories
705 os.rmdir(name)
706 except OSError, e2:
707 # We don't want to fail here, but we do want to know
708 # what went wrong, so print out the exceptions.
709 # ENOENT isn't a problem, though, as we clean files
710 # that don't necessarily exist.
711 if e1.errno != errno.ENOENT:
712 print e1
713 if e2.errno != errno.ENOENT:
714 print e2
715
716 def do_test(name, way, func, args):
717 full_name = name + '(' + way + ')'
718
719 try:
720 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
721 str([t.n_unexpected_passes, \
722 t.n_unexpected_failures, \
723 t.n_framework_failures])
724
725 if config.use_threads:
726 t.lock.release()
727
728 try:
729 preCmd = getTestOpts().pre_cmd
730 if preCmd != None:
731 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
732 if result != 0:
733 framework_fail(name, way, 'pre-command failed: ' + str(result))
734 except e:
735 framework_fail(name, way, 'pre-command exception')
736
737 try:
738 result = apply(func, [name,way] + args)
739 finally:
740 if config.use_threads:
741 t.lock.acquire()
742
743 if getTestOpts().expect != 'pass' and \
744 getTestOpts().expect != 'fail' and \
745 getTestOpts().expect != 'missing-lib':
746 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
747
748 try:
749 passFail = result['passFail']
750 except:
751 passFail = 'No passFail found'
752
753 if passFail == 'pass':
754 if getTestOpts().expect == 'pass' \
755 and way not in getTestOpts().expect_fail_for:
756 t.n_expected_passes = t.n_expected_passes + 1
757 if name in t.expected_passes:
758 t.expected_passes[name].append(way)
759 else:
760 t.expected_passes[name] = [way]
761 else:
762 print '*** unexpected pass for', full_name
763 t.n_unexpected_passes = t.n_unexpected_passes + 1
764 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
765 elif passFail == 'fail':
766 if getTestOpts().expect == 'pass' \
767 and way not in getTestOpts().expect_fail_for:
768 print '*** unexpected failure for', full_name
769 t.n_unexpected_failures = t.n_unexpected_failures + 1
770 reason = result['reason']
771 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
772 else:
773 if getTestOpts().expect == 'missing-lib':
774 t.n_missing_libs = t.n_missing_libs + 1
775 if name in t.missing_libs:
776 t.missing_libs[name].append(way)
777 else:
778 t.missing_libs[name] = [way]
779 else:
780 t.n_expected_failures = t.n_expected_failures + 1
781 if name in t.expected_failures:
782 t.expected_failures[name].append(way)
783 else:
784 t.expected_failures[name] = [way]
785 else:
786 framework_fail(name, way, 'bad result ' + passFail)
787 except KeyboardInterrupt:
788 stopNow()
789 except:
790 framework_fail(name, way, 'do_test exception')
791 traceback.print_exc()
792
793 def addPassingTestInfo (testInfos, directory, name, way):
794 directory = re.sub('^\\.[/\\\\]', '', directory)
795
796 if not directory in testInfos:
797 testInfos[directory] = {}
798
799 if not name in testInfos[directory]:
800 testInfos[directory][name] = []
801
802 testInfos[directory][name].append(way)
803
804 def addFailingTestInfo (testInfos, directory, name, reason, way):
805 directory = re.sub('^\\.[/\\\\]', '', directory)
806
807 if not directory in testInfos:
808 testInfos[directory] = {}
809
810 if not name in testInfos[directory]:
811 testInfos[directory][name] = {}
812
813 if not reason in testInfos[directory][name]:
814 testInfos[directory][name][reason] = []
815
816 testInfos[directory][name][reason].append(way)
817
818 def skiptest (name, way):
819 # print 'Skipping test \"', name, '\"'
820 t.n_tests_skipped = t.n_tests_skipped + 1
821 if name in t.tests_skipped:
822 t.tests_skipped[name].append(way)
823 else:
824 t.tests_skipped[name] = [way]
825
826 def framework_fail( name, way, reason ):
827 full_name = name + '(' + way + ')'
828 print '*** framework failure for', full_name, reason
829 t.n_framework_failures = t.n_framework_failures + 1
830 if name in t.framework_failures:
831 t.framework_failures[name].append(way)
832 else:
833 t.framework_failures[name] = [way]
834
835 def badResult(result):
836 try:
837 if result['passFail'] == 'pass':
838 return False
839 return True
840 except:
841 return True
842
843 def passed():
844 return {'passFail': 'pass'}
845
846 def failBecause(reason):
847 return {'passFail': 'fail', 'reason': reason}
848
849 # -----------------------------------------------------------------------------
850 # Generic command tests
851
852 # A generic command test is expected to run and exit successfully.
853 #
854 # The expected exit code can be changed via exit_code() as normal, and
855 # the expected stdout/stderr are stored in <testname>.stdout and
856 # <testname>.stderr. The output of the command can be ignored
857 # altogether by using run_command_ignore_output instead of
858 # run_command.
859
860 def run_command( name, way, cmd ):
861 return simple_run( name, '', cmd, '' )
862
863 # -----------------------------------------------------------------------------
864 # GHCi tests
865
866 def ghci_script( name, way, script ):
867 # filter out -fforce-recomp from compiler_always_flags, because we're
868 # actually testing the recompilation behaviour in the GHCi tests.
869 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
870 flags.append(getTestOpts().extra_hc_opts)
871 if getTestOpts().outputdir != None:
872 flags.extend(["-outputdir", getTestOpts().outputdir])
873
874 # We pass HC and HC_OPTS as environment variables, so that the
875 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
876 cmd = "HC='" + config.compiler + "' " + \
877 "HC_OPTS='" + join(flags,' ') + "' " + \
878 "'" + config.compiler + "'" + \
879 ' --interactive -v0 -ignore-dot-ghci ' + \
880 join(flags,' ')
881
882 getTestOpts().stdin = script
883 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
884
885 # -----------------------------------------------------------------------------
886 # Compile-only tests
887
888 def compile( name, way, extra_hc_opts ):
889 return do_compile( name, way, 0, '', [], extra_hc_opts )
890
891 def compile_fail( name, way, extra_hc_opts ):
892 return do_compile( name, way, 1, '', [], extra_hc_opts )
893
894 def multimod_compile( name, way, top_mod, extra_hc_opts ):
895 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
896
897 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
898 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
899
900 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
901 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
902
903 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
904 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
905
906 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
907 # print 'Compile only, extra args = ', extra_hc_opts
908 pretest_cleanup(name)
909
910 result = extras_build( way, extra_mods, extra_hc_opts )
911 if badResult(result):
912 return result
913 extra_hc_opts = result['hc_opts']
914
915 force = 0
916 if extra_mods:
917 force = 1
918 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
919
920 if badResult(result):
921 return result
922
923 # the actual stderr should always match the expected, regardless
924 # of whether we expected the compilation to fail or not (successful
925 # compilations may generate warnings).
926
927 if getTestOpts().with_namebase == None:
928 namebase = name
929 else:
930 namebase = getTestOpts().with_namebase
931
932 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
933 actual_stderr_file = qualify(name, 'comp.stderr')
934
935 if not compare_outputs('stderr', \
936 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
937 expected_stderr_file, actual_stderr_file):
938 return failBecause('stderr mismatch')
939
940 # no problems found, this test passed
941 return passed()
942
943 def compile_cmp_asm( name, way, extra_hc_opts ):
944 print 'Compile only, extra args = ', extra_hc_opts
945 pretest_cleanup(name)
946 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
947
948 if badResult(result):
949 return result
950
951 # the actual stderr should always match the expected, regardless
952 # of whether we expected the compilation to fail or not (successful
953 # compilations may generate warnings).
954
955 if getTestOpts().with_namebase == None:
956 namebase = name
957 else:
958 namebase = getTestOpts().with_namebase
959
960 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
961 actual_asm_file = qualify(name, 's')
962
963 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
964 expected_asm_file, actual_asm_file):
965 return failBecause('asm mismatch')
966
967 # no problems found, this test passed
968 return passed()
969
970 # -----------------------------------------------------------------------------
971 # Compile-and-run tests
972
973 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
974 # print 'Compile and run, extra args = ', extra_hc_opts
975 pretest_cleanup(name)
976
977 result = extras_build( way, extra_mods, extra_hc_opts )
978 if badResult(result):
979 return result
980 extra_hc_opts = result['hc_opts']
981
982 if way == 'ghci': # interpreted...
983 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
984 elif way == 'extcore' or way == 'optextcore' :
985 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
986 else: # compiled...
987 force = 0
988 if extra_mods:
989 force = 1
990
991 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
992 if badResult(result):
993 return result
994
995 cmd = './' + name;
996
997 # we don't check the compiler's stderr for a compile-and-run test
998 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
999
1000 def compile_and_run( name, way, extra_hc_opts ):
1001 return compile_and_run__( name, way, '', [], extra_hc_opts)
1002
1003 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1004 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1005
1006 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1007 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1008
1009 def stats( name, way, stats_file ):
1010 opts = getTestOpts()
1011 return checkStats(stats_file, opts.stats_range_fields)
1012
1013 # -----------------------------------------------------------------------------
1014 # Check -t stats info
1015
1016 def checkStats(stats_file, range_fields):
1017 result = passed()
1018 if len(range_fields) > 0:
1019 f = open(in_testdir(stats_file))
1020 contents = f.read()
1021 f.close()
1022
1023 for (field, (expected, dev)) in range_fields.items():
1024 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1025 if m == None:
1026 print 'Failed to find field: ', field
1027 result = failBecause('no such stats field')
1028 val = int(m.group(1))
1029
1030 lowerBound = trunc( expected * ((100 - float(dev))/100));
1031 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1032
1033 if val < lowerBound:
1034 print field, 'value is too low:'
1035 print '(If this is because you have improved GHC, please'
1036 print 'update the test so that GHC doesn\'t regress again)'
1037 result = failBecause('stat too good')
1038 if val > upperBound:
1039 print field, 'value is too high:'
1040 result = failBecause('stat not good enough')
1041
1042 if val < lowerBound or val > upperBound:
1043 valStr = str(val)
1044 valLen = len(valStr)
1045 expectedStr = str(expected)
1046 expectedLen = len(expectedStr)
1047 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1048 def display(descr, val, extra):
1049 print descr, string.rjust(str(val), length), extra
1050 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1051 display(' Lower bound ' + field + ':', lowerBound, '')
1052 display(' Upper bound ' + field + ':', upperBound, '')
1053 display(' Actual ' + field + ':', val, '')
1054
1055 return result
1056
1057 # -----------------------------------------------------------------------------
1058 # Build a single-module program
1059
1060 def extras_build( way, extra_mods, extra_hc_opts ):
1061 for modopts in extra_mods:
1062 mod, opts = modopts
1063 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1064 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1065 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1066 if badResult(result):
1067 return result
1068
1069 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1070
1071
1072 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1073 opts = getTestOpts()
1074 errname = add_suffix(name, 'comp.stderr')
1075 rm_no_fail( qualify(errname, '') )
1076
1077 if top_mod != '':
1078 srcname = top_mod
1079 rm_no_fail( qualify(name, '') )
1080 base, suf = os.path.splitext(top_mod)
1081 rm_no_fail( qualify(base, '') )
1082 rm_no_fail( qualify(base, 'exe') )
1083 elif addsuf:
1084 srcname = add_hs_lhs_suffix(name)
1085 rm_no_fail( qualify(name, '') )
1086 else:
1087 srcname = name
1088 rm_no_fail( qualify(name, 'o') )
1089
1090 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1091
1092 to_do = ''
1093 if top_mod != '':
1094 to_do = '--make '
1095 if link:
1096 to_do = to_do + '-o ' + name
1097 elif link:
1098 to_do = '-o ' + name
1099 elif opts.compile_to_hc:
1100 to_do = '-C'
1101 else:
1102 to_do = '-c' # just compile
1103
1104 stats_file = name + '.comp.stats'
1105 if len(opts.compiler_stats_range_fields) > 0:
1106 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1107
1108 # Required by GHC 7.3+, harmless for earlier versions:
1109 if (getTestOpts().c_src or
1110 getTestOpts().objc_src or
1111 getTestOpts().objcpp_src or
1112 getTestOpts().cmm_src):
1113 extra_hc_opts += ' -no-hs-main '
1114
1115 if getTestOpts().compile_cmd_prefix == '':
1116 cmd_prefix = ''
1117 else:
1118 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1119
1120 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1121 if noforce:
1122 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1123 if getTestOpts().outputdir != None:
1124 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1125
1126 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1127 + config.compiler + "' " \
1128 + join(comp_flags,' ') + ' ' \
1129 + to_do + ' ' + srcname + ' ' \
1130 + join(config.way_flags(name)[way],' ') + ' ' \
1131 + extra_hc_opts + ' ' \
1132 + opts.extra_hc_opts + ' ' \
1133 + '>' + errname + ' 2>&1'
1134
1135 result = runCmdFor(name, cmd)
1136
1137 if result != 0 and not should_fail:
1138 actual_stderr = qualify(name, 'comp.stderr')
1139 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1140 if_verbose_dump(1,actual_stderr)
1141
1142 # ToDo: if the sub-shell was killed by ^C, then exit
1143
1144 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1145
1146 if badResult(statsResult):
1147 return statsResult
1148
1149 if should_fail:
1150 if result == 0:
1151 return failBecause('exit code 0')
1152 else:
1153 if result != 0:
1154 return failBecause('exit code non-0')
1155
1156 return passed()
1157
1158 # -----------------------------------------------------------------------------
1159 # Run a program and check its output
1160 #
1161 # If testname.stdin exists, route input from that, else
1162 # from /dev/null. Route output to testname.run.stdout and
1163 # testname.run.stderr. Returns the exit code of the run.
1164
1165 def simple_run( name, way, prog, args ):
1166 opts = getTestOpts()
1167
1168 # figure out what to use for stdin
1169 if opts.stdin != '':
1170 use_stdin = opts.stdin
1171 else:
1172 stdin_file = add_suffix(name, 'stdin')
1173 if os.path.exists(in_testdir(stdin_file)):
1174 use_stdin = stdin_file
1175 else:
1176 use_stdin = '/dev/null'
1177
1178 run_stdout = add_suffix(name,'run.stdout')
1179 run_stderr = add_suffix(name,'run.stderr')
1180
1181 rm_no_fail(qualify(name,'run.stdout'))
1182 rm_no_fail(qualify(name,'run.stderr'))
1183 rm_no_fail(qualify(name, 'hp'))
1184 rm_no_fail(qualify(name,'ps'))
1185 rm_no_fail(qualify(name, 'prof'))
1186
1187 my_rts_flags = rts_flags(way)
1188
1189 stats_file = name + '.stats'
1190 if len(opts.stats_range_fields) > 0:
1191 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1192
1193 if opts.no_stdin:
1194 stdin_comes_from = ''
1195 else:
1196 stdin_comes_from = ' <' + use_stdin
1197
1198 if opts.combined_output:
1199 redirection = ' >' + run_stdout \
1200 + ' 2>&1'
1201 else:
1202 redirection = ' >' + run_stdout \
1203 + ' 2>' + run_stderr
1204
1205 cmd = prog + ' ' + args + ' ' \
1206 + my_rts_flags + ' ' \
1207 + stdin_comes_from \
1208 + redirection
1209
1210 if opts.cmd_wrapper != None:
1211 cmd = opts.cmd_wrapper(cmd);
1212
1213 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1214
1215 # run the command
1216 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1217
1218 exit_code = result >> 8
1219 signal = result & 0xff
1220
1221 # check the exit code
1222 if exit_code != opts.exit_code:
1223 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1224 dump_stdout(name)
1225 dump_stderr(name)
1226 return failBecause('bad exit code')
1227
1228 check_hp = my_rts_flags.find("-h") != -1
1229 check_prof = my_rts_flags.find("-p") != -1
1230
1231 if not opts.ignore_output:
1232 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1233 bad_stdout = not check_stdout_ok(name)
1234 if bad_stderr:
1235 return failBecause('bad stderr')
1236 if bad_stdout:
1237 return failBecause('bad stdout')
1238 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1239 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1240 return failBecause('bad heap profile')
1241 if check_prof and not check_prof_ok(name):
1242 return failBecause('bad profile')
1243
1244 return checkStats(stats_file, opts.stats_range_fields)
1245
1246 def rts_flags(way):
1247 if (way == ''):
1248 return ''
1249 else:
1250 args = config.way_rts_flags[way]
1251
1252 if args == []:
1253 return ''
1254 else:
1255 return '+RTS ' + join(args,' ') + ' -RTS'
1256
1257 # -----------------------------------------------------------------------------
1258 # Run a program in the interpreter and check its output
1259
1260 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1261 outname = add_suffix(name, 'interp.stdout')
1262 errname = add_suffix(name, 'interp.stderr')
1263 rm_no_fail(outname)
1264 rm_no_fail(errname)
1265 rm_no_fail(name)
1266
1267 if (top_mod == ''):
1268 srcname = add_hs_lhs_suffix(name)
1269 else:
1270 srcname = top_mod
1271
1272 scriptname = add_suffix(name, 'genscript')
1273 qscriptname = in_testdir(scriptname)
1274 rm_no_fail(qscriptname)
1275
1276 delimiter = '===== program output begins here\n'
1277
1278 script = open(qscriptname, 'w')
1279 if not compile_only:
1280 # set the prog name and command-line args to match the compiled
1281 # environment.
1282 script.write(':set prog ' + name + '\n')
1283 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1284 # Add marker lines to the stdout and stderr output files, so we
1285 # can separate GHCi's output from the program's.
1286 script.write(':! echo ' + delimiter)
1287 script.write(':! echo 1>&2 ' + delimiter)
1288 # Set stdout to be line-buffered to match the compiled environment.
1289 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1290 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1291 # in the event of an exception as for the compiled program.
1292 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1293 script.close()
1294
1295 # figure out what to use for stdin
1296 if getTestOpts().stdin != '':
1297 stdin_file = in_testdir(getTestOpts().stdin)
1298 else:
1299 stdin_file = qualify(name, 'stdin')
1300
1301 if os.path.exists(stdin_file):
1302 stdin = open(stdin_file, 'r')
1303 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1304
1305 script.close()
1306
1307 flags = copy.copy(getTestOpts().compiler_always_flags)
1308 if getTestOpts().outputdir != None:
1309 flags.extend(["-outputdir", getTestOpts().outputdir])
1310
1311 cmd = "'" + config.compiler + "' " \
1312 + join(flags,' ') + ' ' \
1313 + srcname + ' ' \
1314 + join(config.way_flags(name)[way],' ') + ' ' \
1315 + extra_hc_opts + ' ' \
1316 + getTestOpts().extra_hc_opts + ' ' \
1317 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1318
1319 if getTestOpts().cmd_wrapper != None:
1320 cmd = getTestOpts().cmd_wrapper(cmd);
1321
1322 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1323
1324 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1325
1326 exit_code = result >> 8
1327 signal = result & 0xff
1328
1329 # split the stdout into compilation/program output
1330 split_file(in_testdir(outname), delimiter,
1331 qualify(name, 'comp.stdout'),
1332 qualify(name, 'run.stdout'))
1333 split_file(in_testdir(errname), delimiter,
1334 qualify(name, 'comp.stderr'),
1335 qualify(name, 'run.stderr'))
1336
1337 # check the exit code
1338 if exit_code != getTestOpts().exit_code:
1339 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1340 dump_stdout(name)
1341 dump_stderr(name)
1342 return failBecause('bad exit code')
1343
1344 # ToDo: if the sub-shell was killed by ^C, then exit
1345
1346 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1347 check_stdout_ok(name)):
1348 return passed()
1349 else:
1350 return failBecause('bad stdout or stderr')
1351
1352
1353 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1354 infile = open(in_fn)
1355 out1 = open(out1_fn, 'w')
1356 out2 = open(out2_fn, 'w')
1357
1358 line = infile.readline()
1359 line = re.sub('\r', '', line) # ignore Windows EOL
1360 while (re.sub('^\s*','',line) != delimiter and line != ''):
1361 out1.write(line)
1362 line = infile.readline()
1363 line = re.sub('\r', '', line)
1364 out1.close()
1365
1366 line = infile.readline()
1367 while (line != ''):
1368 out2.write(line)
1369 line = infile.readline()
1370 out2.close()
1371
1372 # -----------------------------------------------------------------------------
1373 # Generate External Core for the given program, then compile the resulting Core
1374 # and compare its output to the expected output
1375
1376 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1377
1378 depsfilename = qualify(name, 'deps')
1379 errname = add_suffix(name, 'comp.stderr')
1380 qerrname = qualify(errname,'')
1381
1382 hcname = qualify(name, 'hc')
1383 oname = qualify(name, 'o')
1384
1385 rm_no_fail( qerrname )
1386 rm_no_fail( qualify(name, '') )
1387
1388 if (top_mod == ''):
1389 srcname = add_hs_lhs_suffix(name)
1390 else:
1391 srcname = top_mod
1392
1393 qcorefilename = qualify(name, 'hcr')
1394 corefilename = add_suffix(name, 'hcr')
1395 rm_no_fail(qcorefilename)
1396
1397 # Generate External Core
1398
1399 if (top_mod == ''):
1400 to_do = ' ' + srcname + ' '
1401 else:
1402 to_do = ' --make ' + top_mod + ' '
1403
1404 flags = copy.copy(getTestOpts().compiler_always_flags)
1405 if getTestOpts().outputdir != None:
1406 flags.extend(["-outputdir", getTestOpts().outputdir])
1407 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1408 + config.compiler + "' " \
1409 + join(flags,' ') + ' ' \
1410 + join(config.way_flags(name)[way],' ') + ' ' \
1411 + extra_hc_opts + ' ' \
1412 + getTestOpts().extra_hc_opts \
1413 + to_do \
1414 + '>' + errname + ' 2>&1'
1415 result = runCmdFor(name, cmd)
1416
1417 exit_code = result >> 8
1418
1419 if exit_code != 0:
1420 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1421 if_verbose_dump(1,qerrname)
1422 return failBecause('ext core exit code non-0')
1423
1424 # Compile the resulting files -- if there's more than one module, we need to read the output
1425 # of the previous compilation in order to find the dependencies
1426 if (top_mod == ''):
1427 to_compile = corefilename
1428 else:
1429 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1430 deps = open(depsfilename).read()
1431 deplist = string.replace(deps, '\n',' ');
1432 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1433 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1434
1435 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1436 if getTestOpts().outputdir != None:
1437 flags.extend(["-outputdir", getTestOpts().outputdir])
1438
1439 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1440 + config.compiler + "' " \
1441 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1442 + to_compile + ' ' \
1443 + extra_hc_opts + ' ' \
1444 + getTestOpts().extra_hc_opts + ' ' \
1445 + flags \
1446 + ' -fglasgow-exts -o ' + name \
1447 + '>' + errname + ' 2>&1'
1448
1449 result = runCmdFor(name, cmd)
1450 exit_code = result >> 8
1451
1452 if exit_code != 0:
1453 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1454 if_verbose_dump(1,qerrname)
1455 return failBecause('ext core exit code non-0')
1456
1457 # Clean up
1458 rm_no_fail ( oname )
1459 rm_no_fail ( hcname )
1460 rm_no_fail ( qcorefilename )
1461 rm_no_fail ( depsfilename )
1462
1463 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1464
1465 # -----------------------------------------------------------------------------
1466 # Utils
1467
1468 def check_stdout_ok( name ):
1469 if getTestOpts().with_namebase == None:
1470 namebase = name
1471 else:
1472 namebase = getTestOpts().with_namebase
1473
1474 actual_stdout_file = qualify(name, 'run.stdout')
1475 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1476
1477 def norm(str):
1478 if platform_specific:
1479 return str
1480 else:
1481 return normalise_output(str)
1482
1483 return compare_outputs('stdout', \
1484 two_normalisers(norm, getTestOpts().extra_normaliser), \
1485 expected_stdout_file, actual_stdout_file)
1486
1487 def dump_stdout( name ):
1488 print 'Stdout:'
1489 print read_no_crs(qualify(name, 'run.stdout'))
1490
1491 def check_stderr_ok( name ):
1492 if getTestOpts().with_namebase == None:
1493 namebase = name
1494 else:
1495 namebase = getTestOpts().with_namebase
1496
1497 actual_stderr_file = qualify(name, 'run.stderr')
1498 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1499
1500 def norm(str):
1501 if platform_specific:
1502 return str
1503 else:
1504 return normalise_errmsg(str)
1505
1506 return compare_outputs('stderr', \
1507 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1508 expected_stderr_file, actual_stderr_file)
1509
1510 def dump_stderr( name ):
1511 print "Stderr:"
1512 print read_no_crs(qualify(name, 'run.stderr'))
1513
1514 def read_no_crs(file):
1515 str = ''
1516 try:
1517 h = open(file)
1518 str = h.read()
1519 h.close
1520 except:
1521 # On Windows, if the program fails very early, it seems the
1522 # files stdout/stderr are redirected to may not get created
1523 pass
1524 return re.sub('\r', '', str)
1525
1526 def write_file(file, str):
1527 h = open(file, 'w')
1528 h.write(str)
1529 h.close
1530
1531 def check_hp_ok(name):
1532
1533 # do not qualify for hp2ps because we should be in the right directory
1534 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1535
1536 hp2psResult = runCmdExitCode(hp2psCmd)
1537
1538 actual_ps_file = qualify(name, 'ps')
1539
1540 if(hp2psResult == 0):
1541 if (os.path.exists(actual_ps_file)):
1542 if gs_working:
1543 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1544 if (gsResult == 0):
1545 return (True)
1546 else:
1547 print "hp2ps output for " + name + "is not valid PostScript"
1548 else: return (True) # assume postscript is valid without ghostscript
1549 else:
1550 print "hp2ps did not generate PostScript for " + name
1551 return (False)
1552 else:
1553 print "hp2ps error when processing heap profile for " + name
1554 return(False)
1555
1556 def check_prof_ok(name):
1557
1558 prof_file = qualify(name,'prof')
1559
1560 if not os.path.exists(prof_file):
1561 print prof_file + " does not exist"
1562 return(False)
1563
1564 if os.path.getsize(qualify(name,'prof')) == 0:
1565 print prof_file + " is empty"
1566 return(False)
1567
1568 if getTestOpts().with_namebase == None:
1569 namebase = name
1570 else:
1571 namebase = getTestOpts().with_namebase
1572
1573 (platform_specific, expected_prof_file) = \
1574 platform_wordsize_qualify(namebase, 'prof.sample')
1575
1576 # sample prof file is not required
1577 if not os.path.exists(expected_prof_file):
1578 return True
1579 else:
1580 return compare_outputs('prof', \
1581 two_normalisers(normalise_whitespace,normalise_prof), \
1582 expected_prof_file, prof_file)
1583
1584 # Compare expected output to actual output, and optionally accept the
1585 # new output. Returns true if output matched or was accepted, false
1586 # otherwise.
1587 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1588 if os.path.exists(expected_file):
1589 expected_raw = read_no_crs(expected_file)
1590 # print "norm:", normaliser(expected_raw)
1591 expected_str = normaliser(expected_raw)
1592 expected_file_for_diff = expected_file
1593 else:
1594 expected_str = ''
1595 expected_file_for_diff = '/dev/null'
1596
1597 actual_raw = read_no_crs(actual_file)
1598 actual_str = normaliser(actual_raw)
1599
1600 if expected_str == actual_str:
1601 return 1
1602 else:
1603 print 'Actual ' + kind + ' output differs from expected:'
1604
1605 if expected_file_for_diff == '/dev/null':
1606 expected_normalised_file = '/dev/null'
1607 else:
1608 expected_normalised_file = expected_file + ".normalised"
1609 write_file(expected_normalised_file, expected_str)
1610
1611 actual_normalised_file = actual_file + ".normalised"
1612 write_file(actual_normalised_file, actual_str)
1613
1614 # Ignore whitespace when diffing. We should only get to this
1615 # point if there are non-whitespace differences
1616 #
1617 # Note we are diffing the *actual* output, not the normalised
1618 # output. The normalised output may have whitespace squashed
1619 # (including newlines) so the diff would be hard to read.
1620 # This does mean that the diff might contain changes that
1621 # would be normalised away.
1622 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1623 ' ' + actual_file )
1624
1625 # If for some reason there were no non-whitespace differences,
1626 # then do a full diff
1627 if r == 0:
1628 r = os.system( 'diff -u ' + expected_file_for_diff + \
1629 ' ' + actual_file )
1630
1631 if config.accept:
1632 print 'Accepting new output.'
1633 write_file(expected_file, actual_raw)
1634 return 1
1635 else:
1636 return 0
1637
1638
1639 def normalise_whitespace( str ):
1640 # Merge contiguous whitespace characters into a single space.
1641 str = re.sub('[ \t\n]+', ' ', str)
1642 return str
1643
1644 def normalise_errmsg( str ):
1645 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1646 # the colon is there because it appears in error messages; this
1647 # hacky solution is used in place of more sophisticated filename
1648 # mangling
1649 str = re.sub('([^\\s])\\.exe', '\\1', str)
1650 # normalise slashes, minimise Windows/Unix filename differences
1651 str = re.sub('\\\\', '/', str)
1652 # The inplace ghc's are called ghc-stage[123] to avoid filename
1653 # collisions, so we need to normalise that to just "ghc"
1654 str = re.sub('ghc-stage[123]', 'ghc', str)
1655 # We sometimes see the name of the integer-gmp package on stderr,
1656 # but this can change (either the implementation name or the
1657 # version number), so we canonicalise it here
1658 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1659 return str
1660
1661 # normalise a .prof file, so that we can reasonably compare it against
1662 # a sample. This doesn't compare any of the actual profiling data,
1663 # only the shape of the profile and the number of entries.
1664 def normalise_prof (str):
1665 # strip everything up to the line beginning "COST CENTRE"
1666 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1667
1668 # strip results for CAFs, these tend to change unpredictably
1669 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1670
1671 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1672 # sometimes under MAIN.
1673 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1674
1675 # We have somthing like this:
1676
1677 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1678 # k Main 204 1 0.0 0.0 0.0 0.0
1679 # foo Main 205 1 0.0 0.0 0.0 0.0
1680 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1681
1682 # then we remove all the specific profiling data, leaving only the
1683 # cost centre name, module, and entries, to end up with this:
1684
1685 # MAIN MAIN 0
1686 # k Main 1
1687 # foo Main 1
1688 # foo.bar Main 1
1689
1690 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1691 return str
1692
1693 def normalise_slashes_( str ):
1694 str = re.sub('\\\\', '/', str)
1695 return str
1696
1697 def normalise_exe_( str ):
1698 str = re.sub('\.exe', '', str)
1699 return str
1700
1701 def normalise_output( str ):
1702 # Remove a .exe extension (for Windows)
1703 # This can occur in error messages generated by the program.
1704 str = re.sub('([^\\s])\\.exe', '\\1', str)
1705 return str
1706
1707 def normalise_asm( str ):
1708 lines = str.split('\n')
1709 # Only keep instructions and labels not starting with a dot.
1710 metadata = re.compile('^[ \t]*\\..*$')
1711 out = []
1712 for line in lines:
1713 # Drop metadata directives (e.g. ".type")
1714 if not metadata.match(line):
1715 line = re.sub('@plt', '', line)
1716 instr = line.lstrip().split()
1717 # Drop empty lines.
1718 if not instr:
1719 continue
1720 # Drop operands, except for call instructions.
1721 elif instr[0] == 'call':
1722 out.append(instr[0] + ' ' + instr[1])
1723 else:
1724 out.append(instr[0])
1725 out = '\n'.join(out)
1726 return out
1727
1728 def if_verbose( n, str ):
1729 if config.verbose >= n:
1730 print str
1731
1732 def if_verbose_dump( n, f ):
1733 if config.verbose >= n:
1734 try:
1735 print open(f).read()
1736 except:
1737 print ''
1738
1739 def rawSystem(cmd_and_args):
1740 # We prefer subprocess.call to os.spawnv as the latter
1741 # seems to send its arguments through a shell or something
1742 # with the Windows (non-cygwin) python. An argument "a b c"
1743 # turns into three arguments ["a", "b", "c"].
1744
1745 # However, subprocess is new in python 2.4, so fall back to
1746 # using spawnv if we don't have it
1747
1748 if have_subprocess:
1749 return subprocess.call(cmd_and_args)
1750 else:
1751 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1752
1753 # Note that this doesn't handle the timeout itself; it is just used for
1754 # commands that have timeout handling built-in.
1755 def rawSystemWithTimeout(cmd_and_args):
1756 r = rawSystem(cmd_and_args)
1757 if r == 98:
1758 # The python timeout program uses 98 to signal that ^C was pressed
1759 stopNow()
1760 return r
1761
1762 # cmd is a complex command in Bourne-shell syntax
1763 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1764 # Hence it must ultimately be run by a Bourne shell
1765 #
1766 # Mostly it invokes the command wrapped in 'timeout' thus
1767 # timeout 300 'cd . && ...blah blah'
1768 # so it's timeout's job to invoke the Bourne shell
1769 #
1770 # But watch out for the case when there is no timeout program!
1771 # Then, when using the native Python, os.system will invoke the cmd shell
1772
1773 def runCmd( cmd ):
1774 if_verbose( 1, cmd )
1775 r = 0
1776 if config.os == 'mingw32':
1777 # On MinGW, we will always have timeout
1778 assert config.timeout_prog!=''
1779
1780 if config.timeout_prog != '':
1781 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1782 else:
1783 r = os.system(cmd)
1784 return r << 8
1785
1786 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1787 if_verbose( 1, cmd )
1788 r = 0
1789 if config.os == 'mingw32':
1790 # On MinGW, we will always have timeout
1791 assert config.timeout_prog!=''
1792 timeout = int(ceil(config.timeout * timeout_multiplier))
1793
1794 if config.timeout_prog != '':
1795 if config.check_files_written:
1796 fn = name + ".strace"
1797 r = rawSystemWithTimeout(
1798 ["strace", "-o", fn, "-fF",
1799 "-e", "creat,open,chdir,clone,vfork",
1800 config.timeout_prog, str(timeout), cmd])
1801 addTestFilesWritten(name, fn)
1802 rm_no_fail(fn)
1803 else:
1804 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1805 else:
1806 r = os.system(cmd)
1807 return r << 8
1808
1809 def runCmdExitCode( cmd ):
1810 return (runCmd(cmd) >> 8);
1811
1812
1813 # -----------------------------------------------------------------------------
1814 # checking for files being written to by multiple tests
1815
1816 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1817 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1818 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1819 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1820 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1821 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1822 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1823 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1824 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1825 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1826 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1827 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1828 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1829 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1830 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1831 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1832 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1833
1834 # Files that are read or written but shouldn't be:
1835 # * ghci_history shouldn't be read or written by tests
1836 # * things under package.conf.d shouldn't be written by tests
1837 bad_file_usages = {}
1838
1839 # Mapping from tests to the list of files that they write
1840 files_written = {}
1841
1842 # Mapping from tests to the list of files that they write but don't clean
1843 files_written_not_removed = {}
1844
1845 def add_bad_file_usage(name, file):
1846 try:
1847 if not file in bad_file_usages[name]:
1848 bad_file_usages[name].append(file)
1849 except:
1850 bad_file_usages[name] = [file]
1851
1852 def mkPath(curdir, path):
1853 # Given the current full directory is 'curdir', what is the full
1854 # path to 'path'?
1855 return os.path.realpath(os.path.join(curdir, path))
1856
1857 def addTestFilesWritten(name, fn):
1858 if config.use_threads:
1859 with t.lockFilesWritten:
1860 addTestFilesWrittenHelper(name, fn)
1861 else:
1862 addTestFilesWrittenHelper(name, fn)
1863
1864 def addTestFilesWrittenHelper(name, fn):
1865 started = False
1866 working_directories = {}
1867
1868 with open(fn, 'r') as f:
1869 for line in f:
1870 m_pid = re_strace_pid.match(line)
1871 if m_pid:
1872 pid = m_pid.group(1)
1873 content = m_pid.group(2)
1874 elif re_strace_unavailable.match(line):
1875 next
1876 else:
1877 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1878
1879 m_open = re_strace_open.match(content)
1880 m_chdir = re_strace_chdir.match(content)
1881 m_clone = re_strace_clone.match(content)
1882 m_vfork = re_strace_vfork.match(content)
1883
1884 if not started:
1885 working_directories[pid] = os.getcwd()
1886 started = True
1887
1888 if m_open:
1889 file = m_open.group(1)
1890 file = mkPath(working_directories[pid], file)
1891 if file.endswith("ghci_history"):
1892 add_bad_file_usage(name, file)
1893 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1894 flags = m_open.group(2).split('|')
1895 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1896 if re.match('package\.conf\.d', file):
1897 add_bad_file_usage(name, file)
1898 else:
1899 try:
1900 if not file in files_written[name]:
1901 files_written[name].append(file)
1902 except:
1903 files_written[name] = [file]
1904 elif 'O_RDONLY' in flags:
1905 pass
1906 else:
1907 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1908 elif m_chdir:
1909 # We optimistically assume that unfinished chdir's are going to succeed
1910 dir = m_chdir.group(1)
1911 working_directories[pid] = mkPath(working_directories[pid], dir)
1912 elif m_clone:
1913 working_directories[m_clone.group(2)] = working_directories[pid]
1914 elif m_vfork:
1915 working_directories[m_vfork.group(2)] = working_directories[pid]
1916 elif re_strace_open_resumed.match(content):
1917 pass
1918 elif re_strace_chdir_resumed.match(content):
1919 pass
1920 elif re_strace_vfork_unfinished.match(content):
1921 pass
1922 elif re_strace_clone_unfinished.match(content):
1923 pass
1924 elif re_strace_ignore_sigchild.match(content):
1925 pass
1926 elif re_strace_ignore_sigvtalarm.match(content):
1927 pass
1928 elif re_strace_ignore_sigint.match(content):
1929 pass
1930 elif re_strace_ignore_sigfpe.match(content):
1931 pass
1932 elif re_strace_ignore_sigsegv.match(content):
1933 pass
1934 elif re_strace_ignore_sigpipe.match(content):
1935 pass
1936 else:
1937 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1938
1939 def checkForFilesWrittenProblems(file):
1940 foundProblem = False
1941
1942 files_written_inverted = {}
1943 for t in files_written.keys():
1944 for f in files_written[t]:
1945 try:
1946 files_written_inverted[f].append(t)
1947 except:
1948 files_written_inverted[f] = [t]
1949
1950 for f in files_written_inverted.keys():
1951 if len(files_written_inverted[f]) > 1:
1952 if not foundProblem:
1953 foundProblem = True
1954 file.write("\n")
1955 file.write("\nSome files are written by multiple tests:\n")
1956 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1957 if foundProblem:
1958 file.write("\n")
1959
1960 # -----
1961
1962 if len(files_written_not_removed) > 0:
1963 file.write("\n")
1964 file.write("\nSome files written but not removed:\n")
1965 tests = files_written_not_removed.keys()
1966 tests.sort()
1967 for t in tests:
1968 for f in files_written_not_removed[t]:
1969 file.write(" " + t + ": " + f + "\n")
1970 file.write("\n")
1971
1972 # -----
1973
1974 if len(bad_file_usages) > 0:
1975 file.write("\n")
1976 file.write("\nSome bad file usages:\n")
1977 tests = bad_file_usages.keys()
1978 tests.sort()
1979 for t in tests:
1980 for f in bad_file_usages[t]:
1981 file.write(" " + t + ": " + f + "\n")
1982 file.write("\n")
1983
1984 # -----------------------------------------------------------------------------
1985 # checking if ghostscript is available for checking the output of hp2ps
1986
1987 def genGSCmd(psfile):
1988 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1989
1990 def gsNotWorking():
1991 global gs_working
1992 print "GhostScript not available for hp2ps tests"
1993
1994 global gs_working
1995 gs_working = 0
1996 if config.have_profiling:
1997 if config.gs != '':
1998 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
1999 if resultGood == 0:
2000 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2001 if resultBad != 0:
2002 print "GhostScript available for hp2ps tests"
2003 gs_working = 1;
2004 else:
2005 gsNotWorking();
2006 else:
2007 gsNotWorking();
2008 else:
2009 gsNotWorking();
2010
2011 def rm_no_fail( file ):
2012 try:
2013 os.remove( file )
2014 finally:
2015 return
2016
2017 def add_suffix( name, suffix ):
2018 if suffix == '':
2019 return name
2020 else:
2021 return name + '.' + suffix
2022
2023 def add_hs_lhs_suffix(name):
2024 if getTestOpts().c_src:
2025 return add_suffix(name, 'c')
2026 elif getTestOpts().cmm_src:
2027 return add_suffix(name, 'cmm')
2028 elif getTestOpts().objc_src:
2029 return add_suffix(name, 'm')
2030 elif getTestOpts().objcpp_src:
2031 return add_suffix(name, 'mm')
2032 elif getTestOpts().literate:
2033 return add_suffix(name, 'lhs')
2034 else:
2035 return add_suffix(name, 'hs')
2036
2037 def replace_suffix( name, suffix ):
2038 base, suf = os.path.splitext(name)
2039 return base + '.' + suffix
2040
2041 def in_testdir( name ):
2042 return (getTestOpts().testdir + '/' + name)
2043
2044 def qualify( name, suff ):
2045 return in_testdir(add_suffix(name, suff))
2046
2047
2048 # Finding the sample output. The filename is of the form
2049 #
2050 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2051 #
2052 # and we pick the most specific version available. The <version> is
2053 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2054 # more fine-grained control use if_compiler_lt().
2055 #
2056 def platform_wordsize_qualify( name, suff ):
2057
2058 basepath = qualify(name, suff)
2059
2060 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2061 for (platformSpecific, plat) in [(1, '-' + config.platform),
2062 (1, '-' + config.os),
2063 (0, '')]
2064 for ws in ['-ws-' + config.wordsize, '']
2065 for comp in ['-' + config.compiler_type, '']
2066 for vers in ['-' + config.compiler_maj_version, '']]
2067
2068 dir = glob.glob(basepath + '*')
2069 dir = map (lambda d: normalise_slashes_(d), dir)
2070
2071 for (platformSpecific, f) in paths:
2072 if f in dir:
2073 return (platformSpecific,f)
2074
2075 return (0, basepath)
2076
2077 # Clean up prior to the test, so that we can't spuriously conclude
2078 # that it passed on the basis of old run outputs.
2079 def pretest_cleanup(name):
2080 if getTestOpts().outputdir != None:
2081 odir = in_testdir(getTestOpts().outputdir)
2082 try:
2083 shutil.rmtree(odir)
2084 except:
2085 pass
2086 os.mkdir(odir)
2087
2088 rm_no_fail(qualify(name,'comp.stderr'))
2089 rm_no_fail(qualify(name,'run.stderr'))
2090 rm_no_fail(qualify(name,'run.stdout'))
2091 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2092 # simple_build zaps the following:
2093 # rm_nofail(qualify("o"))
2094 # rm_nofail(qualify(""))
2095 # not interested in the return code
2096
2097 # -----------------------------------------------------------------------------
2098 # Return a list of all the files ending in '.T' below the directory dir.
2099
2100 def findTFiles(roots):
2101 return concat(map(findTFiles_,roots))
2102
2103 def findTFiles_(path):
2104 if os.path.isdir(path):
2105 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2106 return findTFiles(paths)
2107 elif path[-2:] == '.T':
2108 return [path]
2109 else:
2110 return []
2111
2112 # -----------------------------------------------------------------------------
2113 # Output a test summary to the specified file object
2114
2115 def summary(t, file):
2116
2117 file.write('\n')
2118 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2119 file.write('OVERALL SUMMARY for test run started at '
2120 + t.start_time + '\n'
2121 + string.rjust(`t.total_tests`, 8)
2122 + ' total tests, which gave rise to\n'
2123 + string.rjust(`t.total_test_cases`, 8)
2124 + ' test cases, of which\n'
2125 + string.rjust(`t.n_tests_skipped`, 8)
2126 + ' were skipped\n'
2127 + '\n'
2128 + string.rjust(`t.n_missing_libs`, 8)
2129 + ' had missing libraries\n'
2130 + string.rjust(`t.n_expected_passes`, 8)
2131 + ' expected passes\n'
2132 + string.rjust(`t.n_expected_failures`, 8)
2133 + ' expected failures\n'
2134 + '\n'
2135 + string.rjust(`t.n_framework_failures`, 8)
2136 + ' caused framework failures\n'
2137 + string.rjust(`t.n_unexpected_passes`, 8)
2138 + ' unexpected passes\n'
2139 + string.rjust(`t.n_unexpected_failures`, 8)
2140 + ' unexpected failures\n'
2141 + '\n')
2142
2143 if t.n_unexpected_passes > 0:
2144 file.write('Unexpected passes:\n')
2145 printPassingTestInfosSummary(file, t.unexpected_passes)
2146
2147 if t.n_unexpected_failures > 0:
2148 file.write('Unexpected failures:\n')
2149 printFailingTestInfosSummary(file, t.unexpected_failures)
2150
2151 if config.check_files_written:
2152 checkForFilesWrittenProblems(file)
2153
2154 if stopping():
2155 file.write('WARNING: Testsuite run was terminated early\n')
2156
2157 def printUnexpectedTests(file, testInfoss):
2158 unexpected = []
2159 for testInfos in testInfoss:
2160 directories = testInfos.keys()
2161 for directory in directories:
2162 tests = testInfos[directory].keys()
2163 unexpected += tests
2164 if unexpected != []:
2165 file.write('Unexpected results from:\n')
2166 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2167 file.write('\n')
2168
2169 def printPassingTestInfosSummary(file, testInfos):
2170 directories = testInfos.keys()
2171 directories.sort()
2172 maxDirLen = max(map ((lambda x : len(x)), directories))
2173 for directory in directories:
2174 tests = testInfos[directory].keys()
2175 tests.sort()
2176 for test in tests:
2177 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2178 ' (' + join(testInfos[directory][test],',') + ')\n')
2179 file.write('\n')
2180
2181 def printFailingTestInfosSummary(file, testInfos):
2182 directories = testInfos.keys()
2183 directories.sort()
2184 maxDirLen = max(map ((lambda x : len(x)), directories))
2185 for directory in directories:
2186 tests = testInfos[directory].keys()
2187 tests.sort()
2188 for test in tests:
2189 reasons = testInfos[directory][test].keys()
2190 for reason in reasons:
2191 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2192 ' [' + reason + ']' + \
2193 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2194 file.write('\n')
2195
2196 def getStdout(cmd):
2197 if have_subprocess:
2198 p = subprocess.Popen(cmd,
2199 stdout=subprocess.PIPE,
2200 stderr=subprocess.PIPE)
2201 (stdout, stderr) = p.communicate()
2202 r = p.wait()
2203 if r != 0:
2204 raise Exception("Command failed: " + str(cmd))
2205 if stderr != '':
2206 raise Exception("stderr from command: " + str(cmd))
2207 return stdout
2208 else:
2209 raise Exception("Need subprocess to get stdout, but don't have it")