Flag to test suite: SKIP_PERF_TESTS
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 import types
20 from math import ceil, trunc
21
22 have_subprocess = False
23 try:
24 import subprocess
25 have_subprocess = True
26 except:
27 print "Warning: subprocess not found, will fall back to spawnv"
28
29 from string import join
30 from testglobals import *
31 from testutil import *
32
33 if config.use_threads:
34 import threading
35 import thread
36
37 global wantToStop
38 wantToStop = False
39 def stopNow():
40 global wantToStop
41 wantToStop = True
42 def stopping():
43 return wantToStop
44
45 # Options valid for the current test only (these get reset to
46 # testdir_testopts after each test).
47
48 global testopts_local
49 if config.use_threads:
50 testopts_local = threading.local()
51 else:
52 class TestOpts_Local:
53 pass
54 testopts_local = TestOpts_Local()
55
56 def getTestOpts():
57 return testopts_local.x
58
59 def setLocalTestOpts(opts):
60 global testopts_local
61 testopts_local.x=opts
62
63 def isStatsTest():
64 opts = getTestOpts()
65 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
66
67
68 # This can be called at the top of a file of tests, to set default test options
69 # for the following tests.
70 def setTestOpts( f ):
71 global thisdir_settings
72 thisdir_settings = [thisdir_settings, f]
73
74 # -----------------------------------------------------------------------------
75 # Canned setup functions for common cases. eg. for a test you might say
76 #
77 # test('test001', normal, compile, [''])
78 #
79 # to run it without any options, but change it to
80 #
81 # test('test001', expect_fail, compile, [''])
82 #
83 # to expect failure for this test.
84
85 def normal( name, opts ):
86 return;
87
88 def skip( name, opts ):
89 opts.skip = 1
90
91 def expect_fail( name, opts ):
92 opts.expect = 'fail';
93
94 def reqlib( lib ):
95 return lambda name, opts, l=lib: _reqlib (name, opts, l )
96
97 # Cache the results of looking to see if we have a library or not.
98 # This makes quite a difference, especially on Windows.
99 have_lib = {}
100
101 def _reqlib( name, opts, lib ):
102 if have_lib.has_key(lib):
103 got_it = have_lib[lib]
104 else:
105 if have_subprocess:
106 # By preference we use subprocess, as the alternative uses
107 # /dev/null which mingw doesn't have.
108 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 stdout=subprocess.PIPE,
110 stderr=subprocess.PIPE)
111 # read from stdout and stderr to avoid blocking due to
112 # buffers filling
113 p.communicate()
114 r = p.wait()
115 else:
116 r = os.system(config.ghc_pkg + ' describe ' + lib
117 + ' > /dev/null 2> /dev/null')
118 got_it = r == 0
119 have_lib[lib] = got_it
120
121 if not got_it:
122 opts.expect = 'missing-lib'
123
124 def req_profiling( name, opts ):
125 if not config.have_profiling:
126 opts.expect = 'fail'
127
128 def req_shared_libs( name, opts ):
129 if not config.have_shared_libs:
130 opts.expect = 'fail'
131
132 def req_interp( name, opts ):
133 if not config.have_interp:
134 opts.expect = 'fail'
135
136 def req_smp( name, opts ):
137 if not config.have_smp:
138 opts.expect = 'fail'
139
140 def ignore_output( name, opts ):
141 opts.ignore_output = 1
142
143 def no_stdin( name, opts ):
144 opts.no_stdin = 1
145
146 def combined_output( name, opts ):
147 opts.combined_output = True
148
149 # -----
150
151 def expect_fail_for( ways ):
152 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153
154 def _expect_fail_for( name, opts, ways ):
155 opts.expect_fail_for = ways
156
157 def expect_broken( bug ):
158 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
159
160 def _expect_broken( name, opts, bug ):
161 record_broken(name, opts, bug)
162 opts.expect = 'fail';
163
164 def expect_broken_for( bug, ways ):
165 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
166
167 def _expect_broken_for( name, opts, bug, ways ):
168 record_broken(name, opts, bug)
169 opts.expect_fail_for = ways
170
171 def record_broken(name, opts, bug):
172 global brokens
173 me = (bug, opts.testdir, name)
174 if not me in brokens:
175 brokens.append(me)
176
177 # -----
178
179 def omit_ways( ways ):
180 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181
182 def _omit_ways( name, opts, ways ):
183 opts.omit_ways = ways
184
185 # -----
186
187 def only_ways( ways ):
188 return lambda name, opts, w=ways: _only_ways( name, opts, w )
189
190 def _only_ways( name, opts, ways ):
191 opts.only_ways = ways
192
193 # -----
194
195 def extra_ways( ways ):
196 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197
198 def _extra_ways( name, opts, ways ):
199 opts.extra_ways = ways
200
201 # -----
202
203 def omit_compiler_types( compiler_types ):
204 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
205
206 def _omit_compiler_types( name, opts, compiler_types ):
207 if config.compiler_type in compiler_types:
208 opts.skip = 1
209
210 # -----
211
212 def only_compiler_types( compiler_types ):
213 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
214
215 def _only_compiler_types( name, opts, compiler_types ):
216 if config.compiler_type not in compiler_types:
217 opts.skip = 1
218
219 # -----
220
221 def set_stdin( file ):
222 return lambda name, opts, f=file: _set_stdin(name, opts, f);
223
224 def _set_stdin( name, opts, f ):
225 opts.stdin = f
226
227 # -----
228
229 def exit_code( val ):
230 return lambda name, opts, v=val: _exit_code(name, opts, v);
231
232 def _exit_code( name, opts, v ):
233 opts.exit_code = v
234
235 # -----
236
237 def timeout_multiplier( val ):
238 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239
240 def _timeout_multiplier( name, opts, v ):
241 opts.timeout_multiplier = v
242
243 # -----
244
245 def extra_run_opts( val ):
246 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247
248 def _extra_run_opts( name, opts, v ):
249 opts.extra_run_opts = v
250
251 # -----
252
253 def extra_hc_opts( val ):
254 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
255
256 def _extra_hc_opts( name, opts, v ):
257 opts.extra_hc_opts = v
258
259 # -----
260
261 def extra_clean( files ):
262 return lambda name, opts, v=files: _extra_clean(name, opts, v);
263
264 def _extra_clean( name, opts, v ):
265 opts.clean_files = v
266
267 # -----
268
269 def stats_num_field( field, expecteds ):
270 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
271
272 def _stats_num_field( name, opts, field, expecteds ):
273 if field in opts.stats_range_fields:
274 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
275
276 if type(expecteds) is types.ListType:
277 for (b, expected, dev) in expecteds:
278 if b:
279 opts.stats_range_fields[field] = (expected, dev)
280 return
281 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282
283 else:
284 (expected, dev) = expecteds
285 opts.stats_range_fields[field] = (expected, dev)
286
287 def compiler_stats_num_field( field, expecteds ):
288 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289
290 def _compiler_stats_num_field( name, opts, field, expecteds ):
291 if field in opts.compiler_stats_range_fields:
292 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
293
294 for (b, expected, dev) in expecteds:
295 if b:
296 opts.compiler_stats_range_fields[field] = (expected, dev)
297 return
298
299 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
300
301 # -----
302
303 def when(b, f):
304 # When list_brokens is on, we want to see all expect_broken calls,
305 # so we always do f
306 if b or config.list_broken:
307 return f
308 else:
309 return normal
310
311 def unless(b, f):
312 return when(not b, f)
313
314 def doing_ghci():
315 return 'ghci' in config.run_ways
316
317 def ghci_dynamic( ):
318 return config.ghc_dynamic
319
320 def fast():
321 return config.fast
322
323 def platform( plat ):
324 return config.platform == plat
325
326 def opsys( os ):
327 return config.os == os
328
329 def arch( arch ):
330 return config.arch == arch
331
332 def wordsize( ws ):
333 return config.wordsize == str(ws)
334
335 def msys( ):
336 return config.msys
337
338 def cygwin( ):
339 return config.cygwin
340
341 def have_vanilla( ):
342 return config.have_vanilla
343
344 def have_dynamic( ):
345 return config.have_dynamic
346
347 def have_profiling( ):
348 return config.have_profiling
349
350 def in_tree_compiler( ):
351 return config.in_tree_compiler
352
353 def compiler_type( compiler ):
354 return config.compiler_type == compiler
355
356 def compiler_lt( compiler, version ):
357 return config.compiler_type == compiler and \
358 version_lt(config.compiler_version, version)
359
360 def compiler_le( compiler, version ):
361 return config.compiler_type == compiler and \
362 version_le(config.compiler_version, version)
363
364 def compiler_gt( compiler, version ):
365 return config.compiler_type == compiler and \
366 version_gt(config.compiler_version, version)
367
368 def compiler_ge( compiler, version ):
369 return config.compiler_type == compiler and \
370 version_ge(config.compiler_version, version)
371
372 def unregisterised( ):
373 return config.unregisterised
374
375 def compiler_profiled( ):
376 return config.compiler_profiled
377
378 def compiler_debugged( ):
379 return config.compiler_debugged
380
381 def tag( t ):
382 return t in config.compiler_tags
383
384 # ---
385
386 def namebase( nb ):
387 return lambda opts, nb=nb: _namebase(opts, nb)
388
389 def _namebase( opts, nb ):
390 opts.with_namebase = nb
391
392 # ---
393
394 def high_memory_usage(name, opts):
395 opts.alone = True
396
397 # If a test is for a multi-CPU race, then running the test alone
398 # increases the chance that we'll actually see it.
399 def multi_cpu_race(name, opts):
400 opts.alone = True
401
402 # ---
403 def literate( name, opts ):
404 opts.literate = 1;
405
406 def c_src( name, opts ):
407 opts.c_src = 1;
408
409 def objc_src( name, opts ):
410 opts.objc_src = 1;
411
412 def objcpp_src( name, opts ):
413 opts.objcpp_src = 1;
414
415 def cmm_src( name, opts ):
416 opts.cmm_src = 1;
417
418 def outputdir( odir ):
419 return lambda name, opts, d=odir: _outputdir(name, opts, d)
420
421 def _outputdir( name, opts, odir ):
422 opts.outputdir = odir;
423
424 # ----
425
426 def pre_cmd( cmd ):
427 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
428
429 def _pre_cmd( name, opts, cmd ):
430 opts.pre_cmd = cmd
431
432 # ----
433
434 def clean_cmd( cmd ):
435 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
436
437 def _clean_cmd( name, opts, cmd ):
438 opts.clean_cmd = cmd
439
440 # ----
441
442 def cmd_prefix( prefix ):
443 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
444
445 def _cmd_prefix( name, opts, prefix ):
446 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
447
448 # ----
449
450 def cmd_wrapper( fun ):
451 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
452
453 def _cmd_wrapper( name, opts, fun ):
454 opts.cmd_wrapper = fun
455
456 # ----
457
458 def compile_cmd_prefix( prefix ):
459 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
460
461 def _compile_cmd_prefix( name, opts, prefix ):
462 opts.compile_cmd_prefix = prefix
463
464 # ----
465
466 def normalise_slashes( name, opts ):
467 opts.extra_normaliser = normalise_slashes_
468
469 def normalise_exe( name, opts ):
470 opts.extra_normaliser = normalise_exe_
471
472 def normalise_fun( fun ):
473 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
474
475 def _normalise_fun( name, opts, f ):
476 opts.extra_normaliser = f
477
478 def normalise_errmsg_fun( fun ):
479 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
480
481 def _normalise_errmsg_fun( name, opts, f ):
482 opts.extra_errmsg_normaliser = f
483
484 def two_normalisers(f, g):
485 return lambda x, f=f, g=g: f(g(x))
486
487 # ----
488 # Function for composing two opt-fns together
489
490 def executeSetups(fs, name, opts):
491 if type(fs) is types.ListType:
492 # If we have a list of setups, then execute each one
493 map (lambda f : executeSetups(f, name, opts), fs)
494 else:
495 # fs is a single function, so just apply it
496 fs(name, opts)
497
498 # -----------------------------------------------------------------------------
499 # The current directory of tests
500
501 def newTestDir( dir ):
502 global thisdir_settings
503 # reset the options for this test directory
504 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
505
506 def _newTestDir( name, opts, dir ):
507 opts.testdir = dir
508 opts.compiler_always_flags = config.compiler_always_flags
509
510 # -----------------------------------------------------------------------------
511 # Actually doing tests
512
513 parallelTests = []
514 aloneTests = []
515 allTestNames = set([])
516
517 def runTest (opts, name, func, args):
518 ok = 0
519
520 if config.use_threads:
521 t.thread_pool.acquire()
522 try:
523 while config.threads<(t.running_threads+1):
524 t.thread_pool.wait()
525 t.running_threads = t.running_threads+1
526 ok=1
527 t.thread_pool.release()
528 thread.start_new_thread(test_common_thread, (name, opts, func, args))
529 except:
530 if not ok:
531 t.thread_pool.release()
532 else:
533 test_common_work (name, opts, func, args)
534
535 # name :: String
536 # setup :: TestOpts -> IO ()
537 def test (name, setup, func, args):
538 global aloneTests
539 global parallelTests
540 global allTestNames
541 global thisdir_settings
542 if name in allTestNames:
543 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
544 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
545 framework_fail(name, 'bad_name', 'This test has an invalid name')
546
547 # Make a deep copy of the default_testopts, as we need our own copy
548 # of any dictionaries etc inside it. Otherwise, if one test modifies
549 # them, all tests will see the modified version!
550 myTestOpts = copy.deepcopy(default_testopts)
551
552 executeSetups([thisdir_settings, setup], name, myTestOpts)
553
554 thisTest = lambda : runTest(myTestOpts, name, func, args)
555 if myTestOpts.alone:
556 aloneTests.append(thisTest)
557 else:
558 parallelTests.append(thisTest)
559 allTestNames.add(name)
560
561 if config.use_threads:
562 def test_common_thread(name, opts, func, args):
563 t.lock.acquire()
564 try:
565 test_common_work(name,opts,func,args)
566 finally:
567 t.lock.release()
568 t.thread_pool.acquire()
569 t.running_threads = t.running_threads - 1
570 t.thread_pool.notify()
571 t.thread_pool.release()
572
573 def get_package_cache_timestamp():
574 if config.package_conf_cache_file == '':
575 return 0.0
576 else:
577 try:
578 return os.stat(config.package_conf_cache_file).st_mtime
579 except:
580 return 0.0
581
582
583 def test_common_work (name, opts, func, args):
584 try:
585 t.total_tests = t.total_tests+1
586 setLocalTestOpts(opts)
587
588 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
589
590 # All the ways we might run this test
591 if func == compile or func == multimod_compile:
592 all_ways = config.compile_ways
593 elif func == compile_and_run or func == multimod_compile_and_run:
594 all_ways = config.run_ways
595 elif func == ghci_script:
596 if 'ghci' in config.run_ways:
597 all_ways = ['ghci']
598 else:
599 all_ways = []
600 else:
601 all_ways = ['normal']
602
603 # A test itself can request extra ways by setting opts.extra_ways
604 all_ways = all_ways + filter(lambda way: way not in all_ways,
605 opts.extra_ways)
606
607 t.total_test_cases = t.total_test_cases + len(all_ways)
608
609 ok_way = lambda way: \
610 not getTestOpts().skip \
611 and (config.only == [] or name in config.only) \
612 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
613 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
614 and (not (config.skip_perf_tests and isStatsTest())) \
615 and way not in getTestOpts().omit_ways
616
617 # Which ways we are asked to skip
618 do_ways = filter (ok_way,all_ways)
619
620 # In fast mode, we skip all but one way
621 if config.fast and len(do_ways) > 0:
622 do_ways = [do_ways[0]]
623
624 if not config.clean_only:
625 # Run the required tests...
626 for way in do_ways:
627 if stopping():
628 break
629 do_test (name, way, func, args)
630
631 for way in all_ways:
632 if way not in do_ways:
633 skiptest (name,way)
634
635 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
636 pretest_cleanup(name)
637 clean(map (lambda suff: name + suff,
638 ['', '.exe', '.exe.manifest', '.genscript',
639 '.stderr.normalised', '.stdout.normalised',
640 '.run.stderr.normalised', '.run.stdout.normalised',
641 '.comp.stderr.normalised', '.comp.stdout.normalised',
642 '.interp.stderr.normalised', '.interp.stdout.normalised',
643 '.stats', '.comp.stats',
644 '.hi', '.o', '.prof', '.exe.prof', '.hc',
645 '_stub.h', '_stub.c', '_stub.o',
646 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
647
648 if func == multi_compile or func == multi_compile_fail:
649 extra_mods = args[1]
650 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
651 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
652
653 clean(getTestOpts().clean_files)
654
655 if getTestOpts().outputdir != None:
656 odir = in_testdir(getTestOpts().outputdir)
657 try:
658 shutil.rmtree(odir)
659 except:
660 pass
661
662 try:
663 shutil.rmtree(in_testdir('.hpc.' + name))
664 except:
665 pass
666
667 try:
668 cleanCmd = getTestOpts().clean_cmd
669 if cleanCmd != None:
670 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
671 if result != 0:
672 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
673 except e:
674 framework_fail(name, 'cleaning', 'clean-command exception')
675
676 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
677
678 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
679 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
680
681 try:
682 for f in files_written[name]:
683 if os.path.exists(f):
684 try:
685 if not f in files_written_not_removed[name]:
686 files_written_not_removed[name].append(f)
687 except:
688 files_written_not_removed[name] = [f]
689 except:
690 pass
691 except Exception, e:
692 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
693
694 def clean(strs):
695 for str in strs:
696 for name in glob.glob(in_testdir(str)):
697 clean_full_path(name)
698
699 def clean_full_path(name):
700 try:
701 # Remove files...
702 os.remove(name)
703 except OSError, e1:
704 try:
705 # ... and empty directories
706 os.rmdir(name)
707 except OSError, e2:
708 # We don't want to fail here, but we do want to know
709 # what went wrong, so print out the exceptions.
710 # ENOENT isn't a problem, though, as we clean files
711 # that don't necessarily exist.
712 if e1.errno != errno.ENOENT:
713 print e1
714 if e2.errno != errno.ENOENT:
715 print e2
716
717 def do_test(name, way, func, args):
718 full_name = name + '(' + way + ')'
719
720 try:
721 if_verbose(2, "=====> %s %d of %d %s " % \
722 (full_name, t.total_tests, len(allTestNames), \
723 [t.n_unexpected_passes, \
724 t.n_unexpected_failures, \
725 t.n_framework_failures]))
726
727 if config.use_threads:
728 t.lock.release()
729
730 try:
731 preCmd = getTestOpts().pre_cmd
732 if preCmd != None:
733 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
734 if result != 0:
735 framework_fail(name, way, 'pre-command failed: ' + str(result))
736 except e:
737 framework_fail(name, way, 'pre-command exception')
738
739 try:
740 result = apply(func, [name,way] + args)
741 finally:
742 if config.use_threads:
743 t.lock.acquire()
744
745 if getTestOpts().expect != 'pass' and \
746 getTestOpts().expect != 'fail' and \
747 getTestOpts().expect != 'missing-lib':
748 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
749
750 try:
751 passFail = result['passFail']
752 except:
753 passFail = 'No passFail found'
754
755 if passFail == 'pass':
756 if getTestOpts().expect == 'pass' \
757 and way not in getTestOpts().expect_fail_for:
758 t.n_expected_passes = t.n_expected_passes + 1
759 if name in t.expected_passes:
760 t.expected_passes[name].append(way)
761 else:
762 t.expected_passes[name] = [way]
763 else:
764 if_verbose(1, '*** unexpected pass for %s' % full_name)
765 t.n_unexpected_passes = t.n_unexpected_passes + 1
766 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
767 elif passFail == 'fail':
768 if getTestOpts().expect == 'pass' \
769 and way not in getTestOpts().expect_fail_for:
770 if_verbose(1, '*** unexpected failure for %s' % full_name)
771 t.n_unexpected_failures = t.n_unexpected_failures + 1
772 reason = result['reason']
773 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
774 else:
775 if getTestOpts().expect == 'missing-lib':
776 t.n_missing_libs = t.n_missing_libs + 1
777 if name in t.missing_libs:
778 t.missing_libs[name].append(way)
779 else:
780 t.missing_libs[name] = [way]
781 else:
782 t.n_expected_failures = t.n_expected_failures + 1
783 if name in t.expected_failures:
784 t.expected_failures[name].append(way)
785 else:
786 t.expected_failures[name] = [way]
787 else:
788 framework_fail(name, way, 'bad result ' + passFail)
789 except KeyboardInterrupt:
790 stopNow()
791 except:
792 framework_fail(name, way, 'do_test exception')
793 traceback.print_exc()
794
795 def addPassingTestInfo (testInfos, directory, name, way):
796 directory = re.sub('^\\.[/\\\\]', '', directory)
797
798 if not directory in testInfos:
799 testInfos[directory] = {}
800
801 if not name in testInfos[directory]:
802 testInfos[directory][name] = []
803
804 testInfos[directory][name].append(way)
805
806 def addFailingTestInfo (testInfos, directory, name, reason, way):
807 directory = re.sub('^\\.[/\\\\]', '', directory)
808
809 if not directory in testInfos:
810 testInfos[directory] = {}
811
812 if not name in testInfos[directory]:
813 testInfos[directory][name] = {}
814
815 if not reason in testInfos[directory][name]:
816 testInfos[directory][name][reason] = []
817
818 testInfos[directory][name][reason].append(way)
819
820 def skiptest (name, way):
821 # print 'Skipping test \"', name, '\"'
822 t.n_tests_skipped = t.n_tests_skipped + 1
823 if name in t.tests_skipped:
824 t.tests_skipped[name].append(way)
825 else:
826 t.tests_skipped[name] = [way]
827
828 def framework_fail( name, way, reason ):
829 full_name = name + '(' + way + ')'
830 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
831 t.n_framework_failures = t.n_framework_failures + 1
832 if name in t.framework_failures:
833 t.framework_failures[name].append(way)
834 else:
835 t.framework_failures[name] = [way]
836
837 def badResult(result):
838 try:
839 if result['passFail'] == 'pass':
840 return False
841 return True
842 except:
843 return True
844
845 def passed():
846 return {'passFail': 'pass'}
847
848 def failBecause(reason):
849 return {'passFail': 'fail', 'reason': reason}
850
851 # -----------------------------------------------------------------------------
852 # Generic command tests
853
854 # A generic command test is expected to run and exit successfully.
855 #
856 # The expected exit code can be changed via exit_code() as normal, and
857 # the expected stdout/stderr are stored in <testname>.stdout and
858 # <testname>.stderr. The output of the command can be ignored
859 # altogether by using run_command_ignore_output instead of
860 # run_command.
861
862 def run_command( name, way, cmd ):
863 return simple_run( name, '', cmd, '' )
864
865 # -----------------------------------------------------------------------------
866 # GHCi tests
867
868 def ghci_script( name, way, script ):
869 # filter out -fforce-recomp from compiler_always_flags, because we're
870 # actually testing the recompilation behaviour in the GHCi tests.
871 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
872 flags.append(getTestOpts().extra_hc_opts)
873 if getTestOpts().outputdir != None:
874 flags.extend(["-outputdir", getTestOpts().outputdir])
875
876 # We pass HC and HC_OPTS as environment variables, so that the
877 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
878 cmd = "HC='" + config.compiler + "' " + \
879 "HC_OPTS='" + join(flags,' ') + "' " + \
880 "'" + config.compiler + "'" + \
881 ' --interactive -v0 -ignore-dot-ghci ' + \
882 join(flags,' ')
883
884 getTestOpts().stdin = script
885 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
886
887 # -----------------------------------------------------------------------------
888 # Compile-only tests
889
890 def compile( name, way, extra_hc_opts ):
891 return do_compile( name, way, 0, '', [], extra_hc_opts )
892
893 def compile_fail( name, way, extra_hc_opts ):
894 return do_compile( name, way, 1, '', [], extra_hc_opts )
895
896 def multimod_compile( name, way, top_mod, extra_hc_opts ):
897 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
898
899 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
900 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
901
902 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
903 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
904
905 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
906 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
907
908 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
909 # print 'Compile only, extra args = ', extra_hc_opts
910 pretest_cleanup(name)
911
912 result = extras_build( way, extra_mods, extra_hc_opts )
913 if badResult(result):
914 return result
915 extra_hc_opts = result['hc_opts']
916
917 force = 0
918 if extra_mods:
919 force = 1
920 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
921
922 if badResult(result):
923 return result
924
925 # the actual stderr should always match the expected, regardless
926 # of whether we expected the compilation to fail or not (successful
927 # compilations may generate warnings).
928
929 if getTestOpts().with_namebase == None:
930 namebase = name
931 else:
932 namebase = getTestOpts().with_namebase
933
934 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
935 actual_stderr_file = qualify(name, 'comp.stderr')
936
937 if not compare_outputs('stderr', \
938 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
939 expected_stderr_file, actual_stderr_file):
940 return failBecause('stderr mismatch')
941
942 # no problems found, this test passed
943 return passed()
944
945 def compile_cmp_asm( name, way, extra_hc_opts ):
946 print 'Compile only, extra args = ', extra_hc_opts
947 pretest_cleanup(name)
948 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
949
950 if badResult(result):
951 return result
952
953 # the actual stderr should always match the expected, regardless
954 # of whether we expected the compilation to fail or not (successful
955 # compilations may generate warnings).
956
957 if getTestOpts().with_namebase == None:
958 namebase = name
959 else:
960 namebase = getTestOpts().with_namebase
961
962 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
963 actual_asm_file = qualify(name, 's')
964
965 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
966 expected_asm_file, actual_asm_file):
967 return failBecause('asm mismatch')
968
969 # no problems found, this test passed
970 return passed()
971
972 # -----------------------------------------------------------------------------
973 # Compile-and-run tests
974
975 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
976 # print 'Compile and run, extra args = ', extra_hc_opts
977 pretest_cleanup(name)
978
979 result = extras_build( way, extra_mods, extra_hc_opts )
980 if badResult(result):
981 return result
982 extra_hc_opts = result['hc_opts']
983
984 if way == 'ghci': # interpreted...
985 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
986 elif way == 'extcore' or way == 'optextcore' :
987 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
988 else: # compiled...
989 force = 0
990 if extra_mods:
991 force = 1
992
993 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
994 if badResult(result):
995 return result
996
997 cmd = './' + name;
998
999 # we don't check the compiler's stderr for a compile-and-run test
1000 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1001
1002 def compile_and_run( name, way, extra_hc_opts ):
1003 return compile_and_run__( name, way, '', [], extra_hc_opts)
1004
1005 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1006 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1007
1008 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1009 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1010
1011 def stats( name, way, stats_file ):
1012 opts = getTestOpts()
1013 return checkStats(stats_file, opts.stats_range_fields)
1014
1015 # -----------------------------------------------------------------------------
1016 # Check -t stats info
1017
1018 def checkStats(stats_file, range_fields):
1019 result = passed()
1020 if len(range_fields) > 0:
1021 f = open(in_testdir(stats_file))
1022 contents = f.read()
1023 f.close()
1024
1025 for (field, (expected, dev)) in range_fields.items():
1026 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1027 if m == None:
1028 print 'Failed to find field: ', field
1029 result = failBecause('no such stats field')
1030 val = int(m.group(1))
1031
1032 lowerBound = trunc( expected * ((100 - float(dev))/100));
1033 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1034
1035 if val < lowerBound:
1036 print field, 'value is too low:'
1037 print '(If this is because you have improved GHC, please'
1038 print 'update the test so that GHC doesn\'t regress again)'
1039 result = failBecause('stat too good')
1040 if val > upperBound:
1041 print field, 'value is too high:'
1042 result = failBecause('stat not good enough')
1043
1044 if val < lowerBound or val > upperBound:
1045 valStr = str(val)
1046 valLen = len(valStr)
1047 expectedStr = str(expected)
1048 expectedLen = len(expectedStr)
1049 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1050 def display(descr, val, extra):
1051 print descr, string.rjust(str(val), length), extra
1052 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1053 display(' Lower bound ' + field + ':', lowerBound, '')
1054 display(' Upper bound ' + field + ':', upperBound, '')
1055 display(' Actual ' + field + ':', val, '')
1056
1057 return result
1058
1059 # -----------------------------------------------------------------------------
1060 # Build a single-module program
1061
1062 def extras_build( way, extra_mods, extra_hc_opts ):
1063 for modopts in extra_mods:
1064 mod, opts = modopts
1065 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1066 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1067 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1068 if badResult(result):
1069 return result
1070
1071 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1072
1073
1074 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1075 opts = getTestOpts()
1076 errname = add_suffix(name, 'comp.stderr')
1077 rm_no_fail( qualify(errname, '') )
1078
1079 if top_mod != '':
1080 srcname = top_mod
1081 rm_no_fail( qualify(name, '') )
1082 base, suf = os.path.splitext(top_mod)
1083 rm_no_fail( qualify(base, '') )
1084 rm_no_fail( qualify(base, 'exe') )
1085 elif addsuf:
1086 srcname = add_hs_lhs_suffix(name)
1087 rm_no_fail( qualify(name, '') )
1088 else:
1089 srcname = name
1090 rm_no_fail( qualify(name, 'o') )
1091
1092 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1093
1094 to_do = ''
1095 if top_mod != '':
1096 to_do = '--make '
1097 if link:
1098 to_do = to_do + '-o ' + name
1099 elif link:
1100 to_do = '-o ' + name
1101 elif opts.compile_to_hc:
1102 to_do = '-C'
1103 else:
1104 to_do = '-c' # just compile
1105
1106 stats_file = name + '.comp.stats'
1107 if len(opts.compiler_stats_range_fields) > 0:
1108 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1109
1110 # Required by GHC 7.3+, harmless for earlier versions:
1111 if (getTestOpts().c_src or
1112 getTestOpts().objc_src or
1113 getTestOpts().objcpp_src or
1114 getTestOpts().cmm_src):
1115 extra_hc_opts += ' -no-hs-main '
1116
1117 if getTestOpts().compile_cmd_prefix == '':
1118 cmd_prefix = ''
1119 else:
1120 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1121
1122 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1123 if noforce:
1124 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1125 if getTestOpts().outputdir != None:
1126 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1127
1128 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1129 + config.compiler + "' " \
1130 + join(comp_flags,' ') + ' ' \
1131 + to_do + ' ' + srcname + ' ' \
1132 + join(config.way_flags(name)[way],' ') + ' ' \
1133 + extra_hc_opts + ' ' \
1134 + opts.extra_hc_opts + ' ' \
1135 + '>' + errname + ' 2>&1'
1136
1137 result = runCmdFor(name, cmd)
1138
1139 if result != 0 and not should_fail:
1140 actual_stderr = qualify(name, 'comp.stderr')
1141 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1142 if_verbose_dump(1,actual_stderr)
1143
1144 # ToDo: if the sub-shell was killed by ^C, then exit
1145
1146 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1147
1148 if badResult(statsResult):
1149 return statsResult
1150
1151 if should_fail:
1152 if result == 0:
1153 return failBecause('exit code 0')
1154 else:
1155 if result != 0:
1156 return failBecause('exit code non-0')
1157
1158 return passed()
1159
1160 # -----------------------------------------------------------------------------
1161 # Run a program and check its output
1162 #
1163 # If testname.stdin exists, route input from that, else
1164 # from /dev/null. Route output to testname.run.stdout and
1165 # testname.run.stderr. Returns the exit code of the run.
1166
1167 def simple_run( name, way, prog, args ):
1168 opts = getTestOpts()
1169
1170 # figure out what to use for stdin
1171 if opts.stdin != '':
1172 use_stdin = opts.stdin
1173 else:
1174 stdin_file = add_suffix(name, 'stdin')
1175 if os.path.exists(in_testdir(stdin_file)):
1176 use_stdin = stdin_file
1177 else:
1178 use_stdin = '/dev/null'
1179
1180 run_stdout = add_suffix(name,'run.stdout')
1181 run_stderr = add_suffix(name,'run.stderr')
1182
1183 rm_no_fail(qualify(name,'run.stdout'))
1184 rm_no_fail(qualify(name,'run.stderr'))
1185 rm_no_fail(qualify(name, 'hp'))
1186 rm_no_fail(qualify(name,'ps'))
1187 rm_no_fail(qualify(name, 'prof'))
1188
1189 my_rts_flags = rts_flags(way)
1190
1191 stats_file = name + '.stats'
1192 if len(opts.stats_range_fields) > 0:
1193 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1194
1195 if opts.no_stdin:
1196 stdin_comes_from = ''
1197 else:
1198 stdin_comes_from = ' <' + use_stdin
1199
1200 if opts.combined_output:
1201 redirection = ' >' + run_stdout \
1202 + ' 2>&1'
1203 else:
1204 redirection = ' >' + run_stdout \
1205 + ' 2>' + run_stderr
1206
1207 cmd = prog + ' ' + args + ' ' \
1208 + my_rts_flags + ' ' \
1209 + stdin_comes_from \
1210 + redirection
1211
1212 if opts.cmd_wrapper != None:
1213 cmd = opts.cmd_wrapper(cmd);
1214
1215 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1216
1217 # run the command
1218 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1219
1220 exit_code = result >> 8
1221 signal = result & 0xff
1222
1223 # check the exit code
1224 if exit_code != opts.exit_code:
1225 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1226 dump_stdout(name)
1227 dump_stderr(name)
1228 return failBecause('bad exit code')
1229
1230 check_hp = my_rts_flags.find("-h") != -1
1231 check_prof = my_rts_flags.find("-p") != -1
1232
1233 if not opts.ignore_output:
1234 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1235 bad_stdout = not check_stdout_ok(name)
1236 if bad_stderr:
1237 return failBecause('bad stderr')
1238 if bad_stdout:
1239 return failBecause('bad stdout')
1240 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1241 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1242 return failBecause('bad heap profile')
1243 if check_prof and not check_prof_ok(name):
1244 return failBecause('bad profile')
1245
1246 return checkStats(stats_file, opts.stats_range_fields)
1247
1248 def rts_flags(way):
1249 if (way == ''):
1250 return ''
1251 else:
1252 args = config.way_rts_flags[way]
1253
1254 if args == []:
1255 return ''
1256 else:
1257 return '+RTS ' + join(args,' ') + ' -RTS'
1258
1259 # -----------------------------------------------------------------------------
1260 # Run a program in the interpreter and check its output
1261
1262 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1263 outname = add_suffix(name, 'interp.stdout')
1264 errname = add_suffix(name, 'interp.stderr')
1265 rm_no_fail(outname)
1266 rm_no_fail(errname)
1267 rm_no_fail(name)
1268
1269 if (top_mod == ''):
1270 srcname = add_hs_lhs_suffix(name)
1271 else:
1272 srcname = top_mod
1273
1274 scriptname = add_suffix(name, 'genscript')
1275 qscriptname = in_testdir(scriptname)
1276 rm_no_fail(qscriptname)
1277
1278 delimiter = '===== program output begins here\n'
1279
1280 script = open(qscriptname, 'w')
1281 if not compile_only:
1282 # set the prog name and command-line args to match the compiled
1283 # environment.
1284 script.write(':set prog ' + name + '\n')
1285 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1286 # Add marker lines to the stdout and stderr output files, so we
1287 # can separate GHCi's output from the program's.
1288 script.write(':! echo ' + delimiter)
1289 script.write(':! echo 1>&2 ' + delimiter)
1290 # Set stdout to be line-buffered to match the compiled environment.
1291 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1292 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1293 # in the event of an exception as for the compiled program.
1294 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1295 script.close()
1296
1297 # figure out what to use for stdin
1298 if getTestOpts().stdin != '':
1299 stdin_file = in_testdir(getTestOpts().stdin)
1300 else:
1301 stdin_file = qualify(name, 'stdin')
1302
1303 if os.path.exists(stdin_file):
1304 stdin = open(stdin_file, 'r')
1305 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1306
1307 script.close()
1308
1309 flags = copy.copy(getTestOpts().compiler_always_flags)
1310 if getTestOpts().outputdir != None:
1311 flags.extend(["-outputdir", getTestOpts().outputdir])
1312
1313 cmd = "'" + config.compiler + "' " \
1314 + join(flags,' ') + ' ' \
1315 + srcname + ' ' \
1316 + join(config.way_flags(name)[way],' ') + ' ' \
1317 + extra_hc_opts + ' ' \
1318 + getTestOpts().extra_hc_opts + ' ' \
1319 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1320
1321 if getTestOpts().cmd_wrapper != None:
1322 cmd = getTestOpts().cmd_wrapper(cmd);
1323
1324 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1325
1326 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1327
1328 exit_code = result >> 8
1329 signal = result & 0xff
1330
1331 # split the stdout into compilation/program output
1332 split_file(in_testdir(outname), delimiter,
1333 qualify(name, 'comp.stdout'),
1334 qualify(name, 'run.stdout'))
1335 split_file(in_testdir(errname), delimiter,
1336 qualify(name, 'comp.stderr'),
1337 qualify(name, 'run.stderr'))
1338
1339 # check the exit code
1340 if exit_code != getTestOpts().exit_code:
1341 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1342 dump_stdout(name)
1343 dump_stderr(name)
1344 return failBecause('bad exit code')
1345
1346 # ToDo: if the sub-shell was killed by ^C, then exit
1347
1348 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1349 check_stdout_ok(name)):
1350 return passed()
1351 else:
1352 return failBecause('bad stdout or stderr')
1353
1354
1355 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1356 infile = open(in_fn)
1357 out1 = open(out1_fn, 'w')
1358 out2 = open(out2_fn, 'w')
1359
1360 line = infile.readline()
1361 line = re.sub('\r', '', line) # ignore Windows EOL
1362 while (re.sub('^\s*','',line) != delimiter and line != ''):
1363 out1.write(line)
1364 line = infile.readline()
1365 line = re.sub('\r', '', line)
1366 out1.close()
1367
1368 line = infile.readline()
1369 while (line != ''):
1370 out2.write(line)
1371 line = infile.readline()
1372 out2.close()
1373
1374 # -----------------------------------------------------------------------------
1375 # Generate External Core for the given program, then compile the resulting Core
1376 # and compare its output to the expected output
1377
1378 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1379
1380 depsfilename = qualify(name, 'deps')
1381 errname = add_suffix(name, 'comp.stderr')
1382 qerrname = qualify(errname,'')
1383
1384 hcname = qualify(name, 'hc')
1385 oname = qualify(name, 'o')
1386
1387 rm_no_fail( qerrname )
1388 rm_no_fail( qualify(name, '') )
1389
1390 if (top_mod == ''):
1391 srcname = add_hs_lhs_suffix(name)
1392 else:
1393 srcname = top_mod
1394
1395 qcorefilename = qualify(name, 'hcr')
1396 corefilename = add_suffix(name, 'hcr')
1397 rm_no_fail(qcorefilename)
1398
1399 # Generate External Core
1400
1401 if (top_mod == ''):
1402 to_do = ' ' + srcname + ' '
1403 else:
1404 to_do = ' --make ' + top_mod + ' '
1405
1406 flags = copy.copy(getTestOpts().compiler_always_flags)
1407 if getTestOpts().outputdir != None:
1408 flags.extend(["-outputdir", getTestOpts().outputdir])
1409 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1410 + config.compiler + "' " \
1411 + join(flags,' ') + ' ' \
1412 + join(config.way_flags(name)[way],' ') + ' ' \
1413 + extra_hc_opts + ' ' \
1414 + getTestOpts().extra_hc_opts \
1415 + to_do \
1416 + '>' + errname + ' 2>&1'
1417 result = runCmdFor(name, cmd)
1418
1419 exit_code = result >> 8
1420
1421 if exit_code != 0:
1422 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1423 if_verbose_dump(1,qerrname)
1424 return failBecause('ext core exit code non-0')
1425
1426 # Compile the resulting files -- if there's more than one module, we need to read the output
1427 # of the previous compilation in order to find the dependencies
1428 if (top_mod == ''):
1429 to_compile = corefilename
1430 else:
1431 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1432 deps = open(depsfilename).read()
1433 deplist = string.replace(deps, '\n',' ');
1434 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1435 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1436
1437 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1438 if getTestOpts().outputdir != None:
1439 flags.extend(["-outputdir", getTestOpts().outputdir])
1440
1441 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1442 + config.compiler + "' " \
1443 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1444 + to_compile + ' ' \
1445 + extra_hc_opts + ' ' \
1446 + getTestOpts().extra_hc_opts + ' ' \
1447 + flags \
1448 + ' -fglasgow-exts -o ' + name \
1449 + '>' + errname + ' 2>&1'
1450
1451 result = runCmdFor(name, cmd)
1452 exit_code = result >> 8
1453
1454 if exit_code != 0:
1455 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1456 if_verbose_dump(1,qerrname)
1457 return failBecause('ext core exit code non-0')
1458
1459 # Clean up
1460 rm_no_fail ( oname )
1461 rm_no_fail ( hcname )
1462 rm_no_fail ( qcorefilename )
1463 rm_no_fail ( depsfilename )
1464
1465 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1466
1467 # -----------------------------------------------------------------------------
1468 # Utils
1469
1470 def check_stdout_ok( name ):
1471 if getTestOpts().with_namebase == None:
1472 namebase = name
1473 else:
1474 namebase = getTestOpts().with_namebase
1475
1476 actual_stdout_file = qualify(name, 'run.stdout')
1477 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1478
1479 def norm(str):
1480 if platform_specific:
1481 return str
1482 else:
1483 return normalise_output(str)
1484
1485 return compare_outputs('stdout', \
1486 two_normalisers(norm, getTestOpts().extra_normaliser), \
1487 expected_stdout_file, actual_stdout_file)
1488
1489 def dump_stdout( name ):
1490 print 'Stdout:'
1491 print read_no_crs(qualify(name, 'run.stdout'))
1492
1493 def check_stderr_ok( name ):
1494 if getTestOpts().with_namebase == None:
1495 namebase = name
1496 else:
1497 namebase = getTestOpts().with_namebase
1498
1499 actual_stderr_file = qualify(name, 'run.stderr')
1500 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1501
1502 def norm(str):
1503 if platform_specific:
1504 return str
1505 else:
1506 return normalise_errmsg(str)
1507
1508 return compare_outputs('stderr', \
1509 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1510 expected_stderr_file, actual_stderr_file)
1511
1512 def dump_stderr( name ):
1513 print "Stderr:"
1514 print read_no_crs(qualify(name, 'run.stderr'))
1515
1516 def read_no_crs(file):
1517 str = ''
1518 try:
1519 h = open(file)
1520 str = h.read()
1521 h.close
1522 except:
1523 # On Windows, if the program fails very early, it seems the
1524 # files stdout/stderr are redirected to may not get created
1525 pass
1526 return re.sub('\r', '', str)
1527
1528 def write_file(file, str):
1529 h = open(file, 'w')
1530 h.write(str)
1531 h.close
1532
1533 def check_hp_ok(name):
1534
1535 # do not qualify for hp2ps because we should be in the right directory
1536 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1537
1538 hp2psResult = runCmdExitCode(hp2psCmd)
1539
1540 actual_ps_file = qualify(name, 'ps')
1541
1542 if(hp2psResult == 0):
1543 if (os.path.exists(actual_ps_file)):
1544 if gs_working:
1545 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1546 if (gsResult == 0):
1547 return (True)
1548 else:
1549 print "hp2ps output for " + name + "is not valid PostScript"
1550 else: return (True) # assume postscript is valid without ghostscript
1551 else:
1552 print "hp2ps did not generate PostScript for " + name
1553 return (False)
1554 else:
1555 print "hp2ps error when processing heap profile for " + name
1556 return(False)
1557
1558 def check_prof_ok(name):
1559
1560 prof_file = qualify(name,'prof')
1561
1562 if not os.path.exists(prof_file):
1563 print prof_file + " does not exist"
1564 return(False)
1565
1566 if os.path.getsize(qualify(name,'prof')) == 0:
1567 print prof_file + " is empty"
1568 return(False)
1569
1570 if getTestOpts().with_namebase == None:
1571 namebase = name
1572 else:
1573 namebase = getTestOpts().with_namebase
1574
1575 (platform_specific, expected_prof_file) = \
1576 platform_wordsize_qualify(namebase, 'prof.sample')
1577
1578 # sample prof file is not required
1579 if not os.path.exists(expected_prof_file):
1580 return True
1581 else:
1582 return compare_outputs('prof', \
1583 two_normalisers(normalise_whitespace,normalise_prof), \
1584 expected_prof_file, prof_file)
1585
1586 # Compare expected output to actual output, and optionally accept the
1587 # new output. Returns true if output matched or was accepted, false
1588 # otherwise.
1589 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1590 if os.path.exists(expected_file):
1591 expected_raw = read_no_crs(expected_file)
1592 # print "norm:", normaliser(expected_raw)
1593 expected_str = normaliser(expected_raw)
1594 expected_file_for_diff = expected_file
1595 else:
1596 expected_str = ''
1597 expected_file_for_diff = '/dev/null'
1598
1599 actual_raw = read_no_crs(actual_file)
1600 actual_str = normaliser(actual_raw)
1601
1602 if expected_str == actual_str:
1603 return 1
1604 else:
1605 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1606
1607 if expected_file_for_diff == '/dev/null':
1608 expected_normalised_file = '/dev/null'
1609 else:
1610 expected_normalised_file = expected_file + ".normalised"
1611 write_file(expected_normalised_file, expected_str)
1612
1613 actual_normalised_file = actual_file + ".normalised"
1614 write_file(actual_normalised_file, actual_str)
1615
1616 # Ignore whitespace when diffing. We should only get to this
1617 # point if there are non-whitespace differences
1618 #
1619 # Note we are diffing the *actual* output, not the normalised
1620 # output. The normalised output may have whitespace squashed
1621 # (including newlines) so the diff would be hard to read.
1622 # This does mean that the diff might contain changes that
1623 # would be normalised away.
1624 if (config.verbose >= 1):
1625 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1626 ' ' + actual_file )
1627
1628 # If for some reason there were no non-whitespace differences,
1629 # then do a full diff
1630 if r == 0:
1631 r = os.system( 'diff -u ' + expected_file_for_diff + \
1632 ' ' + actual_file )
1633
1634 if config.accept:
1635 if_verbose(1, 'Accepting new output.')
1636 write_file(expected_file, actual_raw)
1637 return 1
1638 else:
1639 return 0
1640
1641
1642 def normalise_whitespace( str ):
1643 # Merge contiguous whitespace characters into a single space.
1644 str = re.sub('[ \t\n]+', ' ', str)
1645 return str
1646
1647 def normalise_errmsg( str ):
1648 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1649 # the colon is there because it appears in error messages; this
1650 # hacky solution is used in place of more sophisticated filename
1651 # mangling
1652 str = re.sub('([^\\s])\\.exe', '\\1', str)
1653 # normalise slashes, minimise Windows/Unix filename differences
1654 str = re.sub('\\\\', '/', str)
1655 # The inplace ghc's are called ghc-stage[123] to avoid filename
1656 # collisions, so we need to normalise that to just "ghc"
1657 str = re.sub('ghc-stage[123]', 'ghc', str)
1658 # We sometimes see the name of the integer-gmp package on stderr,
1659 # but this can change (either the implementation name or the
1660 # version number), so we canonicalise it here
1661 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1662 return str
1663
1664 # normalise a .prof file, so that we can reasonably compare it against
1665 # a sample. This doesn't compare any of the actual profiling data,
1666 # only the shape of the profile and the number of entries.
1667 def normalise_prof (str):
1668 # strip everything up to the line beginning "COST CENTRE"
1669 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1670
1671 # strip results for CAFs, these tend to change unpredictably
1672 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1673
1674 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1675 # sometimes under MAIN.
1676 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1677
1678 # We have somthing like this:
1679
1680 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1681 # k Main 204 1 0.0 0.0 0.0 0.0
1682 # foo Main 205 1 0.0 0.0 0.0 0.0
1683 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1684
1685 # then we remove all the specific profiling data, leaving only the
1686 # cost centre name, module, and entries, to end up with this:
1687
1688 # MAIN MAIN 0
1689 # k Main 1
1690 # foo Main 1
1691 # foo.bar Main 1
1692
1693 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1694 return str
1695
1696 def normalise_slashes_( str ):
1697 str = re.sub('\\\\', '/', str)
1698 return str
1699
1700 def normalise_exe_( str ):
1701 str = re.sub('\.exe', '', str)
1702 return str
1703
1704 def normalise_output( str ):
1705 # Remove a .exe extension (for Windows)
1706 # This can occur in error messages generated by the program.
1707 str = re.sub('([^\\s])\\.exe', '\\1', str)
1708 return str
1709
1710 def normalise_asm( str ):
1711 lines = str.split('\n')
1712 # Only keep instructions and labels not starting with a dot.
1713 metadata = re.compile('^[ \t]*\\..*$')
1714 out = []
1715 for line in lines:
1716 # Drop metadata directives (e.g. ".type")
1717 if not metadata.match(line):
1718 line = re.sub('@plt', '', line)
1719 instr = line.lstrip().split()
1720 # Drop empty lines.
1721 if not instr:
1722 continue
1723 # Drop operands, except for call instructions.
1724 elif instr[0] == 'call':
1725 out.append(instr[0] + ' ' + instr[1])
1726 else:
1727 out.append(instr[0])
1728 out = '\n'.join(out)
1729 return out
1730
1731 def if_verbose( n, str ):
1732 if config.verbose >= n:
1733 print str
1734
1735 def if_verbose_dump( n, f ):
1736 if config.verbose >= n:
1737 try:
1738 print open(f).read()
1739 except:
1740 print ''
1741
1742 def rawSystem(cmd_and_args):
1743 # We prefer subprocess.call to os.spawnv as the latter
1744 # seems to send its arguments through a shell or something
1745 # with the Windows (non-cygwin) python. An argument "a b c"
1746 # turns into three arguments ["a", "b", "c"].
1747
1748 # However, subprocess is new in python 2.4, so fall back to
1749 # using spawnv if we don't have it
1750
1751 if have_subprocess:
1752 return subprocess.call(cmd_and_args)
1753 else:
1754 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1755
1756 # Note that this doesn't handle the timeout itself; it is just used for
1757 # commands that have timeout handling built-in.
1758 def rawSystemWithTimeout(cmd_and_args):
1759 r = rawSystem(cmd_and_args)
1760 if r == 98:
1761 # The python timeout program uses 98 to signal that ^C was pressed
1762 stopNow()
1763 return r
1764
1765 # cmd is a complex command in Bourne-shell syntax
1766 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1767 # Hence it must ultimately be run by a Bourne shell
1768 #
1769 # Mostly it invokes the command wrapped in 'timeout' thus
1770 # timeout 300 'cd . && ...blah blah'
1771 # so it's timeout's job to invoke the Bourne shell
1772 #
1773 # But watch out for the case when there is no timeout program!
1774 # Then, when using the native Python, os.system will invoke the cmd shell
1775
1776 def runCmd( cmd ):
1777 if_verbose( 3, cmd )
1778 r = 0
1779 if config.os == 'mingw32':
1780 # On MinGW, we will always have timeout
1781 assert config.timeout_prog!=''
1782
1783 if config.timeout_prog != '':
1784 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1785 else:
1786 r = os.system(cmd)
1787 return r << 8
1788
1789 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1790 if_verbose( 3, cmd )
1791 r = 0
1792 if config.os == 'mingw32':
1793 # On MinGW, we will always have timeout
1794 assert config.timeout_prog!=''
1795 timeout = int(ceil(config.timeout * timeout_multiplier))
1796
1797 if config.timeout_prog != '':
1798 if config.check_files_written:
1799 fn = name + ".strace"
1800 r = rawSystemWithTimeout(
1801 ["strace", "-o", fn, "-fF",
1802 "-e", "creat,open,chdir,clone,vfork",
1803 config.timeout_prog, str(timeout), cmd])
1804 addTestFilesWritten(name, fn)
1805 rm_no_fail(fn)
1806 else:
1807 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1808 else:
1809 r = os.system(cmd)
1810 return r << 8
1811
1812 def runCmdExitCode( cmd ):
1813 return (runCmd(cmd) >> 8);
1814
1815
1816 # -----------------------------------------------------------------------------
1817 # checking for files being written to by multiple tests
1818
1819 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1820 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1821 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1822 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1823 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1824 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1825 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1826 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1827 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1828 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1829 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1830 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1831 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1832 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1833 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1834 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1835 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1836
1837 # Files that are read or written but shouldn't be:
1838 # * ghci_history shouldn't be read or written by tests
1839 # * things under package.conf.d shouldn't be written by tests
1840 bad_file_usages = {}
1841
1842 # Mapping from tests to the list of files that they write
1843 files_written = {}
1844
1845 # Mapping from tests to the list of files that they write but don't clean
1846 files_written_not_removed = {}
1847
1848 def add_bad_file_usage(name, file):
1849 try:
1850 if not file in bad_file_usages[name]:
1851 bad_file_usages[name].append(file)
1852 except:
1853 bad_file_usages[name] = [file]
1854
1855 def mkPath(curdir, path):
1856 # Given the current full directory is 'curdir', what is the full
1857 # path to 'path'?
1858 return os.path.realpath(os.path.join(curdir, path))
1859
1860 def addTestFilesWritten(name, fn):
1861 if config.use_threads:
1862 with t.lockFilesWritten:
1863 addTestFilesWrittenHelper(name, fn)
1864 else:
1865 addTestFilesWrittenHelper(name, fn)
1866
1867 def addTestFilesWrittenHelper(name, fn):
1868 started = False
1869 working_directories = {}
1870
1871 with open(fn, 'r') as f:
1872 for line in f:
1873 m_pid = re_strace_pid.match(line)
1874 if m_pid:
1875 pid = m_pid.group(1)
1876 content = m_pid.group(2)
1877 elif re_strace_unavailable.match(line):
1878 next
1879 else:
1880 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1881
1882 m_open = re_strace_open.match(content)
1883 m_chdir = re_strace_chdir.match(content)
1884 m_clone = re_strace_clone.match(content)
1885 m_vfork = re_strace_vfork.match(content)
1886
1887 if not started:
1888 working_directories[pid] = os.getcwd()
1889 started = True
1890
1891 if m_open:
1892 file = m_open.group(1)
1893 file = mkPath(working_directories[pid], file)
1894 if file.endswith("ghci_history"):
1895 add_bad_file_usage(name, file)
1896 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1897 flags = m_open.group(2).split('|')
1898 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1899 if re.match('package\.conf\.d', file):
1900 add_bad_file_usage(name, file)
1901 else:
1902 try:
1903 if not file in files_written[name]:
1904 files_written[name].append(file)
1905 except:
1906 files_written[name] = [file]
1907 elif 'O_RDONLY' in flags:
1908 pass
1909 else:
1910 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1911 elif m_chdir:
1912 # We optimistically assume that unfinished chdir's are going to succeed
1913 dir = m_chdir.group(1)
1914 working_directories[pid] = mkPath(working_directories[pid], dir)
1915 elif m_clone:
1916 working_directories[m_clone.group(2)] = working_directories[pid]
1917 elif m_vfork:
1918 working_directories[m_vfork.group(2)] = working_directories[pid]
1919 elif re_strace_open_resumed.match(content):
1920 pass
1921 elif re_strace_chdir_resumed.match(content):
1922 pass
1923 elif re_strace_vfork_unfinished.match(content):
1924 pass
1925 elif re_strace_clone_unfinished.match(content):
1926 pass
1927 elif re_strace_ignore_sigchild.match(content):
1928 pass
1929 elif re_strace_ignore_sigvtalarm.match(content):
1930 pass
1931 elif re_strace_ignore_sigint.match(content):
1932 pass
1933 elif re_strace_ignore_sigfpe.match(content):
1934 pass
1935 elif re_strace_ignore_sigsegv.match(content):
1936 pass
1937 elif re_strace_ignore_sigpipe.match(content):
1938 pass
1939 else:
1940 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1941
1942 def checkForFilesWrittenProblems(file):
1943 foundProblem = False
1944
1945 files_written_inverted = {}
1946 for t in files_written.keys():
1947 for f in files_written[t]:
1948 try:
1949 files_written_inverted[f].append(t)
1950 except:
1951 files_written_inverted[f] = [t]
1952
1953 for f in files_written_inverted.keys():
1954 if len(files_written_inverted[f]) > 1:
1955 if not foundProblem:
1956 foundProblem = True
1957 file.write("\n")
1958 file.write("\nSome files are written by multiple tests:\n")
1959 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
1960 if foundProblem:
1961 file.write("\n")
1962
1963 # -----
1964
1965 if len(files_written_not_removed) > 0:
1966 file.write("\n")
1967 file.write("\nSome files written but not removed:\n")
1968 tests = files_written_not_removed.keys()
1969 tests.sort()
1970 for t in tests:
1971 for f in files_written_not_removed[t]:
1972 file.write(" " + t + ": " + f + "\n")
1973 file.write("\n")
1974
1975 # -----
1976
1977 if len(bad_file_usages) > 0:
1978 file.write("\n")
1979 file.write("\nSome bad file usages:\n")
1980 tests = bad_file_usages.keys()
1981 tests.sort()
1982 for t in tests:
1983 for f in bad_file_usages[t]:
1984 file.write(" " + t + ": " + f + "\n")
1985 file.write("\n")
1986
1987 # -----------------------------------------------------------------------------
1988 # checking if ghostscript is available for checking the output of hp2ps
1989
1990 def genGSCmd(psfile):
1991 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
1992
1993 def gsNotWorking():
1994 global gs_working
1995 print "GhostScript not available for hp2ps tests"
1996
1997 global gs_working
1998 gs_working = 0
1999 if config.have_profiling:
2000 if config.gs != '':
2001 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2002 if resultGood == 0:
2003 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2004 if resultBad != 0:
2005 print "GhostScript available for hp2ps tests"
2006 gs_working = 1;
2007 else:
2008 gsNotWorking();
2009 else:
2010 gsNotWorking();
2011 else:
2012 gsNotWorking();
2013
2014 def rm_no_fail( file ):
2015 try:
2016 os.remove( file )
2017 finally:
2018 return
2019
2020 def add_suffix( name, suffix ):
2021 if suffix == '':
2022 return name
2023 else:
2024 return name + '.' + suffix
2025
2026 def add_hs_lhs_suffix(name):
2027 if getTestOpts().c_src:
2028 return add_suffix(name, 'c')
2029 elif getTestOpts().cmm_src:
2030 return add_suffix(name, 'cmm')
2031 elif getTestOpts().objc_src:
2032 return add_suffix(name, 'm')
2033 elif getTestOpts().objcpp_src:
2034 return add_suffix(name, 'mm')
2035 elif getTestOpts().literate:
2036 return add_suffix(name, 'lhs')
2037 else:
2038 return add_suffix(name, 'hs')
2039
2040 def replace_suffix( name, suffix ):
2041 base, suf = os.path.splitext(name)
2042 return base + '.' + suffix
2043
2044 def in_testdir( name ):
2045 return (getTestOpts().testdir + '/' + name)
2046
2047 def qualify( name, suff ):
2048 return in_testdir(add_suffix(name, suff))
2049
2050
2051 # Finding the sample output. The filename is of the form
2052 #
2053 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2054 #
2055 # and we pick the most specific version available. The <version> is
2056 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2057 # more fine-grained control use if_compiler_lt().
2058 #
2059 def platform_wordsize_qualify( name, suff ):
2060
2061 basepath = qualify(name, suff)
2062
2063 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2064 for (platformSpecific, plat) in [(1, '-' + config.platform),
2065 (1, '-' + config.os),
2066 (0, '')]
2067 for ws in ['-ws-' + config.wordsize, '']
2068 for comp in ['-' + config.compiler_type, '']
2069 for vers in ['-' + config.compiler_maj_version, '']]
2070
2071 dir = glob.glob(basepath + '*')
2072 dir = map (lambda d: normalise_slashes_(d), dir)
2073
2074 for (platformSpecific, f) in paths:
2075 if f in dir:
2076 return (platformSpecific,f)
2077
2078 return (0, basepath)
2079
2080 # Clean up prior to the test, so that we can't spuriously conclude
2081 # that it passed on the basis of old run outputs.
2082 def pretest_cleanup(name):
2083 if getTestOpts().outputdir != None:
2084 odir = in_testdir(getTestOpts().outputdir)
2085 try:
2086 shutil.rmtree(odir)
2087 except:
2088 pass
2089 os.mkdir(odir)
2090
2091 rm_no_fail(qualify(name,'interp.stderr'))
2092 rm_no_fail(qualify(name,'interp.stdout'))
2093 rm_no_fail(qualify(name,'comp.stderr'))
2094 rm_no_fail(qualify(name,'comp.stdout'))
2095 rm_no_fail(qualify(name,'run.stderr'))
2096 rm_no_fail(qualify(name,'run.stdout'))
2097 rm_no_fail(qualify(name,'tix'))
2098 rm_no_fail(qualify(name,'exe.tix'))
2099 # simple_build zaps the following:
2100 # rm_nofail(qualify("o"))
2101 # rm_nofail(qualify(""))
2102 # not interested in the return code
2103
2104 # -----------------------------------------------------------------------------
2105 # Return a list of all the files ending in '.T' below the directory dir.
2106
2107 def findTFiles(roots):
2108 return concat(map(findTFiles_,roots))
2109
2110 def findTFiles_(path):
2111 if os.path.isdir(path):
2112 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2113 return findTFiles(paths)
2114 elif path[-2:] == '.T':
2115 return [path]
2116 else:
2117 return []
2118
2119 # -----------------------------------------------------------------------------
2120 # Output a test summary to the specified file object
2121
2122 def summary(t, file):
2123
2124 file.write('\n')
2125 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2126 file.write('OVERALL SUMMARY for test run started at '
2127 + time.strftime("%c %Z", t.start_time) + '\n'
2128 + string.rjust(str(datetime.timedelta(seconds=
2129 round(time.time() - time.mktime(t.start_time)))), 8)
2130 + ' spent to go through\n'
2131 + string.rjust(`t.total_tests`, 8)
2132 + ' total tests, which gave rise to\n'
2133 + string.rjust(`t.total_test_cases`, 8)
2134 + ' test cases, of which\n'
2135 + string.rjust(`t.n_tests_skipped`, 8)
2136 + ' were skipped\n'
2137 + '\n'
2138 + string.rjust(`t.n_missing_libs`, 8)
2139 + ' had missing libraries\n'
2140 + string.rjust(`t.n_expected_passes`, 8)
2141 + ' expected passes\n'
2142 + string.rjust(`t.n_expected_failures`, 8)
2143 + ' expected failures\n'
2144 + '\n'
2145 + string.rjust(`t.n_framework_failures`, 8)
2146 + ' caused framework failures\n'
2147 + string.rjust(`t.n_unexpected_passes`, 8)
2148 + ' unexpected passes\n'
2149 + string.rjust(`t.n_unexpected_failures`, 8)
2150 + ' unexpected failures\n'
2151 + '\n')
2152
2153 if t.n_unexpected_passes > 0:
2154 file.write('Unexpected passes:\n')
2155 printPassingTestInfosSummary(file, t.unexpected_passes)
2156
2157 if t.n_unexpected_failures > 0:
2158 file.write('Unexpected failures:\n')
2159 printFailingTestInfosSummary(file, t.unexpected_failures)
2160
2161 if config.check_files_written:
2162 checkForFilesWrittenProblems(file)
2163
2164 if stopping():
2165 file.write('WARNING: Testsuite run was terminated early\n')
2166
2167 def printUnexpectedTests(file, testInfoss):
2168 unexpected = []
2169 for testInfos in testInfoss:
2170 directories = testInfos.keys()
2171 for directory in directories:
2172 tests = testInfos[directory].keys()
2173 unexpected += tests
2174 if unexpected != []:
2175 file.write('Unexpected results from:\n')
2176 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2177 file.write('\n')
2178
2179 def printPassingTestInfosSummary(file, testInfos):
2180 directories = testInfos.keys()
2181 directories.sort()
2182 maxDirLen = max(map ((lambda x : len(x)), directories))
2183 for directory in directories:
2184 tests = testInfos[directory].keys()
2185 tests.sort()
2186 for test in tests:
2187 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2188 ' (' + join(testInfos[directory][test],',') + ')\n')
2189 file.write('\n')
2190
2191 def printFailingTestInfosSummary(file, testInfos):
2192 directories = testInfos.keys()
2193 directories.sort()
2194 maxDirLen = max(map ((lambda x : len(x)), directories))
2195 for directory in directories:
2196 tests = testInfos[directory].keys()
2197 tests.sort()
2198 for test in tests:
2199 reasons = testInfos[directory][test].keys()
2200 for reason in reasons:
2201 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2202 ' [' + reason + ']' + \
2203 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2204 file.write('\n')
2205
2206 def getStdout(cmd):
2207 if have_subprocess:
2208 p = subprocess.Popen(cmd,
2209 stdout=subprocess.PIPE,
2210 stderr=subprocess.PIPE)
2211 (stdout, stderr) = p.communicate()
2212 r = p.wait()
2213 if r != 0:
2214 raise Exception("Command failed: " + str(cmd))
2215 if stderr != '':
2216 raise Exception("stderr from command: " + str(cmd))
2217 return stdout
2218 else:
2219 raise Exception("Need subprocess to get stdout, but don't have it")