Replace 'if_platform' and 'unless_platform' with 'platform'
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 opts.expect = 'fail';
138
139 def ignore_output( name, opts ):
140 opts.ignore_output = 1
141
142 def no_stdin( name, opts ):
143 opts.no_stdin = 1
144
145 def combined_output( name, opts ):
146 opts.combined_output = True
147
148 # -----
149
150 def expect_fail_for( ways ):
151 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
152
153 def _expect_fail_for( name, opts, ways ):
154 opts.expect_fail_for = ways
155
156 def expect_broken_for( bug, ways ):
157 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
158
159 def _expect_broken_for( name, opts, bug, ways ):
160 opts.expect_fail_for = ways
161
162 # -----
163
164 def omit_ways( ways ):
165 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
166
167 def _omit_ways( name, opts, ways ):
168 opts.omit_ways = ways
169
170 # -----
171
172 def only_ways( ways ):
173 return lambda name, opts, w=ways: _only_ways( name, opts, w )
174
175 def _only_ways( name, opts, ways ):
176 opts.only_ways = ways
177
178 # -----
179
180 def extra_ways( ways ):
181 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
182
183 def _extra_ways( name, opts, ways ):
184 opts.extra_ways = ways
185
186 # -----
187
188 def omit_compiler_types( compiler_types ):
189 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
190
191 def _omit_compiler_types( name, opts, compiler_types ):
192 if config.compiler_type in compiler_types:
193 opts.skip = 1
194
195 # -----
196
197 def only_compiler_types( compiler_types ):
198 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
199
200 def _only_compiler_types( name, opts, compiler_types ):
201 if config.compiler_type not in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def set_stdin( file ):
207 return lambda name, opts, f=file: _set_stdin(name, opts, f);
208
209 def _set_stdin( name, opts, f ):
210 opts.stdin = f
211
212 # -----
213
214 def exit_code( val ):
215 return lambda name, opts, v=val: _exit_code(name, opts, v);
216
217 def _exit_code( name, opts, v ):
218 opts.exit_code = v
219
220 # -----
221
222 def timeout_multiplier( val ):
223 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
224
225 def _timeout_multiplier( name, opts, v ):
226 opts.timeout_multiplier = v
227
228 # -----
229
230 def extra_run_opts( val ):
231 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
232
233 def _extra_run_opts( name, opts, v ):
234 opts.extra_run_opts = v
235
236 # -----
237
238 def extra_hc_opts( val ):
239 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
240
241 def _extra_hc_opts( name, opts, v ):
242 opts.extra_hc_opts = v
243
244 # -----
245
246 def extra_clean( files ):
247 return lambda name, opts, v=files: _extra_clean(name, opts, v);
248
249 def _extra_clean( name, opts, v ):
250 opts.clean_files = v
251
252 # -----
253
254 def stats_num_field( field, expecteds ):
255 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
256
257 def _stats_num_field( name, opts, field, expecteds ):
258 if field in opts.stats_range_fields:
259 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
260
261 if type(expecteds) is types.ListType:
262 for (b, expected, dev) in expecteds:
263 if b:
264 opts.stats_range_fields[field] = (expected, dev)
265 return
266 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
267
268 else:
269 (expected, dev) = expecteds
270 opts.stats_range_fields[field] = (expected, dev)
271
272 def stats_range_field( field, expected, dev ):
273 return stats_num_field( field, [(True, expected, dev)] )
274
275 def compiler_stats_num_field( field, expecteds ):
276 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
277
278 def _compiler_stats_num_field( name, opts, field, expecteds ):
279 if field in opts.compiler_stats_range_fields:
280 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
281
282 for (b, expected, dev) in expecteds:
283 if b:
284 opts.compiler_stats_range_fields[field] = (expected, dev)
285 return
286
287 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
288
289 def compiler_stats_range_field( field, expected, dev ):
290 return compiler_stats_num_field( field, [(True, expected, dev)] )
291
292 # -----
293
294 def skip_if_no_ghci(name, opts):
295 if not ('ghci' in config.run_ways):
296 opts.skip = 1
297
298 # ----
299
300 def skip_if_fast(name, opts):
301 if config.fast:
302 opts.skip = 1
303
304 # -----
305
306 def when(b, f):
307 if b:
308 return f
309 else:
310 return normal
311
312 def unless(b, f):
313 return when(not b, f)
314
315 def platform( plat ):
316 return config.platform == plat
317
318 def if_os( os, f ):
319 if config.os == os:
320 return f
321 else:
322 return normal
323
324 def unless_os( os, f ):
325 if config.os == os:
326 return normal
327 else:
328 return f
329
330 def if_arch( arch, f ):
331 if config.arch == arch:
332 return f
333 else:
334 return normal
335
336 def unless_arch( arch, f ):
337 if config.arch == arch:
338 return normal
339 else:
340 return f
341
342 def wordsize( ws ):
343 return config.wordsize == str(ws)
344
345 def if_unregisterised( f ):
346 if config.unregisterised:
347 return f
348 else:
349 return normal
350
351 def unless_unregisterised( f ):
352 if config.unregisterised:
353 return normal
354 else:
355 return f
356
357 def if_msys( f ):
358 if config.msys:
359 return f
360 else:
361 return normal
362
363 def if_cygwin( f ):
364 if config.cygwin:
365 return f
366 else:
367 return normal
368
369 def when_have_vanilla( f ):
370 if config.have_vanilla:
371 return f
372 else:
373 return normal
374
375 def unless_have_vanilla( f ):
376 if config.have_vanilla:
377 return normal
378 else:
379 return f
380
381 def when_have_dynamic( f ):
382 if config.have_dynamic:
383 return f
384 else:
385 return normal
386
387 def unless_have_dynamic( f ):
388 if config.have_dynamic:
389 return normal
390 else:
391 return f
392
393 def when_have_profiling( f ):
394 if config.have_profiling:
395 return f
396 else:
397 return normal
398
399 def unless_have_profiling( f ):
400 if config.have_profiling:
401 return normal
402 else:
403 return f
404
405 # ---
406
407 def if_ghci_dynamic( f ):
408 if config.ghc_dynamic_by_default:
409 return f
410 else:
411 return normal
412
413 def if_in_tree_compiler( f ):
414 if config.in_tree_compiler:
415 return f
416 else:
417 return normal
418
419 def unless_in_tree_compiler( f ):
420 if config.in_tree_compiler:
421 return normal
422 else:
423 return f
424
425 def if_compiler_type( compiler, f ):
426 if config.compiler_type == compiler:
427 return f
428 else:
429 return normal
430
431 def if_compiler_profiled( f ):
432 if config.compiler_profiled:
433 return f
434 else:
435 return normal
436
437 def unless_compiler_profiled( f ):
438 if config.compiler_profiled:
439 return normal
440 else:
441 return f
442
443 def if_compiler_lt( compiler, version, f ):
444 if config.compiler_type == compiler and \
445 version_lt(config.compiler_version, version):
446 return f
447 else:
448 return normal
449
450 def if_compiler_le( compiler, version, f ):
451 if config.compiler_type == compiler and \
452 version_le(config.compiler_version, version):
453 return f
454 else:
455 return normal
456
457 def if_compiler_gt( compiler, version, f ):
458 if config.compiler_type == compiler and \
459 version_gt(config.compiler_version, version):
460 return f
461 else:
462 return normal
463
464 def if_compiler_ge( compiler, version, f ):
465 if config.compiler_type == compiler and \
466 version_ge(config.compiler_version, version):
467 return f
468 else:
469 return normal
470
471 def if_compiler_debugged( f ):
472 if config.compiler_debugged:
473 return f
474 else:
475 return normal
476
477 def namebase( nb ):
478 return lambda opts, nb=nb: _namebase(opts, nb)
479
480 def _namebase( opts, nb ):
481 opts.with_namebase = nb
482
483 # ---
484
485 def if_tag( tag, f ):
486 if tag in config.compiler_tags:
487 return f
488 else:
489 return normal
490
491 def unless_tag( tag, f ):
492 if not (tag in config.compiler_tags):
493 return f
494 else:
495 return normal
496
497 # ---
498 def high_memory_usage(name, opts):
499 opts.alone = True
500
501 # ---
502 def literate( name, opts ):
503 opts.literate = 1;
504
505 def c_src( name, opts ):
506 opts.c_src = 1;
507
508 def objc_src( name, opts ):
509 opts.objc_src = 1;
510
511 def objcpp_src( name, opts ):
512 opts.objcpp_src = 1;
513
514 def cmm_src( name, opts ):
515 opts.cmm_src = 1;
516
517 def outputdir( odir ):
518 return lambda name, opts, d=odir: _outputdir(name, opts, d)
519
520 def _outputdir( name, opts, odir ):
521 opts.outputdir = odir;
522
523 # ----
524
525 def pre_cmd( cmd ):
526 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
527
528 def _pre_cmd( name, opts, cmd ):
529 opts.pre_cmd = cmd
530
531 # ----
532
533 def clean_cmd( cmd ):
534 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
535
536 def _clean_cmd( name, opts, cmd ):
537 opts.clean_cmd = cmd
538
539 # ----
540
541 def cmd_prefix( prefix ):
542 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
543
544 def _cmd_prefix( name, opts, prefix ):
545 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
546
547 # ----
548
549 def cmd_wrapper( fun ):
550 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
551
552 def _cmd_wrapper( name, opts, fun ):
553 opts.cmd_wrapper = fun
554
555 # ----
556
557 def compile_cmd_prefix( prefix ):
558 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
559
560 def _compile_cmd_prefix( name, opts, prefix ):
561 opts.compile_cmd_prefix = prefix
562
563 # ----
564
565 def normalise_slashes( name, opts ):
566 opts.extra_normaliser = normalise_slashes_
567
568 def normalise_exe( name, opts ):
569 opts.extra_normaliser = normalise_exe_
570
571 def normalise_fun( fun ):
572 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
573
574 def _normalise_fun( name, opts, f ):
575 opts.extra_normaliser = f
576
577 def normalise_errmsg_fun( fun ):
578 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
579
580 def _normalise_errmsg_fun( name, opts, f ):
581 opts.extra_errmsg_normaliser = f
582
583 def two_normalisers(f, g):
584 return lambda x, f=f, g=g: f(g(x))
585
586 # ----
587 # Function for composing two opt-fns together
588
589 def composes( fs ):
590 return reduce(lambda f, g: compose(f, g), fs)
591
592 def compose( f, g ):
593 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
594
595 def _compose( name, opts, f, g ):
596 f(name, opts)
597 g(name, opts)
598
599 # -----------------------------------------------------------------------------
600 # The current directory of tests
601
602 def newTestDir( dir ):
603 global thisdir_settings
604 # reset the options for this test directory
605 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
606
607 def _newTestDir( name, opts, dir ):
608 opts.testdir = dir
609 opts.compiler_always_flags = config.compiler_always_flags
610
611 # -----------------------------------------------------------------------------
612 # Actually doing tests
613
614 parallelTests = []
615 aloneTests = []
616 allTestNames = set([])
617
618 def runTest (opts, name, func, args):
619 ok = 0
620
621 if config.use_threads:
622 t.thread_pool.acquire()
623 try:
624 while config.threads<(t.running_threads+1):
625 t.thread_pool.wait()
626 t.running_threads = t.running_threads+1
627 ok=1
628 t.thread_pool.release()
629 thread.start_new_thread(test_common_thread, (name, opts, func, args))
630 except:
631 if not ok:
632 t.thread_pool.release()
633 else:
634 test_common_work (name, opts, func, args)
635
636 # name :: String
637 # setup :: TestOpts -> IO ()
638 def test (name, setup, func, args):
639 global aloneTests
640 global parallelTests
641 global allTestNames
642 global thisdir_settings
643 if name in allTestNames:
644 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
645 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
646 framework_fail(name, 'bad_name', 'This test has an invalid name')
647
648 # Make a deep copy of the default_testopts, as we need our own copy
649 # of any dictionaries etc inside it. Otherwise, if one test modifies
650 # them, all tests will see the modified version!
651 myTestOpts = copy.deepcopy(default_testopts)
652
653 if type(setup) is types.ListType:
654 setup = composes(setup)
655
656 setup = compose(thisdir_settings, setup)
657 setup(name, myTestOpts)
658
659 thisTest = lambda : runTest(myTestOpts, name, func, args)
660 if myTestOpts.alone:
661 aloneTests.append(thisTest)
662 else:
663 parallelTests.append(thisTest)
664 allTestNames.add(name)
665
666 if config.use_threads:
667 def test_common_thread(name, opts, func, args):
668 t.lock.acquire()
669 try:
670 test_common_work(name,opts,func,args)
671 finally:
672 t.lock.release()
673 t.thread_pool.acquire()
674 t.running_threads = t.running_threads - 1
675 t.thread_pool.notify()
676 t.thread_pool.release()
677
678 def get_package_cache_timestamp():
679 if config.package_conf_cache_file == '':
680 return 0.0
681 else:
682 try:
683 return os.stat(config.package_conf_cache_file).st_mtime
684 except:
685 return 0.0
686
687
688 def test_common_work (name, opts, func, args):
689 try:
690 t.total_tests = t.total_tests+1
691 setLocalTestOpts(opts)
692
693 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
694
695 # All the ways we might run this test
696 if func == compile or func == multimod_compile:
697 all_ways = config.compile_ways
698 elif func == compile_and_run or func == multimod_compile_and_run:
699 all_ways = config.run_ways
700 elif func == ghci_script:
701 if 'ghci' in config.run_ways:
702 all_ways = ['ghci']
703 else:
704 all_ways = []
705 else:
706 all_ways = ['normal']
707
708 # A test itself can request extra ways by setting opts.extra_ways
709 all_ways = all_ways + filter(lambda way: way not in all_ways,
710 opts.extra_ways)
711
712 t.total_test_cases = t.total_test_cases + len(all_ways)
713
714 ok_way = lambda way: \
715 not getTestOpts().skip \
716 and (config.only == [] or name in config.only) \
717 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
718 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
719 and way not in getTestOpts().omit_ways
720
721 # Which ways we are asked to skip
722 do_ways = filter (ok_way,all_ways)
723
724 # In fast mode, we skip all but one way
725 if config.fast and len(do_ways) > 0:
726 do_ways = [do_ways[0]]
727
728 if not config.clean_only:
729 # Run the required tests...
730 for way in do_ways:
731 if stopping():
732 break
733 do_test (name, way, func, args)
734
735 for way in all_ways:
736 if way not in do_ways:
737 skiptest (name,way)
738
739 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
740 clean(map (lambda suff: name + suff,
741 ['', '.exe', '.exe.manifest', '.genscript',
742 '.stderr.normalised', '.stdout.normalised',
743 '.run.stderr', '.run.stdout',
744 '.run.stderr.normalised', '.run.stdout.normalised',
745 '.comp.stderr', '.comp.stdout',
746 '.comp.stderr.normalised', '.comp.stdout.normalised',
747 '.interp.stderr', '.interp.stdout',
748 '.interp.stderr.normalised', '.interp.stdout.normalised',
749 '.stats', '.comp.stats',
750 '.hi', '.o', '.prof', '.exe.prof', '.hc',
751 '_stub.h', '_stub.c', '_stub.o',
752 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
753
754 if func == multi_compile or func == multi_compile_fail:
755 extra_mods = args[1]
756 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
757 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
758
759 clean(getTestOpts().clean_files)
760
761 if getTestOpts().outputdir != None:
762 odir = in_testdir(getTestOpts().outputdir)
763 try:
764 shutil.rmtree(odir)
765 except:
766 pass
767
768 try:
769 shutil.rmtree(in_testdir('.hpc.' + name))
770 except:
771 pass
772
773 try:
774 cleanCmd = getTestOpts().clean_cmd
775 if cleanCmd != None:
776 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
777 if result != 0:
778 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
779 except e:
780 framework_fail(name, 'cleaning', 'clean-command exception')
781
782 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
783
784 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
785 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
786
787 try:
788 for f in files_written[name]:
789 if os.path.exists(f):
790 try:
791 if not f in files_written_not_removed[name]:
792 files_written_not_removed[name].append(f)
793 except:
794 files_written_not_removed[name] = [f]
795 except:
796 pass
797 except Exception, e:
798 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
799
800 def clean(strs):
801 for str in strs:
802 for name in glob.glob(in_testdir(str)):
803 clean_full_path(name)
804
805 def clean_full_path(name):
806 try:
807 # Remove files...
808 os.remove(name)
809 except OSError, e1:
810 try:
811 # ... and empty directories
812 os.rmdir(name)
813 except OSError, e2:
814 # We don't want to fail here, but we do want to know
815 # what went wrong, so print out the exceptions.
816 # ENOENT isn't a problem, though, as we clean files
817 # that don't necessarily exist.
818 if e1.errno != errno.ENOENT:
819 print e1
820 if e2.errno != errno.ENOENT:
821 print e2
822
823 def do_test(name, way, func, args):
824 full_name = name + '(' + way + ')'
825
826 try:
827 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
828 str([t.n_unexpected_passes, \
829 t.n_unexpected_failures, \
830 t.n_framework_failures])
831
832 if config.use_threads:
833 t.lock.release()
834
835 try:
836 preCmd = getTestOpts().pre_cmd
837 if preCmd != None:
838 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
839 if result != 0:
840 framework_fail(name, way, 'pre-command failed: ' + str(result))
841 except e:
842 framework_fail(name, way, 'pre-command exception')
843
844 try:
845 result = apply(func, [name,way] + args)
846 finally:
847 if config.use_threads:
848 t.lock.acquire()
849
850 if getTestOpts().expect != 'pass' and \
851 getTestOpts().expect != 'fail' and \
852 getTestOpts().expect != 'missing-lib':
853 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
854
855 try:
856 passFail = result['passFail']
857 except:
858 passFail = 'No passFail found'
859
860 if passFail == 'pass':
861 if getTestOpts().expect == 'pass' \
862 and way not in getTestOpts().expect_fail_for:
863 t.n_expected_passes = t.n_expected_passes + 1
864 if name in t.expected_passes:
865 t.expected_passes[name].append(way)
866 else:
867 t.expected_passes[name] = [way]
868 else:
869 print '*** unexpected pass for', full_name
870 t.n_unexpected_passes = t.n_unexpected_passes + 1
871 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
872 elif passFail == 'fail':
873 if getTestOpts().expect == 'pass' \
874 and way not in getTestOpts().expect_fail_for:
875 print '*** unexpected failure for', full_name
876 t.n_unexpected_failures = t.n_unexpected_failures + 1
877 reason = result['reason']
878 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
879 else:
880 if getTestOpts().expect == 'missing-lib':
881 t.n_missing_libs = t.n_missing_libs + 1
882 if name in t.missing_libs:
883 t.missing_libs[name].append(way)
884 else:
885 t.missing_libs[name] = [way]
886 else:
887 t.n_expected_failures = t.n_expected_failures + 1
888 if name in t.expected_failures:
889 t.expected_failures[name].append(way)
890 else:
891 t.expected_failures[name] = [way]
892 else:
893 framework_fail(name, way, 'bad result ' + passFail)
894 except KeyboardInterrupt:
895 stopNow()
896 except:
897 framework_fail(name, way, 'do_test exception')
898 traceback.print_exc()
899
900 def addPassingTestInfo (testInfos, directory, name, way):
901 directory = re.sub('^\\.[/\\\\]', '', directory)
902
903 if not directory in testInfos:
904 testInfos[directory] = {}
905
906 if not name in testInfos[directory]:
907 testInfos[directory][name] = []
908
909 testInfos[directory][name].append(way)
910
911 def addFailingTestInfo (testInfos, directory, name, reason, way):
912 directory = re.sub('^\\.[/\\\\]', '', directory)
913
914 if not directory in testInfos:
915 testInfos[directory] = {}
916
917 if not name in testInfos[directory]:
918 testInfos[directory][name] = {}
919
920 if not reason in testInfos[directory][name]:
921 testInfos[directory][name][reason] = []
922
923 testInfos[directory][name][reason].append(way)
924
925 def skiptest (name, way):
926 # print 'Skipping test \"', name, '\"'
927 t.n_tests_skipped = t.n_tests_skipped + 1
928 if name in t.tests_skipped:
929 t.tests_skipped[name].append(way)
930 else:
931 t.tests_skipped[name] = [way]
932
933 def framework_fail( name, way, reason ):
934 full_name = name + '(' + way + ')'
935 print '*** framework failure for', full_name, reason
936 t.n_framework_failures = t.n_framework_failures + 1
937 if name in t.framework_failures:
938 t.framework_failures[name].append(way)
939 else:
940 t.framework_failures[name] = [way]
941
942 def badResult(result):
943 try:
944 if result['passFail'] == 'pass':
945 return False
946 return True
947 except:
948 return True
949
950 def passed():
951 return {'passFail': 'pass'}
952
953 def failBecause(reason):
954 return {'passFail': 'fail', 'reason': reason}
955
956 # -----------------------------------------------------------------------------
957 # Generic command tests
958
959 # A generic command test is expected to run and exit successfully.
960 #
961 # The expected exit code can be changed via exit_code() as normal, and
962 # the expected stdout/stderr are stored in <testname>.stdout and
963 # <testname>.stderr. The output of the command can be ignored
964 # altogether by using run_command_ignore_output instead of
965 # run_command.
966
967 def run_command( name, way, cmd ):
968 return simple_run( name, '', cmd, '' )
969
970 # -----------------------------------------------------------------------------
971 # GHCi tests
972
973 def ghci_script( name, way, script ):
974 # filter out -fforce-recomp from compiler_always_flags, because we're
975 # actually testing the recompilation behaviour in the GHCi tests.
976 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
977 flags.append(getTestOpts().extra_hc_opts)
978 if getTestOpts().outputdir != None:
979 flags.extend(["-outputdir", getTestOpts().outputdir])
980
981 # We pass HC and HC_OPTS as environment variables, so that the
982 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
983 cmd = "HC='" + config.compiler + "' " + \
984 "HC_OPTS='" + join(flags,' ') + "' " + \
985 "'" + config.compiler + "'" + \
986 ' --interactive -v0 -ignore-dot-ghci ' + \
987 join(flags,' ')
988
989 getTestOpts().stdin = script
990 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
991
992 # -----------------------------------------------------------------------------
993 # Compile-only tests
994
995 def compile( name, way, extra_hc_opts ):
996 return do_compile( name, way, 0, '', [], extra_hc_opts )
997
998 def compile_fail( name, way, extra_hc_opts ):
999 return do_compile( name, way, 1, '', [], extra_hc_opts )
1000
1001 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1002 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1003
1004 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1005 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1006
1007 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1008 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1009
1010 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1011 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1012
1013 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1014 # print 'Compile only, extra args = ', extra_hc_opts
1015 pretest_cleanup(name)
1016
1017 result = extras_build( way, extra_mods, extra_hc_opts )
1018 if badResult(result):
1019 return result
1020 extra_hc_opts = result['hc_opts']
1021
1022 force = 0
1023 if extra_mods:
1024 force = 1
1025 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1026
1027 if badResult(result):
1028 return result
1029
1030 # the actual stderr should always match the expected, regardless
1031 # of whether we expected the compilation to fail or not (successful
1032 # compilations may generate warnings).
1033
1034 if getTestOpts().with_namebase == None:
1035 namebase = name
1036 else:
1037 namebase = getTestOpts().with_namebase
1038
1039 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1040 actual_stderr_file = qualify(name, 'comp.stderr')
1041
1042 if not compare_outputs('stderr', \
1043 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1044 expected_stderr_file, actual_stderr_file):
1045 return failBecause('stderr mismatch')
1046
1047 # no problems found, this test passed
1048 return passed()
1049
1050 def compile_cmp_asm( name, way, extra_hc_opts ):
1051 print 'Compile only, extra args = ', extra_hc_opts
1052 pretest_cleanup(name)
1053 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1054
1055 if badResult(result):
1056 return result
1057
1058 # the actual stderr should always match the expected, regardless
1059 # of whether we expected the compilation to fail or not (successful
1060 # compilations may generate warnings).
1061
1062 if getTestOpts().with_namebase == None:
1063 namebase = name
1064 else:
1065 namebase = getTestOpts().with_namebase
1066
1067 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1068 actual_asm_file = qualify(name, 's')
1069
1070 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1071 expected_asm_file, actual_asm_file):
1072 return failBecause('asm mismatch')
1073
1074 # no problems found, this test passed
1075 return passed()
1076
1077 # -----------------------------------------------------------------------------
1078 # Compile-and-run tests
1079
1080 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1081 # print 'Compile and run, extra args = ', extra_hc_opts
1082 pretest_cleanup(name)
1083
1084 result = extras_build( way, extra_mods, extra_hc_opts )
1085 if badResult(result):
1086 return result
1087 extra_hc_opts = result['hc_opts']
1088
1089 if way == 'ghci': # interpreted...
1090 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1091 elif way == 'extcore' or way == 'optextcore' :
1092 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1093 else: # compiled...
1094 force = 0
1095 if extra_mods:
1096 force = 1
1097
1098 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1099 if badResult(result):
1100 return result
1101
1102 cmd = './' + name;
1103
1104 # we don't check the compiler's stderr for a compile-and-run test
1105 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1106
1107 def compile_and_run( name, way, extra_hc_opts ):
1108 return compile_and_run__( name, way, '', [], extra_hc_opts)
1109
1110 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1111 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1112
1113 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1114 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1115
1116 def stats( name, way, stats_file ):
1117 opts = getTestOpts()
1118 return checkStats(stats_file, opts.stats_range_fields)
1119
1120 # -----------------------------------------------------------------------------
1121 # Check -t stats info
1122
1123 def checkStats(stats_file, range_fields):
1124 result = passed()
1125 if len(range_fields) > 0:
1126 f = open(in_testdir(stats_file))
1127 contents = f.read()
1128 f.close()
1129
1130 for (field, (expected, dev)) in range_fields.items():
1131 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1132 if m == None:
1133 print 'Failed to find field: ', field
1134 result = failBecause('no such stats field')
1135 val = int(m.group(1))
1136
1137 lowerBound = trunc( expected * ((100 - float(dev))/100));
1138 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1139
1140 if val < lowerBound:
1141 print field, 'value is too low:'
1142 print '(If this is because you have improved GHC, please'
1143 print 'update the test so that GHC doesn\'t regress again)'
1144 result = failBecause('stat too good')
1145 if val > upperBound:
1146 print field, 'value is too high:'
1147 result = failBecause('stat not good enough')
1148
1149 if val < lowerBound or val > upperBound:
1150 valStr = str(val)
1151 valLen = len(valStr)
1152 expectedStr = str(expected)
1153 expectedLen = len(expectedStr)
1154 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1155 def display(descr, val, extra):
1156 print descr, string.rjust(str(val), length), extra
1157 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1158 display(' Lower bound ' + field + ':', lowerBound, '')
1159 display(' Upper bound ' + field + ':', upperBound, '')
1160 display(' Actual ' + field + ':', val, '')
1161
1162 return result
1163
1164 # -----------------------------------------------------------------------------
1165 # Build a single-module program
1166
1167 def extras_build( way, extra_mods, extra_hc_opts ):
1168 for modopts in extra_mods:
1169 mod, opts = modopts
1170 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1171 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1172 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1173 if badResult(result):
1174 return result
1175
1176 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1177
1178
1179 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1180 opts = getTestOpts()
1181 errname = add_suffix(name, 'comp.stderr')
1182 rm_no_fail( qualify(errname, '') )
1183
1184 if top_mod != '':
1185 srcname = top_mod
1186 rm_no_fail( qualify(name, '') )
1187 base, suf = os.path.splitext(top_mod)
1188 rm_no_fail( qualify(base, '') )
1189 rm_no_fail( qualify(base, 'exe') )
1190 elif addsuf:
1191 srcname = add_hs_lhs_suffix(name)
1192 rm_no_fail( qualify(name, '') )
1193 else:
1194 srcname = name
1195 rm_no_fail( qualify(name, 'o') )
1196
1197 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1198
1199 to_do = ''
1200 if top_mod != '':
1201 to_do = '--make '
1202 if link:
1203 to_do = to_do + '-o ' + name
1204 elif link:
1205 to_do = '-o ' + name
1206 elif opts.compile_to_hc:
1207 to_do = '-C'
1208 else:
1209 to_do = '-c' # just compile
1210
1211 stats_file = name + '.comp.stats'
1212 if len(opts.compiler_stats_range_fields) > 0:
1213 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1214
1215 # Required by GHC 7.3+, harmless for earlier versions:
1216 if (getTestOpts().c_src or
1217 getTestOpts().objc_src or
1218 getTestOpts().objcpp_src or
1219 getTestOpts().cmm_src):
1220 extra_hc_opts += ' -no-hs-main '
1221
1222 if getTestOpts().compile_cmd_prefix == '':
1223 cmd_prefix = ''
1224 else:
1225 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1226
1227 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1228 if noforce:
1229 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1230 if getTestOpts().outputdir != None:
1231 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1232
1233 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1234 + config.compiler + "' " \
1235 + join(comp_flags,' ') + ' ' \
1236 + to_do + ' ' + srcname + ' ' \
1237 + join(config.way_flags(name)[way],' ') + ' ' \
1238 + extra_hc_opts + ' ' \
1239 + opts.extra_hc_opts + ' ' \
1240 + '>' + errname + ' 2>&1'
1241
1242 result = runCmdFor(name, cmd)
1243
1244 if result != 0 and not should_fail:
1245 actual_stderr = qualify(name, 'comp.stderr')
1246 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1247 if_verbose_dump(1,actual_stderr)
1248
1249 # ToDo: if the sub-shell was killed by ^C, then exit
1250
1251 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1252
1253 if badResult(statsResult):
1254 return statsResult
1255
1256 if should_fail:
1257 if result == 0:
1258 return failBecause('exit code 0')
1259 else:
1260 if result != 0:
1261 return failBecause('exit code non-0')
1262
1263 return passed()
1264
1265 # -----------------------------------------------------------------------------
1266 # Run a program and check its output
1267 #
1268 # If testname.stdin exists, route input from that, else
1269 # from /dev/null. Route output to testname.run.stdout and
1270 # testname.run.stderr. Returns the exit code of the run.
1271
1272 def simple_run( name, way, prog, args ):
1273 opts = getTestOpts()
1274
1275 # figure out what to use for stdin
1276 if opts.stdin != '':
1277 use_stdin = opts.stdin
1278 else:
1279 stdin_file = add_suffix(name, 'stdin')
1280 if os.path.exists(in_testdir(stdin_file)):
1281 use_stdin = stdin_file
1282 else:
1283 use_stdin = '/dev/null'
1284
1285 run_stdout = add_suffix(name,'run.stdout')
1286 run_stderr = add_suffix(name,'run.stderr')
1287
1288 rm_no_fail(qualify(name,'run.stdout'))
1289 rm_no_fail(qualify(name,'run.stderr'))
1290 rm_no_fail(qualify(name, 'hp'))
1291 rm_no_fail(qualify(name,'ps'))
1292 rm_no_fail(qualify(name, 'prof'))
1293
1294 my_rts_flags = rts_flags(way)
1295
1296 stats_file = name + '.stats'
1297 if len(opts.stats_range_fields) > 0:
1298 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1299
1300 if opts.no_stdin:
1301 stdin_comes_from = ''
1302 else:
1303 stdin_comes_from = ' <' + use_stdin
1304
1305 if opts.combined_output:
1306 redirection = ' >' + run_stdout \
1307 + ' 2>&1'
1308 else:
1309 redirection = ' >' + run_stdout \
1310 + ' 2>' + run_stderr
1311
1312 cmd = prog + ' ' + args + ' ' \
1313 + my_rts_flags + ' ' \
1314 + stdin_comes_from \
1315 + redirection
1316
1317 if opts.cmd_wrapper != None:
1318 cmd = opts.cmd_wrapper(cmd);
1319
1320 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1321
1322 # run the command
1323 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1324
1325 exit_code = result >> 8
1326 signal = result & 0xff
1327
1328 # check the exit code
1329 if exit_code != opts.exit_code:
1330 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1331 dump_stdout(name)
1332 dump_stderr(name)
1333 return failBecause('bad exit code')
1334
1335 check_hp = my_rts_flags.find("-h") != -1
1336 check_prof = my_rts_flags.find("-p") != -1
1337
1338 if not opts.ignore_output:
1339 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1340 bad_stdout = not check_stdout_ok(name)
1341 if bad_stderr:
1342 return failBecause('bad stderr')
1343 if bad_stdout:
1344 return failBecause('bad stdout')
1345 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1346 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1347 return failBecause('bad heap profile')
1348 if check_prof and not check_prof_ok(name):
1349 return failBecause('bad profile')
1350
1351 return checkStats(stats_file, opts.stats_range_fields)
1352
1353 def rts_flags(way):
1354 if (way == ''):
1355 return ''
1356 else:
1357 args = config.way_rts_flags[way]
1358
1359 if args == []:
1360 return ''
1361 else:
1362 return '+RTS ' + join(args,' ') + ' -RTS'
1363
1364 # -----------------------------------------------------------------------------
1365 # Run a program in the interpreter and check its output
1366
1367 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1368 outname = add_suffix(name, 'interp.stdout')
1369 errname = add_suffix(name, 'interp.stderr')
1370 rm_no_fail(outname)
1371 rm_no_fail(errname)
1372 rm_no_fail(name)
1373
1374 if (top_mod == ''):
1375 srcname = add_hs_lhs_suffix(name)
1376 else:
1377 srcname = top_mod
1378
1379 scriptname = add_suffix(name, 'genscript')
1380 qscriptname = in_testdir(scriptname)
1381 rm_no_fail(qscriptname)
1382
1383 delimiter = '===== program output begins here\n'
1384
1385 script = open(qscriptname, 'w')
1386 if not compile_only:
1387 # set the prog name and command-line args to match the compiled
1388 # environment.
1389 script.write(':set prog ' + name + '\n')
1390 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1391 # Add marker lines to the stdout and stderr output files, so we
1392 # can separate GHCi's output from the program's.
1393 script.write(':! echo ' + delimiter)
1394 script.write(':! echo 1>&2 ' + delimiter)
1395 # Set stdout to be line-buffered to match the compiled environment.
1396 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1397 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1398 # in the event of an exception as for the compiled program.
1399 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1400 script.close()
1401
1402 # figure out what to use for stdin
1403 if getTestOpts().stdin != '':
1404 stdin_file = in_testdir(getTestOpts().stdin)
1405 else:
1406 stdin_file = qualify(name, 'stdin')
1407
1408 if os.path.exists(stdin_file):
1409 stdin = open(stdin_file, 'r')
1410 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1411
1412 script.close()
1413
1414 flags = copy.copy(getTestOpts().compiler_always_flags)
1415 if getTestOpts().outputdir != None:
1416 flags.extend(["-outputdir", getTestOpts().outputdir])
1417
1418 cmd = "'" + config.compiler + "' " \
1419 + join(flags,' ') + ' ' \
1420 + srcname + ' ' \
1421 + join(config.way_flags(name)[way],' ') + ' ' \
1422 + extra_hc_opts + ' ' \
1423 + getTestOpts().extra_hc_opts + ' ' \
1424 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1425
1426 if getTestOpts().cmd_wrapper != None:
1427 cmd = getTestOpts().cmd_wrapper(cmd);
1428
1429 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1430
1431 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1432
1433 exit_code = result >> 8
1434 signal = result & 0xff
1435
1436 # split the stdout into compilation/program output
1437 split_file(in_testdir(outname), delimiter,
1438 qualify(name, 'comp.stdout'),
1439 qualify(name, 'run.stdout'))
1440 split_file(in_testdir(errname), delimiter,
1441 qualify(name, 'comp.stderr'),
1442 qualify(name, 'run.stderr'))
1443
1444 # check the exit code
1445 if exit_code != getTestOpts().exit_code:
1446 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1447 dump_stdout(name)
1448 dump_stderr(name)
1449 return failBecause('bad exit code')
1450
1451 # ToDo: if the sub-shell was killed by ^C, then exit
1452
1453 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1454 check_stdout_ok(name)):
1455 return passed()
1456 else:
1457 return failBecause('bad stdout or stderr')
1458
1459
1460 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1461 infile = open(in_fn)
1462 out1 = open(out1_fn, 'w')
1463 out2 = open(out2_fn, 'w')
1464
1465 line = infile.readline()
1466 line = re.sub('\r', '', line) # ignore Windows EOL
1467 while (re.sub('^\s*','',line) != delimiter and line != ''):
1468 out1.write(line)
1469 line = infile.readline()
1470 line = re.sub('\r', '', line)
1471 out1.close()
1472
1473 line = infile.readline()
1474 while (line != ''):
1475 out2.write(line)
1476 line = infile.readline()
1477 out2.close()
1478
1479 # -----------------------------------------------------------------------------
1480 # Generate External Core for the given program, then compile the resulting Core
1481 # and compare its output to the expected output
1482
1483 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1484
1485 depsfilename = qualify(name, 'deps')
1486 errname = add_suffix(name, 'comp.stderr')
1487 qerrname = qualify(errname,'')
1488
1489 hcname = qualify(name, 'hc')
1490 oname = qualify(name, 'o')
1491
1492 rm_no_fail( qerrname )
1493 rm_no_fail( qualify(name, '') )
1494
1495 if (top_mod == ''):
1496 srcname = add_hs_lhs_suffix(name)
1497 else:
1498 srcname = top_mod
1499
1500 qcorefilename = qualify(name, 'hcr')
1501 corefilename = add_suffix(name, 'hcr')
1502 rm_no_fail(qcorefilename)
1503
1504 # Generate External Core
1505
1506 if (top_mod == ''):
1507 to_do = ' ' + srcname + ' '
1508 else:
1509 to_do = ' --make ' + top_mod + ' '
1510
1511 flags = copy.copy(getTestOpts().compiler_always_flags)
1512 if getTestOpts().outputdir != None:
1513 flags.extend(["-outputdir", getTestOpts().outputdir])
1514 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1515 + config.compiler + "' " \
1516 + join(flags,' ') + ' ' \
1517 + join(config.way_flags(name)[way],' ') + ' ' \
1518 + extra_hc_opts + ' ' \
1519 + getTestOpts().extra_hc_opts \
1520 + to_do \
1521 + '>' + errname + ' 2>&1'
1522 result = runCmdFor(name, cmd)
1523
1524 exit_code = result >> 8
1525
1526 if exit_code != 0:
1527 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1528 if_verbose_dump(1,qerrname)
1529 return failBecause('ext core exit code non-0')
1530
1531 # Compile the resulting files -- if there's more than one module, we need to read the output
1532 # of the previous compilation in order to find the dependencies
1533 if (top_mod == ''):
1534 to_compile = corefilename
1535 else:
1536 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1537 deps = open(depsfilename).read()
1538 deplist = string.replace(deps, '\n',' ');
1539 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1540 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1541
1542 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1543 if getTestOpts().outputdir != None:
1544 flags.extend(["-outputdir", getTestOpts().outputdir])
1545
1546 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1547 + config.compiler + "' " \
1548 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1549 + to_compile + ' ' \
1550 + extra_hc_opts + ' ' \
1551 + getTestOpts().extra_hc_opts + ' ' \
1552 + flags \
1553 + ' -fglasgow-exts -o ' + name \
1554 + '>' + errname + ' 2>&1'
1555
1556 result = runCmdFor(name, cmd)
1557 exit_code = result >> 8
1558
1559 if exit_code != 0:
1560 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1561 if_verbose_dump(1,qerrname)
1562 return failBecause('ext core exit code non-0')
1563
1564 # Clean up
1565 rm_no_fail ( oname )
1566 rm_no_fail ( hcname )
1567 rm_no_fail ( qcorefilename )
1568 rm_no_fail ( depsfilename )
1569
1570 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1571
1572 # -----------------------------------------------------------------------------
1573 # Utils
1574
1575 def check_stdout_ok( name ):
1576 if getTestOpts().with_namebase == None:
1577 namebase = name
1578 else:
1579 namebase = getTestOpts().with_namebase
1580
1581 actual_stdout_file = qualify(name, 'run.stdout')
1582 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1583
1584 def norm(str):
1585 if platform_specific:
1586 return str
1587 else:
1588 return normalise_output(str)
1589
1590 return compare_outputs('stdout', \
1591 two_normalisers(norm, getTestOpts().extra_normaliser), \
1592 expected_stdout_file, actual_stdout_file)
1593
1594 def dump_stdout( name ):
1595 print 'Stdout:'
1596 print read_no_crs(qualify(name, 'run.stdout'))
1597
1598 def check_stderr_ok( name ):
1599 if getTestOpts().with_namebase == None:
1600 namebase = name
1601 else:
1602 namebase = getTestOpts().with_namebase
1603
1604 actual_stderr_file = qualify(name, 'run.stderr')
1605 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1606
1607 def norm(str):
1608 if platform_specific:
1609 return str
1610 else:
1611 return normalise_errmsg(str)
1612
1613 return compare_outputs('stderr', \
1614 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1615 expected_stderr_file, actual_stderr_file)
1616
1617 def dump_stderr( name ):
1618 print "Stderr:"
1619 print read_no_crs(qualify(name, 'run.stderr'))
1620
1621 def read_no_crs(file):
1622 str = ''
1623 try:
1624 h = open(file)
1625 str = h.read()
1626 h.close
1627 except:
1628 # On Windows, if the program fails very early, it seems the
1629 # files stdout/stderr are redirected to may not get created
1630 pass
1631 return re.sub('\r', '', str)
1632
1633 def write_file(file, str):
1634 h = open(file, 'w')
1635 h.write(str)
1636 h.close
1637
1638 def check_hp_ok(name):
1639
1640 # do not qualify for hp2ps because we should be in the right directory
1641 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1642
1643 hp2psResult = runCmdExitCode(hp2psCmd)
1644
1645 actual_ps_file = qualify(name, 'ps')
1646
1647 if(hp2psResult == 0):
1648 if (os.path.exists(actual_ps_file)):
1649 if gs_working:
1650 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1651 if (gsResult == 0):
1652 return (True)
1653 else:
1654 print "hp2ps output for " + name + "is not valid PostScript"
1655 else: return (True) # assume postscript is valid without ghostscript
1656 else:
1657 print "hp2ps did not generate PostScript for " + name
1658 return (False)
1659 else:
1660 print "hp2ps error when processing heap profile for " + name
1661 return(False)
1662
1663 def check_prof_ok(name):
1664
1665 prof_file = qualify(name,'prof')
1666
1667 if not os.path.exists(prof_file):
1668 print prof_file + " does not exist"
1669 return(False)
1670
1671 if os.path.getsize(qualify(name,'prof')) == 0:
1672 print prof_file + " is empty"
1673 return(False)
1674
1675 if getTestOpts().with_namebase == None:
1676 namebase = name
1677 else:
1678 namebase = getTestOpts().with_namebase
1679
1680 (platform_specific, expected_prof_file) = \
1681 platform_wordsize_qualify(namebase, 'prof.sample')
1682
1683 # sample prof file is not required
1684 if not os.path.exists(expected_prof_file):
1685 return True
1686 else:
1687 return compare_outputs('prof', \
1688 two_normalisers(normalise_whitespace,normalise_prof), \
1689 expected_prof_file, prof_file)
1690
1691 # Compare expected output to actual output, and optionally accept the
1692 # new output. Returns true if output matched or was accepted, false
1693 # otherwise.
1694 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1695 if os.path.exists(expected_file):
1696 expected_raw = read_no_crs(expected_file)
1697 # print "norm:", normaliser(expected_raw)
1698 expected_str = normaliser(expected_raw)
1699 expected_file_for_diff = expected_file
1700 else:
1701 expected_str = ''
1702 expected_file_for_diff = '/dev/null'
1703
1704 actual_raw = read_no_crs(actual_file)
1705 actual_str = normaliser(actual_raw)
1706
1707 if expected_str == actual_str:
1708 return 1
1709 else:
1710 print 'Actual ' + kind + ' output differs from expected:'
1711
1712 if expected_file_for_diff == '/dev/null':
1713 expected_normalised_file = '/dev/null'
1714 else:
1715 expected_normalised_file = expected_file + ".normalised"
1716 write_file(expected_normalised_file, expected_str)
1717
1718 actual_normalised_file = actual_file + ".normalised"
1719 write_file(actual_normalised_file, actual_str)
1720
1721 # Ignore whitespace when diffing. We should only get to this
1722 # point if there are non-whitespace differences
1723 #
1724 # Note we are diffing the *actual* output, not the normalised
1725 # output. The normalised output may have whitespace squashed
1726 # (including newlines) so the diff would be hard to read.
1727 # This does mean that the diff might contain changes that
1728 # would be normalised away.
1729 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1730 ' ' + actual_file )
1731
1732 # If for some reason there were no non-whitespace differences,
1733 # then do a full diff
1734 if r == 0:
1735 r = os.system( 'diff -u ' + expected_file_for_diff + \
1736 ' ' + actual_file )
1737
1738 if config.accept:
1739 print 'Accepting new output.'
1740 write_file(expected_file, actual_raw)
1741 return 1
1742 else:
1743 return 0
1744
1745
1746 def normalise_whitespace( str ):
1747 # Merge contiguous whitespace characters into a single space.
1748 str = re.sub('[ \t\n]+', ' ', str)
1749 return str
1750
1751 def normalise_errmsg( str ):
1752 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1753 # the colon is there because it appears in error messages; this
1754 # hacky solution is used in place of more sophisticated filename
1755 # mangling
1756 str = re.sub('([^\\s])\\.exe', '\\1', str)
1757 # normalise slashes, minimise Windows/Unix filename differences
1758 str = re.sub('\\\\', '/', str)
1759 # The inplace ghc's are called ghc-stage[123] to avoid filename
1760 # collisions, so we need to normalise that to just "ghc"
1761 str = re.sub('ghc-stage[123]', 'ghc', str)
1762 # We sometimes see the name of the integer-gmp package on stderr,
1763 # but this can change (either the implementation name or the
1764 # version number), so we canonicalise it here
1765 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1766 return str
1767
1768 # normalise a .prof file, so that we can reasonably compare it against
1769 # a sample. This doesn't compare any of the actual profiling data,
1770 # only the shape of the profile and the number of entries.
1771 def normalise_prof (str):
1772 # strip everything up to the line beginning "COST CENTRE"
1773 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1774
1775 # strip results for CAFs, these tend to change unpredictably
1776 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1777
1778 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1779 # sometimes under MAIN.
1780 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1781
1782 # We have somthing like this:
1783
1784 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1785 # k Main 204 1 0.0 0.0 0.0 0.0
1786 # foo Main 205 1 0.0 0.0 0.0 0.0
1787 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1788
1789 # then we remove all the specific profiling data, leaving only the
1790 # cost centre name, module, and entries, to end up with this:
1791
1792 # MAIN MAIN 0
1793 # k Main 1
1794 # foo Main 1
1795 # foo.bar Main 1
1796
1797 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1798 return str
1799
1800 def normalise_slashes_( str ):
1801 str = re.sub('\\\\', '/', str)
1802 return str
1803
1804 def normalise_exe_( str ):
1805 str = re.sub('\.exe', '', str)
1806 return str
1807
1808 def normalise_output( str ):
1809 # Remove a .exe extension (for Windows)
1810 # This can occur in error messages generated by the program.
1811 str = re.sub('([^\\s])\\.exe', '\\1', str)
1812 return str
1813
1814 def normalise_asm( str ):
1815 lines = str.split('\n')
1816 # Only keep instructions and labels not starting with a dot.
1817 metadata = re.compile('^[ \t]*\\..*$')
1818 out = []
1819 for line in lines:
1820 # Drop metadata directives (e.g. ".type")
1821 if not metadata.match(line):
1822 line = re.sub('@plt', '', line)
1823 instr = line.lstrip().split()
1824 # Drop empty lines.
1825 if not instr:
1826 continue
1827 # Drop operands, except for call instructions.
1828 elif instr[0] == 'call':
1829 out.append(instr[0] + ' ' + instr[1])
1830 else:
1831 out.append(instr[0])
1832 out = '\n'.join(out)
1833 return out
1834
1835 def if_verbose( n, str ):
1836 if config.verbose >= n:
1837 print str
1838
1839 def if_verbose_dump( n, f ):
1840 if config.verbose >= n:
1841 try:
1842 print open(f).read()
1843 except:
1844 print ''
1845
1846 def rawSystem(cmd_and_args):
1847 # We prefer subprocess.call to os.spawnv as the latter
1848 # seems to send its arguments through a shell or something
1849 # with the Windows (non-cygwin) python. An argument "a b c"
1850 # turns into three arguments ["a", "b", "c"].
1851
1852 # However, subprocess is new in python 2.4, so fall back to
1853 # using spawnv if we don't have it
1854
1855 if have_subprocess:
1856 return subprocess.call(cmd_and_args)
1857 else:
1858 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1859
1860 # Note that this doesn't handle the timeout itself; it is just used for
1861 # commands that have timeout handling built-in.
1862 def rawSystemWithTimeout(cmd_and_args):
1863 r = rawSystem(cmd_and_args)
1864 if r == 98:
1865 # The python timeout program uses 98 to signal that ^C was pressed
1866 stopNow()
1867 return r
1868
1869 # cmd is a complex command in Bourne-shell syntax
1870 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1871 # Hence it must ultimately be run by a Bourne shell
1872 #
1873 # Mostly it invokes the command wrapped in 'timeout' thus
1874 # timeout 300 'cd . && ...blah blah'
1875 # so it's timeout's job to invoke the Bourne shell
1876 #
1877 # But watch out for the case when there is no timeout program!
1878 # Then, when using the native Python, os.system will invoke the cmd shell
1879
1880 def runCmd( cmd ):
1881 if_verbose( 1, cmd )
1882 r = 0
1883 if config.os == 'mingw32':
1884 # On MinGW, we will always have timeout
1885 assert config.timeout_prog!=''
1886
1887 if config.timeout_prog != '':
1888 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1889 else:
1890 r = os.system(cmd)
1891 return r << 8
1892
1893 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1894 if_verbose( 1, cmd )
1895 r = 0
1896 if config.os == 'mingw32':
1897 # On MinGW, we will always have timeout
1898 assert config.timeout_prog!=''
1899 timeout = int(ceil(config.timeout * timeout_multiplier))
1900
1901 if config.timeout_prog != '':
1902 if config.check_files_written:
1903 fn = name + ".strace"
1904 r = rawSystemWithTimeout(
1905 ["strace", "-o", fn, "-fF",
1906 "-e", "creat,open,chdir,clone,vfork",
1907 config.timeout_prog, str(timeout), cmd])
1908 addTestFilesWritten(name, fn)
1909 rm_no_fail(fn)
1910 else:
1911 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1912 else:
1913 r = os.system(cmd)
1914 return r << 8
1915
1916 def runCmdExitCode( cmd ):
1917 return (runCmd(cmd) >> 8);
1918
1919
1920 # -----------------------------------------------------------------------------
1921 # checking for files being written to by multiple tests
1922
1923 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1924 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1925 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1926 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1927 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1928 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1929 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1930 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1931 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1932 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1933 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1934 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1935 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1936 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1937 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1938 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1939 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1940
1941 # Files that are read or written but shouldn't be:
1942 # * ghci_history shouldn't be read or written by tests
1943 # * things under package.conf.d shouldn't be written by tests
1944 bad_file_usages = {}
1945
1946 # Mapping from tests to the list of files that they write
1947 files_written = {}
1948
1949 # Mapping from tests to the list of files that they write but don't clean
1950 files_written_not_removed = {}
1951
1952 def add_bad_file_usage(name, file):
1953 try:
1954 if not file in bad_file_usages[name]:
1955 bad_file_usages[name].append(file)
1956 except:
1957 bad_file_usages[name] = [file]
1958
1959 def mkPath(curdir, path):
1960 # Given the current full directory is 'curdir', what is the full
1961 # path to 'path'?
1962 return os.path.realpath(os.path.join(curdir, path))
1963
1964 def addTestFilesWritten(name, fn):
1965 if config.use_threads:
1966 with t.lockFilesWritten:
1967 addTestFilesWrittenHelper(name, fn)
1968 else:
1969 addTestFilesWrittenHelper(name, fn)
1970
1971 def addTestFilesWrittenHelper(name, fn):
1972 started = False
1973 working_directories = {}
1974
1975 with open(fn, 'r') as f:
1976 for line in f:
1977 m_pid = re_strace_pid.match(line)
1978 if m_pid:
1979 pid = m_pid.group(1)
1980 content = m_pid.group(2)
1981 elif re_strace_unavailable.match(line):
1982 next
1983 else:
1984 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1985
1986 m_open = re_strace_open.match(content)
1987 m_chdir = re_strace_chdir.match(content)
1988 m_clone = re_strace_clone.match(content)
1989 m_vfork = re_strace_vfork.match(content)
1990
1991 if not started:
1992 working_directories[pid] = os.getcwd()
1993 started = True
1994
1995 if m_open:
1996 file = m_open.group(1)
1997 file = mkPath(working_directories[pid], file)
1998 if file.endswith("ghci_history"):
1999 add_bad_file_usage(name, file)
2000 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
2001 flags = m_open.group(2).split('|')
2002 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
2003 if re.match('package\.conf\.d', file):
2004 add_bad_file_usage(name, file)
2005 else:
2006 try:
2007 if not file in files_written[name]:
2008 files_written[name].append(file)
2009 except:
2010 files_written[name] = [file]
2011 elif 'O_RDONLY' in flags:
2012 pass
2013 else:
2014 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2015 elif m_chdir:
2016 # We optimistically assume that unfinished chdir's are going to succeed
2017 dir = m_chdir.group(1)
2018 working_directories[pid] = mkPath(working_directories[pid], dir)
2019 elif m_clone:
2020 working_directories[m_clone.group(2)] = working_directories[pid]
2021 elif m_vfork:
2022 working_directories[m_vfork.group(2)] = working_directories[pid]
2023 elif re_strace_open_resumed.match(content):
2024 pass
2025 elif re_strace_chdir_resumed.match(content):
2026 pass
2027 elif re_strace_vfork_unfinished.match(content):
2028 pass
2029 elif re_strace_clone_unfinished.match(content):
2030 pass
2031 elif re_strace_ignore_sigchild.match(content):
2032 pass
2033 elif re_strace_ignore_sigvtalarm.match(content):
2034 pass
2035 elif re_strace_ignore_sigint.match(content):
2036 pass
2037 elif re_strace_ignore_sigfpe.match(content):
2038 pass
2039 elif re_strace_ignore_sigsegv.match(content):
2040 pass
2041 elif re_strace_ignore_sigpipe.match(content):
2042 pass
2043 else:
2044 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2045
2046 def checkForFilesWrittenProblems(file):
2047 foundProblem = False
2048
2049 files_written_inverted = {}
2050 for t in files_written.keys():
2051 for f in files_written[t]:
2052 try:
2053 files_written_inverted[f].append(t)
2054 except:
2055 files_written_inverted[f] = [t]
2056
2057 for f in files_written_inverted.keys():
2058 if len(files_written_inverted[f]) > 1:
2059 if not foundProblem:
2060 foundProblem = True
2061 file.write("\n")
2062 file.write("\nSome files are written by multiple tests:\n")
2063 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2064 if foundProblem:
2065 file.write("\n")
2066
2067 # -----
2068
2069 if len(files_written_not_removed) > 0:
2070 file.write("\n")
2071 file.write("\nSome files written but not removed:\n")
2072 tests = files_written_not_removed.keys()
2073 tests.sort()
2074 for t in tests:
2075 for f in files_written_not_removed[t]:
2076 file.write(" " + t + ": " + f + "\n")
2077 file.write("\n")
2078
2079 # -----
2080
2081 if len(bad_file_usages) > 0:
2082 file.write("\n")
2083 file.write("\nSome bad file usages:\n")
2084 tests = bad_file_usages.keys()
2085 tests.sort()
2086 for t in tests:
2087 for f in bad_file_usages[t]:
2088 file.write(" " + t + ": " + f + "\n")
2089 file.write("\n")
2090
2091 # -----------------------------------------------------------------------------
2092 # checking if ghostscript is available for checking the output of hp2ps
2093
2094 def genGSCmd(psfile):
2095 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2096
2097 def gsNotWorking():
2098 global gs_working
2099 print "GhostScript not available for hp2ps tests"
2100
2101 global gs_working
2102 gs_working = 0
2103 if config.have_profiling:
2104 if config.gs != '':
2105 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2106 if resultGood == 0:
2107 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2108 if resultBad != 0:
2109 print "GhostScript available for hp2ps tests"
2110 gs_working = 1;
2111 else:
2112 gsNotWorking();
2113 else:
2114 gsNotWorking();
2115 else:
2116 gsNotWorking();
2117
2118 def rm_no_fail( file ):
2119 try:
2120 os.remove( file )
2121 finally:
2122 return
2123
2124 def add_suffix( name, suffix ):
2125 if suffix == '':
2126 return name
2127 else:
2128 return name + '.' + suffix
2129
2130 def add_hs_lhs_suffix(name):
2131 if getTestOpts().c_src:
2132 return add_suffix(name, 'c')
2133 elif getTestOpts().cmm_src:
2134 return add_suffix(name, 'cmm')
2135 elif getTestOpts().objc_src:
2136 return add_suffix(name, 'm')
2137 elif getTestOpts().objcpp_src:
2138 return add_suffix(name, 'mm')
2139 elif getTestOpts().literate:
2140 return add_suffix(name, 'lhs')
2141 else:
2142 return add_suffix(name, 'hs')
2143
2144 def replace_suffix( name, suffix ):
2145 base, suf = os.path.splitext(name)
2146 return base + '.' + suffix
2147
2148 def in_testdir( name ):
2149 return (getTestOpts().testdir + '/' + name)
2150
2151 def qualify( name, suff ):
2152 return in_testdir(add_suffix(name, suff))
2153
2154
2155 # Finding the sample output. The filename is of the form
2156 #
2157 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2158 #
2159 # and we pick the most specific version available. The <version> is
2160 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2161 # more fine-grained control use if_compiler_lt().
2162 #
2163 def platform_wordsize_qualify( name, suff ):
2164
2165 basepath = qualify(name, suff)
2166
2167 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2168 for (platformSpecific, plat) in [(1, '-' + config.platform),
2169 (1, '-' + config.os),
2170 (0, '')]
2171 for ws in ['-ws-' + config.wordsize, '']
2172 for comp in ['-' + config.compiler_type, '']
2173 for vers in ['-' + config.compiler_maj_version, '']]
2174
2175 dir = glob.glob(basepath + '*')
2176 dir = map (lambda d: normalise_slashes_(d), dir)
2177
2178 for (platformSpecific, f) in paths:
2179 if f in dir:
2180 return (platformSpecific,f)
2181
2182 return (0, basepath)
2183
2184 # Clean up prior to the test, so that we can't spuriously conclude
2185 # that it passed on the basis of old run outputs.
2186 def pretest_cleanup(name):
2187 if getTestOpts().outputdir != None:
2188 odir = in_testdir(getTestOpts().outputdir)
2189 try:
2190 shutil.rmtree(odir)
2191 except:
2192 pass
2193 os.mkdir(odir)
2194
2195 rm_no_fail(qualify(name,'comp.stderr'))
2196 rm_no_fail(qualify(name,'run.stderr'))
2197 rm_no_fail(qualify(name,'run.stdout'))
2198 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2199 # simple_build zaps the following:
2200 # rm_nofail(qualify("o"))
2201 # rm_nofail(qualify(""))
2202 # not interested in the return code
2203
2204 # -----------------------------------------------------------------------------
2205 # Return a list of all the files ending in '.T' below the directory dir.
2206
2207 def findTFiles(roots):
2208 return concat(map(findTFiles_,roots))
2209
2210 def findTFiles_(path):
2211 if os.path.isdir(path):
2212 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2213 return findTFiles(paths)
2214 elif path[-2:] == '.T':
2215 return [path]
2216 else:
2217 return []
2218
2219 # -----------------------------------------------------------------------------
2220 # Output a test summary to the specified file object
2221
2222 def summary(t, file):
2223
2224 file.write('\n')
2225 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2226 file.write('OVERALL SUMMARY for test run started at ' \
2227 + t.start_time + '\n'\
2228 + string.rjust(`t.total_tests`, 8) \
2229 + ' total tests, which gave rise to\n' \
2230 + string.rjust(`t.total_test_cases`, 8) \
2231 + ' test cases, of which\n' \
2232 + string.rjust(`t.n_framework_failures`, 8) \
2233 + ' caused framework failures\n' \
2234 + string.rjust(`t.n_tests_skipped`, 8)
2235 + ' were skipped\n\n' \
2236 + string.rjust(`t.n_expected_passes`, 8)
2237 + ' expected passes\n' \
2238 + string.rjust(`t.n_missing_libs`, 8)
2239 + ' had missing libraries\n' \
2240 + string.rjust(`t.n_expected_failures`, 8) \
2241 + ' expected failures\n' \
2242 + string.rjust(`t.n_unexpected_passes`, 8) \
2243 + ' unexpected passes\n'
2244 + string.rjust(`t.n_unexpected_failures`, 8) \
2245 + ' unexpected failures\n'
2246 + '\n')
2247
2248 if t.n_unexpected_passes > 0:
2249 file.write('Unexpected passes:\n')
2250 printPassingTestInfosSummary(file, t.unexpected_passes)
2251
2252 if t.n_unexpected_failures > 0:
2253 file.write('Unexpected failures:\n')
2254 printFailingTestInfosSummary(file, t.unexpected_failures)
2255
2256 if config.check_files_written:
2257 checkForFilesWrittenProblems(file)
2258
2259 if stopping():
2260 file.write('WARNING: Testsuite run was terminated early\n')
2261
2262 def printUnexpectedTests(file, testInfoss):
2263 unexpected = []
2264 for testInfos in testInfoss:
2265 directories = testInfos.keys()
2266 for directory in directories:
2267 tests = testInfos[directory].keys()
2268 unexpected += tests
2269 if unexpected != []:
2270 file.write('Unexpected results from:\n')
2271 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2272 file.write('\n')
2273
2274 def printPassingTestInfosSummary(file, testInfos):
2275 directories = testInfos.keys()
2276 directories.sort()
2277 maxDirLen = max(map ((lambda x : len(x)), directories))
2278 for directory in directories:
2279 tests = testInfos[directory].keys()
2280 tests.sort()
2281 for test in tests:
2282 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2283 ' (' + join(testInfos[directory][test],',') + ')\n')
2284 file.write('\n')
2285
2286 def printFailingTestInfosSummary(file, testInfos):
2287 directories = testInfos.keys()
2288 directories.sort()
2289 maxDirLen = max(map ((lambda x : len(x)), directories))
2290 for directory in directories:
2291 tests = testInfos[directory].keys()
2292 tests.sort()
2293 for test in tests:
2294 reasons = testInfos[directory][test].keys()
2295 for reason in reasons:
2296 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2297 ' [' + reason + ']' + \
2298 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2299 file.write('\n')
2300
2301 def getStdout(cmd):
2302 if have_subprocess:
2303 p = subprocess.Popen(cmd,
2304 stdout=subprocess.PIPE,
2305 stderr=subprocess.PIPE)
2306 (stdout, stderr) = p.communicate()
2307 r = p.wait()
2308 if r != 0:
2309 raise Exception("Command failed: " + str(cmd))
2310 if stderr != '':
2311 raise Exception("stderr from command: " + str(cmd))
2312 return stdout
2313 else:
2314 raise Exception("Need subprocess to get stdout, but don't have it")