Merge branch 'master' of http://darcs.haskell.org/testsuite
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 opts.expect = 'fail';
138
139 def ignore_output( name, opts ):
140 opts.ignore_output = 1
141
142 def no_stdin( name, opts ):
143 opts.no_stdin = 1
144
145 def combined_output( name, opts ):
146 opts.combined_output = True
147
148 # -----
149
150 def expect_fail_for( ways ):
151 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
152
153 def _expect_fail_for( name, opts, ways ):
154 opts.expect_fail_for = ways
155
156 def expect_broken_for( bug, ways ):
157 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
158
159 def _expect_broken_for( name, opts, bug, ways ):
160 opts.expect_fail_for = ways
161
162 # -----
163
164 def omit_ways( ways ):
165 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
166
167 def _omit_ways( name, opts, ways ):
168 opts.omit_ways = ways
169
170 # -----
171
172 def only_ways( ways ):
173 return lambda name, opts, w=ways: _only_ways( name, opts, w )
174
175 def _only_ways( name, opts, ways ):
176 opts.only_ways = ways
177
178 # -----
179
180 def extra_ways( ways ):
181 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
182
183 def _extra_ways( name, opts, ways ):
184 opts.extra_ways = ways
185
186 # -----
187
188 def omit_compiler_types( compiler_types ):
189 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
190
191 def _omit_compiler_types( name, opts, compiler_types ):
192 if config.compiler_type in compiler_types:
193 opts.skip = 1
194
195 # -----
196
197 def only_compiler_types( compiler_types ):
198 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
199
200 def _only_compiler_types( name, opts, compiler_types ):
201 if config.compiler_type not in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def set_stdin( file ):
207 return lambda name, opts, f=file: _set_stdin(name, opts, f);
208
209 def _set_stdin( name, opts, f ):
210 opts.stdin = f
211
212 # -----
213
214 def exit_code( val ):
215 return lambda name, opts, v=val: _exit_code(name, opts, v);
216
217 def _exit_code( name, opts, v ):
218 opts.exit_code = v
219
220 # -----
221
222 def timeout_multiplier( val ):
223 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
224
225 def _timeout_multiplier( name, opts, v ):
226 opts.timeout_multiplier = v
227
228 # -----
229
230 def extra_run_opts( val ):
231 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
232
233 def _extra_run_opts( name, opts, v ):
234 opts.extra_run_opts = v
235
236 # -----
237
238 def extra_hc_opts( val ):
239 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
240
241 def _extra_hc_opts( name, opts, v ):
242 opts.extra_hc_opts = v
243
244 # -----
245
246 def extra_clean( files ):
247 return lambda name, opts, v=files: _extra_clean(name, opts, v);
248
249 def _extra_clean( name, opts, v ):
250 opts.clean_files = v
251
252 # -----
253
254 def stats_num_field( field, expecteds ):
255 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
256
257 def _stats_num_field( name, opts, field, expecteds ):
258 if field in opts.stats_range_fields:
259 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
260
261 if type(expecteds) is types.ListType:
262 for (b, expected, dev) in expecteds:
263 if b:
264 opts.stats_range_fields[field] = (expected, dev)
265 return
266 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
267
268 else:
269 (expected, dev) = expecteds
270 opts.stats_range_fields[field] = (expected, dev)
271
272 def stats_range_field( field, expected, dev ):
273 return stats_num_field( field, [(True, expected, dev)] )
274
275 def compiler_stats_num_field( field, expecteds ):
276 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
277
278 def _compiler_stats_num_field( name, opts, field, expecteds ):
279 if field in opts.compiler_stats_range_fields:
280 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
281
282 for (b, expected, dev) in expecteds:
283 if b:
284 opts.compiler_stats_range_fields[field] = (expected, dev)
285 return
286
287 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
288
289 def compiler_stats_range_field( field, expected, dev ):
290 return compiler_stats_num_field( field, [(True, expected, dev)] )
291
292 # -----
293
294 def skip_if_no_ghci(name, opts):
295 if not ('ghci' in config.run_ways):
296 opts.skip = 1
297
298 # ----
299
300 def skip_if_fast(name, opts):
301 if config.fast:
302 opts.skip = 1
303
304 # -----
305
306 def when(b, f):
307 if b:
308 return f
309 else:
310 return normal
311
312 def unless(b, f):
313 return when(not b, f)
314
315 def if_platform( plat, f ):
316 if config.platform == plat:
317 return f
318 else:
319 return normal
320
321 def unless_platform( plat, f ):
322 if config.platform != plat:
323 return f
324 else:
325 return normal
326
327 def if_os( os, f ):
328 if config.os == os:
329 return f
330 else:
331 return normal
332
333 def unless_os( os, f ):
334 if config.os == os:
335 return normal
336 else:
337 return f
338
339 def if_arch( arch, f ):
340 if config.arch == arch:
341 return f
342 else:
343 return normal
344
345 def unless_arch( arch, f ):
346 if config.arch == arch:
347 return normal
348 else:
349 return f
350
351 def wordsize( ws ):
352 return config.wordsize == str(ws)
353
354 def if_unregisterised( f ):
355 if config.unregisterised:
356 return f
357 else:
358 return normal
359
360 def unless_unregisterised( f ):
361 if config.unregisterised:
362 return normal
363 else:
364 return f
365
366 def if_msys( f ):
367 if config.msys:
368 return f
369 else:
370 return normal
371
372 def if_cygwin( f ):
373 if config.cygwin:
374 return f
375 else:
376 return normal
377
378 def when_have_vanilla( f ):
379 if config.have_vanilla:
380 return f
381 else:
382 return normal
383
384 def unless_have_vanilla( f ):
385 if config.have_vanilla:
386 return normal
387 else:
388 return f
389
390 def when_have_dynamic( f ):
391 if config.have_dynamic:
392 return f
393 else:
394 return normal
395
396 def unless_have_dynamic( f ):
397 if config.have_dynamic:
398 return normal
399 else:
400 return f
401
402 def when_have_profiling( f ):
403 if config.have_profiling:
404 return f
405 else:
406 return normal
407
408 def unless_have_profiling( f ):
409 if config.have_profiling:
410 return normal
411 else:
412 return f
413
414 # ---
415
416 def if_ghci_dynamic( f ):
417 if config.ghc_dynamic_by_default:
418 return f
419 else:
420 return normal
421
422 def if_in_tree_compiler( f ):
423 if config.in_tree_compiler:
424 return f
425 else:
426 return normal
427
428 def unless_in_tree_compiler( f ):
429 if config.in_tree_compiler:
430 return normal
431 else:
432 return f
433
434 def if_compiler_type( compiler, f ):
435 if config.compiler_type == compiler:
436 return f
437 else:
438 return normal
439
440 def if_compiler_profiled( f ):
441 if config.compiler_profiled:
442 return f
443 else:
444 return normal
445
446 def unless_compiler_profiled( f ):
447 if config.compiler_profiled:
448 return normal
449 else:
450 return f
451
452 def if_compiler_lt( compiler, version, f ):
453 if config.compiler_type == compiler and \
454 version_lt(config.compiler_version, version):
455 return f
456 else:
457 return normal
458
459 def if_compiler_le( compiler, version, f ):
460 if config.compiler_type == compiler and \
461 version_le(config.compiler_version, version):
462 return f
463 else:
464 return normal
465
466 def if_compiler_gt( compiler, version, f ):
467 if config.compiler_type == compiler and \
468 version_gt(config.compiler_version, version):
469 return f
470 else:
471 return normal
472
473 def if_compiler_ge( compiler, version, f ):
474 if config.compiler_type == compiler and \
475 version_ge(config.compiler_version, version):
476 return f
477 else:
478 return normal
479
480 def if_compiler_debugged( f ):
481 if config.compiler_debugged:
482 return f
483 else:
484 return normal
485
486 def namebase( nb ):
487 return lambda opts, nb=nb: _namebase(opts, nb)
488
489 def _namebase( opts, nb ):
490 opts.with_namebase = nb
491
492 # ---
493
494 def if_tag( tag, f ):
495 if tag in config.compiler_tags:
496 return f
497 else:
498 return normal
499
500 def unless_tag( tag, f ):
501 if not (tag in config.compiler_tags):
502 return f
503 else:
504 return normal
505
506 # ---
507 def high_memory_usage(name, opts):
508 opts.alone = True
509
510 # ---
511 def literate( name, opts ):
512 opts.literate = 1;
513
514 def c_src( name, opts ):
515 opts.c_src = 1;
516
517 def objc_src( name, opts ):
518 opts.objc_src = 1;
519
520 def objcpp_src( name, opts ):
521 opts.objcpp_src = 1;
522
523 def cmm_src( name, opts ):
524 opts.cmm_src = 1;
525
526 def outputdir( odir ):
527 return lambda name, opts, d=odir: _outputdir(name, opts, d)
528
529 def _outputdir( name, opts, odir ):
530 opts.outputdir = odir;
531
532 # ----
533
534 def pre_cmd( cmd ):
535 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
536
537 def _pre_cmd( name, opts, cmd ):
538 opts.pre_cmd = cmd
539
540 # ----
541
542 def clean_cmd( cmd ):
543 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
544
545 def _clean_cmd( name, opts, cmd ):
546 opts.clean_cmd = cmd
547
548 # ----
549
550 def cmd_prefix( prefix ):
551 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
552
553 def _cmd_prefix( name, opts, prefix ):
554 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
555
556 # ----
557
558 def cmd_wrapper( fun ):
559 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
560
561 def _cmd_wrapper( name, opts, fun ):
562 opts.cmd_wrapper = fun
563
564 # ----
565
566 def compile_cmd_prefix( prefix ):
567 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
568
569 def _compile_cmd_prefix( name, opts, prefix ):
570 opts.compile_cmd_prefix = prefix
571
572 # ----
573
574 def normalise_slashes( name, opts ):
575 opts.extra_normaliser = normalise_slashes_
576
577 def normalise_exe( name, opts ):
578 opts.extra_normaliser = normalise_exe_
579
580 def normalise_fun( fun ):
581 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
582
583 def _normalise_fun( name, opts, f ):
584 opts.extra_normaliser = f
585
586 def normalise_errmsg_fun( fun ):
587 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
588
589 def _normalise_errmsg_fun( name, opts, f ):
590 opts.extra_errmsg_normaliser = f
591
592 def two_normalisers(f, g):
593 return lambda x, f=f, g=g: f(g(x))
594
595 # ----
596 # Function for composing two opt-fns together
597
598 def composes( fs ):
599 return reduce(lambda f, g: compose(f, g), fs)
600
601 def compose( f, g ):
602 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
603
604 def _compose( name, opts, f, g ):
605 f(name, opts)
606 g(name, opts)
607
608 # -----------------------------------------------------------------------------
609 # The current directory of tests
610
611 def newTestDir( dir ):
612 global thisdir_settings
613 # reset the options for this test directory
614 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
615
616 def _newTestDir( name, opts, dir ):
617 opts.testdir = dir
618 opts.compiler_always_flags = config.compiler_always_flags
619
620 # -----------------------------------------------------------------------------
621 # Actually doing tests
622
623 parallelTests = []
624 aloneTests = []
625 allTestNames = set([])
626
627 def runTest (opts, name, func, args):
628 ok = 0
629
630 if config.use_threads:
631 t.thread_pool.acquire()
632 try:
633 while config.threads<(t.running_threads+1):
634 t.thread_pool.wait()
635 t.running_threads = t.running_threads+1
636 ok=1
637 t.thread_pool.release()
638 thread.start_new_thread(test_common_thread, (name, opts, func, args))
639 except:
640 if not ok:
641 t.thread_pool.release()
642 else:
643 test_common_work (name, opts, func, args)
644
645 # name :: String
646 # setup :: TestOpts -> IO ()
647 def test (name, setup, func, args):
648 global aloneTests
649 global parallelTests
650 global allTestNames
651 global thisdir_settings
652 if name in allTestNames:
653 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
654 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
655 framework_fail(name, 'bad_name', 'This test has an invalid name')
656
657 # Make a deep copy of the default_testopts, as we need our own copy
658 # of any dictionaries etc inside it. Otherwise, if one test modifies
659 # them, all tests will see the modified version!
660 myTestOpts = copy.deepcopy(default_testopts)
661
662 if type(setup) is types.ListType:
663 setup = composes(setup)
664
665 setup = compose(thisdir_settings, setup)
666 setup(name, myTestOpts)
667
668 thisTest = lambda : runTest(myTestOpts, name, func, args)
669 if myTestOpts.alone:
670 aloneTests.append(thisTest)
671 else:
672 parallelTests.append(thisTest)
673 allTestNames.add(name)
674
675 if config.use_threads:
676 def test_common_thread(name, opts, func, args):
677 t.lock.acquire()
678 try:
679 test_common_work(name,opts,func,args)
680 finally:
681 t.lock.release()
682 t.thread_pool.acquire()
683 t.running_threads = t.running_threads - 1
684 t.thread_pool.notify()
685 t.thread_pool.release()
686
687 def get_package_cache_timestamp():
688 if config.package_conf_cache_file == '':
689 return 0.0
690 else:
691 try:
692 return os.stat(config.package_conf_cache_file).st_mtime
693 except:
694 return 0.0
695
696
697 def test_common_work (name, opts, func, args):
698 try:
699 t.total_tests = t.total_tests+1
700 setLocalTestOpts(opts)
701
702 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
703
704 # All the ways we might run this test
705 if func == compile or func == multimod_compile:
706 all_ways = config.compile_ways
707 elif func == compile_and_run or func == multimod_compile_and_run:
708 all_ways = config.run_ways
709 elif func == ghci_script:
710 if 'ghci' in config.run_ways:
711 all_ways = ['ghci']
712 else:
713 all_ways = []
714 else:
715 all_ways = ['normal']
716
717 # A test itself can request extra ways by setting opts.extra_ways
718 all_ways = all_ways + filter(lambda way: way not in all_ways,
719 opts.extra_ways)
720
721 t.total_test_cases = t.total_test_cases + len(all_ways)
722
723 ok_way = lambda way: \
724 not getTestOpts().skip \
725 and (config.only == [] or name in config.only) \
726 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
727 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
728 and way not in getTestOpts().omit_ways
729
730 # Which ways we are asked to skip
731 do_ways = filter (ok_way,all_ways)
732
733 # In fast mode, we skip all but one way
734 if config.fast and len(do_ways) > 0:
735 do_ways = [do_ways[0]]
736
737 if not config.clean_only:
738 # Run the required tests...
739 for way in do_ways:
740 if stopping():
741 break
742 do_test (name, way, func, args)
743
744 for way in all_ways:
745 if way not in do_ways:
746 skiptest (name,way)
747
748 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
749 clean(map (lambda suff: name + suff,
750 ['', '.exe', '.exe.manifest', '.genscript',
751 '.stderr.normalised', '.stdout.normalised',
752 '.run.stderr', '.run.stdout',
753 '.run.stderr.normalised', '.run.stdout.normalised',
754 '.comp.stderr', '.comp.stdout',
755 '.comp.stderr.normalised', '.comp.stdout.normalised',
756 '.interp.stderr', '.interp.stdout',
757 '.interp.stderr.normalised', '.interp.stdout.normalised',
758 '.stats', '.comp.stats',
759 '.hi', '.o', '.prof', '.exe.prof', '.hc',
760 '_stub.h', '_stub.c', '_stub.o',
761 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
762
763 if func == multi_compile or func == multi_compile_fail:
764 extra_mods = args[1]
765 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
766 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
767
768 clean(getTestOpts().clean_files)
769
770 if getTestOpts().outputdir != None:
771 odir = in_testdir(getTestOpts().outputdir)
772 try:
773 shutil.rmtree(odir)
774 except:
775 pass
776
777 try:
778 shutil.rmtree(in_testdir('.hpc.' + name))
779 except:
780 pass
781
782 try:
783 cleanCmd = getTestOpts().clean_cmd
784 if cleanCmd != None:
785 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
786 if result != 0:
787 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
788 except e:
789 framework_fail(name, 'cleaning', 'clean-command exception')
790
791 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
792
793 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
794 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
795
796 try:
797 for f in files_written[name]:
798 if os.path.exists(f):
799 try:
800 if not f in files_written_not_removed[name]:
801 files_written_not_removed[name].append(f)
802 except:
803 files_written_not_removed[name] = [f]
804 except:
805 pass
806 except Exception, e:
807 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
808
809 def clean(strs):
810 for str in strs:
811 for name in glob.glob(in_testdir(str)):
812 clean_full_path(name)
813
814 def clean_full_path(name):
815 try:
816 # Remove files...
817 os.remove(name)
818 except OSError, e1:
819 try:
820 # ... and empty directories
821 os.rmdir(name)
822 except OSError, e2:
823 # We don't want to fail here, but we do want to know
824 # what went wrong, so print out the exceptions.
825 # ENOENT isn't a problem, though, as we clean files
826 # that don't necessarily exist.
827 if e1.errno != errno.ENOENT:
828 print e1
829 if e2.errno != errno.ENOENT:
830 print e2
831
832 def do_test(name, way, func, args):
833 full_name = name + '(' + way + ')'
834
835 try:
836 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
837 str([t.n_unexpected_passes, \
838 t.n_unexpected_failures, \
839 t.n_framework_failures])
840
841 if config.use_threads:
842 t.lock.release()
843
844 try:
845 preCmd = getTestOpts().pre_cmd
846 if preCmd != None:
847 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
848 if result != 0:
849 framework_fail(name, way, 'pre-command failed: ' + str(result))
850 except e:
851 framework_fail(name, way, 'pre-command exception')
852
853 try:
854 result = apply(func, [name,way] + args)
855 finally:
856 if config.use_threads:
857 t.lock.acquire()
858
859 if getTestOpts().expect != 'pass' and \
860 getTestOpts().expect != 'fail' and \
861 getTestOpts().expect != 'missing-lib':
862 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
863
864 try:
865 passFail = result['passFail']
866 except:
867 passFail = 'No passFail found'
868
869 if passFail == 'pass':
870 if getTestOpts().expect == 'pass' \
871 and way not in getTestOpts().expect_fail_for:
872 t.n_expected_passes = t.n_expected_passes + 1
873 if name in t.expected_passes:
874 t.expected_passes[name].append(way)
875 else:
876 t.expected_passes[name] = [way]
877 else:
878 print '*** unexpected pass for', full_name
879 t.n_unexpected_passes = t.n_unexpected_passes + 1
880 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
881 elif passFail == 'fail':
882 if getTestOpts().expect == 'pass' \
883 and way not in getTestOpts().expect_fail_for:
884 print '*** unexpected failure for', full_name
885 t.n_unexpected_failures = t.n_unexpected_failures + 1
886 reason = result['reason']
887 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
888 else:
889 if getTestOpts().expect == 'missing-lib':
890 t.n_missing_libs = t.n_missing_libs + 1
891 if name in t.missing_libs:
892 t.missing_libs[name].append(way)
893 else:
894 t.missing_libs[name] = [way]
895 else:
896 t.n_expected_failures = t.n_expected_failures + 1
897 if name in t.expected_failures:
898 t.expected_failures[name].append(way)
899 else:
900 t.expected_failures[name] = [way]
901 else:
902 framework_fail(name, way, 'bad result ' + passFail)
903 except KeyboardInterrupt:
904 stopNow()
905 except:
906 framework_fail(name, way, 'do_test exception')
907 traceback.print_exc()
908
909 def addPassingTestInfo (testInfos, directory, name, way):
910 directory = re.sub('^\\.[/\\\\]', '', directory)
911
912 if not directory in testInfos:
913 testInfos[directory] = {}
914
915 if not name in testInfos[directory]:
916 testInfos[directory][name] = []
917
918 testInfos[directory][name].append(way)
919
920 def addFailingTestInfo (testInfos, directory, name, reason, way):
921 directory = re.sub('^\\.[/\\\\]', '', directory)
922
923 if not directory in testInfos:
924 testInfos[directory] = {}
925
926 if not name in testInfos[directory]:
927 testInfos[directory][name] = {}
928
929 if not reason in testInfos[directory][name]:
930 testInfos[directory][name][reason] = []
931
932 testInfos[directory][name][reason].append(way)
933
934 def skiptest (name, way):
935 # print 'Skipping test \"', name, '\"'
936 t.n_tests_skipped = t.n_tests_skipped + 1
937 if name in t.tests_skipped:
938 t.tests_skipped[name].append(way)
939 else:
940 t.tests_skipped[name] = [way]
941
942 def framework_fail( name, way, reason ):
943 full_name = name + '(' + way + ')'
944 print '*** framework failure for', full_name, reason
945 t.n_framework_failures = t.n_framework_failures + 1
946 if name in t.framework_failures:
947 t.framework_failures[name].append(way)
948 else:
949 t.framework_failures[name] = [way]
950
951 def badResult(result):
952 try:
953 if result['passFail'] == 'pass':
954 return False
955 return True
956 except:
957 return True
958
959 def passed():
960 return {'passFail': 'pass'}
961
962 def failBecause(reason):
963 return {'passFail': 'fail', 'reason': reason}
964
965 # -----------------------------------------------------------------------------
966 # Generic command tests
967
968 # A generic command test is expected to run and exit successfully.
969 #
970 # The expected exit code can be changed via exit_code() as normal, and
971 # the expected stdout/stderr are stored in <testname>.stdout and
972 # <testname>.stderr. The output of the command can be ignored
973 # altogether by using run_command_ignore_output instead of
974 # run_command.
975
976 def run_command( name, way, cmd ):
977 return simple_run( name, '', cmd, '' )
978
979 # -----------------------------------------------------------------------------
980 # GHCi tests
981
982 def ghci_script( name, way, script ):
983 # filter out -fforce-recomp from compiler_always_flags, because we're
984 # actually testing the recompilation behaviour in the GHCi tests.
985 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
986 flags.append(getTestOpts().extra_hc_opts)
987 if getTestOpts().outputdir != None:
988 flags.extend(["-outputdir", getTestOpts().outputdir])
989
990 # We pass HC and HC_OPTS as environment variables, so that the
991 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
992 cmd = "HC='" + config.compiler + "' " + \
993 "HC_OPTS='" + join(flags,' ') + "' " + \
994 "'" + config.compiler + "'" + \
995 ' --interactive -v0 -ignore-dot-ghci ' + \
996 join(flags,' ')
997
998 getTestOpts().stdin = script
999 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1000
1001 # -----------------------------------------------------------------------------
1002 # Compile-only tests
1003
1004 def compile( name, way, extra_hc_opts ):
1005 return do_compile( name, way, 0, '', [], extra_hc_opts )
1006
1007 def compile_fail( name, way, extra_hc_opts ):
1008 return do_compile( name, way, 1, '', [], extra_hc_opts )
1009
1010 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1011 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1012
1013 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1014 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1015
1016 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1017 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1018
1019 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1020 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1021
1022 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1023 # print 'Compile only, extra args = ', extra_hc_opts
1024 pretest_cleanup(name)
1025
1026 result = extras_build( way, extra_mods, extra_hc_opts )
1027 if badResult(result):
1028 return result
1029 extra_hc_opts = result['hc_opts']
1030
1031 force = 0
1032 if extra_mods:
1033 force = 1
1034 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1035
1036 if badResult(result):
1037 return result
1038
1039 # the actual stderr should always match the expected, regardless
1040 # of whether we expected the compilation to fail or not (successful
1041 # compilations may generate warnings).
1042
1043 if getTestOpts().with_namebase == None:
1044 namebase = name
1045 else:
1046 namebase = getTestOpts().with_namebase
1047
1048 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1049 actual_stderr_file = qualify(name, 'comp.stderr')
1050
1051 if not compare_outputs('stderr', \
1052 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1053 expected_stderr_file, actual_stderr_file):
1054 return failBecause('stderr mismatch')
1055
1056 # no problems found, this test passed
1057 return passed()
1058
1059 def compile_cmp_asm( name, way, extra_hc_opts ):
1060 print 'Compile only, extra args = ', extra_hc_opts
1061 pretest_cleanup(name)
1062 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1063
1064 if badResult(result):
1065 return result
1066
1067 # the actual stderr should always match the expected, regardless
1068 # of whether we expected the compilation to fail or not (successful
1069 # compilations may generate warnings).
1070
1071 if getTestOpts().with_namebase == None:
1072 namebase = name
1073 else:
1074 namebase = getTestOpts().with_namebase
1075
1076 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1077 actual_asm_file = qualify(name, 's')
1078
1079 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1080 expected_asm_file, actual_asm_file):
1081 return failBecause('asm mismatch')
1082
1083 # no problems found, this test passed
1084 return passed()
1085
1086 # -----------------------------------------------------------------------------
1087 # Compile-and-run tests
1088
1089 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1090 # print 'Compile and run, extra args = ', extra_hc_opts
1091 pretest_cleanup(name)
1092
1093 result = extras_build( way, extra_mods, extra_hc_opts )
1094 if badResult(result):
1095 return result
1096 extra_hc_opts = result['hc_opts']
1097
1098 if way == 'ghci': # interpreted...
1099 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1100 elif way == 'extcore' or way == 'optextcore' :
1101 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1102 else: # compiled...
1103 force = 0
1104 if extra_mods:
1105 force = 1
1106
1107 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1108 if badResult(result):
1109 return result
1110
1111 cmd = './' + name;
1112
1113 # we don't check the compiler's stderr for a compile-and-run test
1114 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1115
1116 def compile_and_run( name, way, extra_hc_opts ):
1117 return compile_and_run__( name, way, '', [], extra_hc_opts)
1118
1119 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1120 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1121
1122 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1123 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1124
1125 def stats( name, way, stats_file ):
1126 opts = getTestOpts()
1127 return checkStats(stats_file, opts.stats_range_fields)
1128
1129 # -----------------------------------------------------------------------------
1130 # Check -t stats info
1131
1132 def checkStats(stats_file, range_fields):
1133 result = passed()
1134 if len(range_fields) > 0:
1135 f = open(in_testdir(stats_file))
1136 contents = f.read()
1137 f.close()
1138
1139 for (field, (expected, dev)) in range_fields.items():
1140 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1141 if m == None:
1142 print 'Failed to find field: ', field
1143 result = failBecause('no such stats field')
1144 val = int(m.group(1))
1145
1146 lowerBound = trunc( expected * ((100 - float(dev))/100));
1147 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1148
1149 if val < lowerBound:
1150 print field, 'value is too low:'
1151 print '(If this is because you have improved GHC, please'
1152 print 'update the test so that GHC doesn\'t regress again)'
1153 result = failBecause('stat too good')
1154 if val > upperBound:
1155 print field, 'value is too high:'
1156 result = failBecause('stat not good enough')
1157
1158 if val < lowerBound or val > upperBound:
1159 valStr = str(val)
1160 valLen = len(valStr)
1161 expectedStr = str(expected)
1162 expectedLen = len(expectedStr)
1163 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1164 def display(descr, val, extra):
1165 print descr, string.rjust(str(val), length), extra
1166 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1167 display(' Lower bound ' + field + ':', lowerBound, '')
1168 display(' Upper bound ' + field + ':', upperBound, '')
1169 display(' Actual ' + field + ':', val, '')
1170
1171 return result
1172
1173 # -----------------------------------------------------------------------------
1174 # Build a single-module program
1175
1176 def extras_build( way, extra_mods, extra_hc_opts ):
1177 for modopts in extra_mods:
1178 mod, opts = modopts
1179 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1180 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1181 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1182 if badResult(result):
1183 return result
1184
1185 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1186
1187
1188 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1189 opts = getTestOpts()
1190 errname = add_suffix(name, 'comp.stderr')
1191 rm_no_fail( qualify(errname, '') )
1192
1193 if top_mod != '':
1194 srcname = top_mod
1195 rm_no_fail( qualify(name, '') )
1196 base, suf = os.path.splitext(top_mod)
1197 rm_no_fail( qualify(base, '') )
1198 rm_no_fail( qualify(base, 'exe') )
1199 elif addsuf:
1200 srcname = add_hs_lhs_suffix(name)
1201 rm_no_fail( qualify(name, '') )
1202 else:
1203 srcname = name
1204 rm_no_fail( qualify(name, 'o') )
1205
1206 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1207
1208 to_do = ''
1209 if top_mod != '':
1210 to_do = '--make '
1211 if link:
1212 to_do = to_do + '-o ' + name
1213 elif link:
1214 to_do = '-o ' + name
1215 elif opts.compile_to_hc:
1216 to_do = '-C'
1217 else:
1218 to_do = '-c' # just compile
1219
1220 stats_file = name + '.comp.stats'
1221 if len(opts.compiler_stats_range_fields) > 0:
1222 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1223
1224 # Required by GHC 7.3+, harmless for earlier versions:
1225 if (getTestOpts().c_src or
1226 getTestOpts().objc_src or
1227 getTestOpts().objcpp_src or
1228 getTestOpts().cmm_src):
1229 extra_hc_opts += ' -no-hs-main '
1230
1231 if getTestOpts().compile_cmd_prefix == '':
1232 cmd_prefix = ''
1233 else:
1234 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1235
1236 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1237 if noforce:
1238 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1239 if getTestOpts().outputdir != None:
1240 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1241
1242 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1243 + config.compiler + "' " \
1244 + join(comp_flags,' ') + ' ' \
1245 + to_do + ' ' + srcname + ' ' \
1246 + join(config.way_flags(name)[way],' ') + ' ' \
1247 + extra_hc_opts + ' ' \
1248 + opts.extra_hc_opts + ' ' \
1249 + '>' + errname + ' 2>&1'
1250
1251 result = runCmdFor(name, cmd)
1252
1253 if result != 0 and not should_fail:
1254 actual_stderr = qualify(name, 'comp.stderr')
1255 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1256 if_verbose_dump(1,actual_stderr)
1257
1258 # ToDo: if the sub-shell was killed by ^C, then exit
1259
1260 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1261
1262 if badResult(statsResult):
1263 return statsResult
1264
1265 if should_fail:
1266 if result == 0:
1267 return failBecause('exit code 0')
1268 else:
1269 if result != 0:
1270 return failBecause('exit code non-0')
1271
1272 return passed()
1273
1274 # -----------------------------------------------------------------------------
1275 # Run a program and check its output
1276 #
1277 # If testname.stdin exists, route input from that, else
1278 # from /dev/null. Route output to testname.run.stdout and
1279 # testname.run.stderr. Returns the exit code of the run.
1280
1281 def simple_run( name, way, prog, args ):
1282 opts = getTestOpts()
1283
1284 # figure out what to use for stdin
1285 if opts.stdin != '':
1286 use_stdin = opts.stdin
1287 else:
1288 stdin_file = add_suffix(name, 'stdin')
1289 if os.path.exists(in_testdir(stdin_file)):
1290 use_stdin = stdin_file
1291 else:
1292 use_stdin = '/dev/null'
1293
1294 run_stdout = add_suffix(name,'run.stdout')
1295 run_stderr = add_suffix(name,'run.stderr')
1296
1297 rm_no_fail(qualify(name,'run.stdout'))
1298 rm_no_fail(qualify(name,'run.stderr'))
1299 rm_no_fail(qualify(name, 'hp'))
1300 rm_no_fail(qualify(name,'ps'))
1301 rm_no_fail(qualify(name, 'prof'))
1302
1303 my_rts_flags = rts_flags(way)
1304
1305 stats_file = name + '.stats'
1306 if len(opts.stats_range_fields) > 0:
1307 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1308
1309 if opts.no_stdin:
1310 stdin_comes_from = ''
1311 else:
1312 stdin_comes_from = ' <' + use_stdin
1313
1314 if opts.combined_output:
1315 redirection = ' >' + run_stdout \
1316 + ' 2>&1'
1317 else:
1318 redirection = ' >' + run_stdout \
1319 + ' 2>' + run_stderr
1320
1321 cmd = prog + ' ' + args + ' ' \
1322 + my_rts_flags + ' ' \
1323 + stdin_comes_from \
1324 + redirection
1325
1326 if opts.cmd_wrapper != None:
1327 cmd = opts.cmd_wrapper(cmd);
1328
1329 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1330
1331 # run the command
1332 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1333
1334 exit_code = result >> 8
1335 signal = result & 0xff
1336
1337 # check the exit code
1338 if exit_code != opts.exit_code:
1339 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1340 dump_stdout(name)
1341 dump_stderr(name)
1342 return failBecause('bad exit code')
1343
1344 check_hp = my_rts_flags.find("-h") != -1
1345 check_prof = my_rts_flags.find("-p") != -1
1346
1347 if not opts.ignore_output:
1348 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1349 bad_stdout = not check_stdout_ok(name)
1350 if bad_stderr:
1351 return failBecause('bad stderr')
1352 if bad_stdout:
1353 return failBecause('bad stdout')
1354 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1355 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1356 return failBecause('bad heap profile')
1357 if check_prof and not check_prof_ok(name):
1358 return failBecause('bad profile')
1359
1360 return checkStats(stats_file, opts.stats_range_fields)
1361
1362 def rts_flags(way):
1363 if (way == ''):
1364 return ''
1365 else:
1366 args = config.way_rts_flags[way]
1367
1368 if args == []:
1369 return ''
1370 else:
1371 return '+RTS ' + join(args,' ') + ' -RTS'
1372
1373 # -----------------------------------------------------------------------------
1374 # Run a program in the interpreter and check its output
1375
1376 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1377 outname = add_suffix(name, 'interp.stdout')
1378 errname = add_suffix(name, 'interp.stderr')
1379 rm_no_fail(outname)
1380 rm_no_fail(errname)
1381 rm_no_fail(name)
1382
1383 if (top_mod == ''):
1384 srcname = add_hs_lhs_suffix(name)
1385 else:
1386 srcname = top_mod
1387
1388 scriptname = add_suffix(name, 'genscript')
1389 qscriptname = in_testdir(scriptname)
1390 rm_no_fail(qscriptname)
1391
1392 delimiter = '===== program output begins here\n'
1393
1394 script = open(qscriptname, 'w')
1395 if not compile_only:
1396 # set the prog name and command-line args to match the compiled
1397 # environment.
1398 script.write(':set prog ' + name + '\n')
1399 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1400 # Add marker lines to the stdout and stderr output files, so we
1401 # can separate GHCi's output from the program's.
1402 script.write(':! echo ' + delimiter)
1403 script.write(':! echo 1>&2 ' + delimiter)
1404 # Set stdout to be line-buffered to match the compiled environment.
1405 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1406 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1407 # in the event of an exception as for the compiled program.
1408 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1409 script.close()
1410
1411 # figure out what to use for stdin
1412 if getTestOpts().stdin != '':
1413 stdin_file = in_testdir(getTestOpts().stdin)
1414 else:
1415 stdin_file = qualify(name, 'stdin')
1416
1417 if os.path.exists(stdin_file):
1418 stdin = open(stdin_file, 'r')
1419 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1420
1421 script.close()
1422
1423 flags = copy.copy(getTestOpts().compiler_always_flags)
1424 if getTestOpts().outputdir != None:
1425 flags.extend(["-outputdir", getTestOpts().outputdir])
1426
1427 cmd = "'" + config.compiler + "' " \
1428 + join(flags,' ') + ' ' \
1429 + srcname + ' ' \
1430 + join(config.way_flags(name)[way],' ') + ' ' \
1431 + extra_hc_opts + ' ' \
1432 + getTestOpts().extra_hc_opts + ' ' \
1433 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1434
1435 if getTestOpts().cmd_wrapper != None:
1436 cmd = getTestOpts().cmd_wrapper(cmd);
1437
1438 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1439
1440 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1441
1442 exit_code = result >> 8
1443 signal = result & 0xff
1444
1445 # split the stdout into compilation/program output
1446 split_file(in_testdir(outname), delimiter,
1447 qualify(name, 'comp.stdout'),
1448 qualify(name, 'run.stdout'))
1449 split_file(in_testdir(errname), delimiter,
1450 qualify(name, 'comp.stderr'),
1451 qualify(name, 'run.stderr'))
1452
1453 # check the exit code
1454 if exit_code != getTestOpts().exit_code:
1455 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1456 dump_stdout(name)
1457 dump_stderr(name)
1458 return failBecause('bad exit code')
1459
1460 # ToDo: if the sub-shell was killed by ^C, then exit
1461
1462 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1463 check_stdout_ok(name)):
1464 return passed()
1465 else:
1466 return failBecause('bad stdout or stderr')
1467
1468
1469 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1470 infile = open(in_fn)
1471 out1 = open(out1_fn, 'w')
1472 out2 = open(out2_fn, 'w')
1473
1474 line = infile.readline()
1475 line = re.sub('\r', '', line) # ignore Windows EOL
1476 while (re.sub('^\s*','',line) != delimiter and line != ''):
1477 out1.write(line)
1478 line = infile.readline()
1479 line = re.sub('\r', '', line)
1480 out1.close()
1481
1482 line = infile.readline()
1483 while (line != ''):
1484 out2.write(line)
1485 line = infile.readline()
1486 out2.close()
1487
1488 # -----------------------------------------------------------------------------
1489 # Generate External Core for the given program, then compile the resulting Core
1490 # and compare its output to the expected output
1491
1492 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1493
1494 depsfilename = qualify(name, 'deps')
1495 errname = add_suffix(name, 'comp.stderr')
1496 qerrname = qualify(errname,'')
1497
1498 hcname = qualify(name, 'hc')
1499 oname = qualify(name, 'o')
1500
1501 rm_no_fail( qerrname )
1502 rm_no_fail( qualify(name, '') )
1503
1504 if (top_mod == ''):
1505 srcname = add_hs_lhs_suffix(name)
1506 else:
1507 srcname = top_mod
1508
1509 qcorefilename = qualify(name, 'hcr')
1510 corefilename = add_suffix(name, 'hcr')
1511 rm_no_fail(qcorefilename)
1512
1513 # Generate External Core
1514
1515 if (top_mod == ''):
1516 to_do = ' ' + srcname + ' '
1517 else:
1518 to_do = ' --make ' + top_mod + ' '
1519
1520 flags = copy.copy(getTestOpts().compiler_always_flags)
1521 if getTestOpts().outputdir != None:
1522 flags.extend(["-outputdir", getTestOpts().outputdir])
1523 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1524 + config.compiler + "' " \
1525 + join(flags,' ') + ' ' \
1526 + join(config.way_flags(name)[way],' ') + ' ' \
1527 + extra_hc_opts + ' ' \
1528 + getTestOpts().extra_hc_opts \
1529 + to_do \
1530 + '>' + errname + ' 2>&1'
1531 result = runCmdFor(name, cmd)
1532
1533 exit_code = result >> 8
1534
1535 if exit_code != 0:
1536 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1537 if_verbose_dump(1,qerrname)
1538 return failBecause('ext core exit code non-0')
1539
1540 # Compile the resulting files -- if there's more than one module, we need to read the output
1541 # of the previous compilation in order to find the dependencies
1542 if (top_mod == ''):
1543 to_compile = corefilename
1544 else:
1545 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1546 deps = open(depsfilename).read()
1547 deplist = string.replace(deps, '\n',' ');
1548 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1549 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1550
1551 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1552 if getTestOpts().outputdir != None:
1553 flags.extend(["-outputdir", getTestOpts().outputdir])
1554
1555 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1556 + config.compiler + "' " \
1557 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1558 + to_compile + ' ' \
1559 + extra_hc_opts + ' ' \
1560 + getTestOpts().extra_hc_opts + ' ' \
1561 + flags \
1562 + ' -fglasgow-exts -o ' + name \
1563 + '>' + errname + ' 2>&1'
1564
1565 result = runCmdFor(name, cmd)
1566 exit_code = result >> 8
1567
1568 if exit_code != 0:
1569 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1570 if_verbose_dump(1,qerrname)
1571 return failBecause('ext core exit code non-0')
1572
1573 # Clean up
1574 rm_no_fail ( oname )
1575 rm_no_fail ( hcname )
1576 rm_no_fail ( qcorefilename )
1577 rm_no_fail ( depsfilename )
1578
1579 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1580
1581 # -----------------------------------------------------------------------------
1582 # Utils
1583
1584 def check_stdout_ok( name ):
1585 if getTestOpts().with_namebase == None:
1586 namebase = name
1587 else:
1588 namebase = getTestOpts().with_namebase
1589
1590 actual_stdout_file = qualify(name, 'run.stdout')
1591 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1592
1593 def norm(str):
1594 if platform_specific:
1595 return str
1596 else:
1597 return normalise_output(str)
1598
1599 return compare_outputs('stdout', \
1600 two_normalisers(norm, getTestOpts().extra_normaliser), \
1601 expected_stdout_file, actual_stdout_file)
1602
1603 def dump_stdout( name ):
1604 print 'Stdout:'
1605 print read_no_crs(qualify(name, 'run.stdout'))
1606
1607 def check_stderr_ok( name ):
1608 if getTestOpts().with_namebase == None:
1609 namebase = name
1610 else:
1611 namebase = getTestOpts().with_namebase
1612
1613 actual_stderr_file = qualify(name, 'run.stderr')
1614 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1615
1616 def norm(str):
1617 if platform_specific:
1618 return str
1619 else:
1620 return normalise_errmsg(str)
1621
1622 return compare_outputs('stderr', \
1623 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1624 expected_stderr_file, actual_stderr_file)
1625
1626 def dump_stderr( name ):
1627 print "Stderr:"
1628 print read_no_crs(qualify(name, 'run.stderr'))
1629
1630 def read_no_crs(file):
1631 str = ''
1632 try:
1633 h = open(file)
1634 str = h.read()
1635 h.close
1636 except:
1637 # On Windows, if the program fails very early, it seems the
1638 # files stdout/stderr are redirected to may not get created
1639 pass
1640 return re.sub('\r', '', str)
1641
1642 def write_file(file, str):
1643 h = open(file, 'w')
1644 h.write(str)
1645 h.close
1646
1647 def check_hp_ok(name):
1648
1649 # do not qualify for hp2ps because we should be in the right directory
1650 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1651
1652 hp2psResult = runCmdExitCode(hp2psCmd)
1653
1654 actual_ps_file = qualify(name, 'ps')
1655
1656 if(hp2psResult == 0):
1657 if (os.path.exists(actual_ps_file)):
1658 if gs_working:
1659 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1660 if (gsResult == 0):
1661 return (True)
1662 else:
1663 print "hp2ps output for " + name + "is not valid PostScript"
1664 else: return (True) # assume postscript is valid without ghostscript
1665 else:
1666 print "hp2ps did not generate PostScript for " + name
1667 return (False)
1668 else:
1669 print "hp2ps error when processing heap profile for " + name
1670 return(False)
1671
1672 def check_prof_ok(name):
1673
1674 prof_file = qualify(name,'prof')
1675
1676 if not os.path.exists(prof_file):
1677 print prof_file + " does not exist"
1678 return(False)
1679
1680 if os.path.getsize(qualify(name,'prof')) == 0:
1681 print prof_file + " is empty"
1682 return(False)
1683
1684 if getTestOpts().with_namebase == None:
1685 namebase = name
1686 else:
1687 namebase = getTestOpts().with_namebase
1688
1689 (platform_specific, expected_prof_file) = \
1690 platform_wordsize_qualify(namebase, 'prof.sample')
1691
1692 # sample prof file is not required
1693 if not os.path.exists(expected_prof_file):
1694 return True
1695 else:
1696 return compare_outputs('prof', \
1697 two_normalisers(normalise_whitespace,normalise_prof), \
1698 expected_prof_file, prof_file)
1699
1700 # Compare expected output to actual output, and optionally accept the
1701 # new output. Returns true if output matched or was accepted, false
1702 # otherwise.
1703 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1704 if os.path.exists(expected_file):
1705 expected_raw = read_no_crs(expected_file)
1706 # print "norm:", normaliser(expected_raw)
1707 expected_str = normaliser(expected_raw)
1708 expected_file_for_diff = expected_file
1709 else:
1710 expected_str = ''
1711 expected_file_for_diff = '/dev/null'
1712
1713 actual_raw = read_no_crs(actual_file)
1714 actual_str = normaliser(actual_raw)
1715
1716 if expected_str == actual_str:
1717 return 1
1718 else:
1719 print 'Actual ' + kind + ' output differs from expected:'
1720
1721 if expected_file_for_diff == '/dev/null':
1722 expected_normalised_file = '/dev/null'
1723 else:
1724 expected_normalised_file = expected_file + ".normalised"
1725 write_file(expected_normalised_file, expected_str)
1726
1727 actual_normalised_file = actual_file + ".normalised"
1728 write_file(actual_normalised_file, actual_str)
1729
1730 # Ignore whitespace when diffing. We should only get to this
1731 # point if there are non-whitespace differences
1732 #
1733 # Note we are diffing the *actual* output, not the normalised
1734 # output. The normalised output may have whitespace squashed
1735 # (including newlines) so the diff would be hard to read.
1736 # This does mean that the diff might contain changes that
1737 # would be normalised away.
1738 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1739 ' ' + actual_file )
1740
1741 # If for some reason there were no non-whitespace differences,
1742 # then do a full diff
1743 if r == 0:
1744 r = os.system( 'diff -u ' + expected_file_for_diff + \
1745 ' ' + actual_file )
1746
1747 if config.accept:
1748 print 'Accepting new output.'
1749 write_file(expected_file, actual_raw)
1750 return 1
1751 else:
1752 return 0
1753
1754
1755 def normalise_whitespace( str ):
1756 # Merge contiguous whitespace characters into a single space.
1757 str = re.sub('[ \t\n]+', ' ', str)
1758 return str
1759
1760 def normalise_errmsg( str ):
1761 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1762 # the colon is there because it appears in error messages; this
1763 # hacky solution is used in place of more sophisticated filename
1764 # mangling
1765 str = re.sub('([^\\s])\\.exe', '\\1', str)
1766 # normalise slashes, minimise Windows/Unix filename differences
1767 str = re.sub('\\\\', '/', str)
1768 # The inplace ghc's are called ghc-stage[123] to avoid filename
1769 # collisions, so we need to normalise that to just "ghc"
1770 str = re.sub('ghc-stage[123]', 'ghc', str)
1771 # We sometimes see the name of the integer-gmp package on stderr,
1772 # but this can change (either the implementation name or the
1773 # version number), so we canonicalise it here
1774 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1775 return str
1776
1777 # normalise a .prof file, so that we can reasonably compare it against
1778 # a sample. This doesn't compare any of the actual profiling data,
1779 # only the shape of the profile and the number of entries.
1780 def normalise_prof (str):
1781 # strip everything up to the line beginning "COST CENTRE"
1782 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1783
1784 # strip results for CAFs, these tend to change unpredictably
1785 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1786
1787 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1788 # sometimes under MAIN.
1789 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1790
1791 # We have somthing like this:
1792
1793 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1794 # k Main 204 1 0.0 0.0 0.0 0.0
1795 # foo Main 205 1 0.0 0.0 0.0 0.0
1796 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1797
1798 # then we remove all the specific profiling data, leaving only the
1799 # cost centre name, module, and entries, to end up with this:
1800
1801 # MAIN MAIN 0
1802 # k Main 1
1803 # foo Main 1
1804 # foo.bar Main 1
1805
1806 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1807 return str
1808
1809 def normalise_slashes_( str ):
1810 str = re.sub('\\\\', '/', str)
1811 return str
1812
1813 def normalise_exe_( str ):
1814 str = re.sub('\.exe', '', str)
1815 return str
1816
1817 def normalise_output( str ):
1818 # Remove a .exe extension (for Windows)
1819 # This can occur in error messages generated by the program.
1820 str = re.sub('([^\\s])\\.exe', '\\1', str)
1821 return str
1822
1823 def normalise_asm( str ):
1824 lines = str.split('\n')
1825 # Only keep instructions and labels not starting with a dot.
1826 metadata = re.compile('^[ \t]*\\..*$')
1827 out = []
1828 for line in lines:
1829 # Drop metadata directives (e.g. ".type")
1830 if not metadata.match(line):
1831 line = re.sub('@plt', '', line)
1832 instr = line.lstrip().split()
1833 # Drop empty lines.
1834 if not instr:
1835 continue
1836 # Drop operands, except for call instructions.
1837 elif instr[0] == 'call':
1838 out.append(instr[0] + ' ' + instr[1])
1839 else:
1840 out.append(instr[0])
1841 out = '\n'.join(out)
1842 return out
1843
1844 def if_verbose( n, str ):
1845 if config.verbose >= n:
1846 print str
1847
1848 def if_verbose_dump( n, f ):
1849 if config.verbose >= n:
1850 try:
1851 print open(f).read()
1852 except:
1853 print ''
1854
1855 def rawSystem(cmd_and_args):
1856 # We prefer subprocess.call to os.spawnv as the latter
1857 # seems to send its arguments through a shell or something
1858 # with the Windows (non-cygwin) python. An argument "a b c"
1859 # turns into three arguments ["a", "b", "c"].
1860
1861 # However, subprocess is new in python 2.4, so fall back to
1862 # using spawnv if we don't have it
1863
1864 if have_subprocess:
1865 return subprocess.call(cmd_and_args)
1866 else:
1867 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1868
1869 # Note that this doesn't handle the timeout itself; it is just used for
1870 # commands that have timeout handling built-in.
1871 def rawSystemWithTimeout(cmd_and_args):
1872 r = rawSystem(cmd_and_args)
1873 if r == 98:
1874 # The python timeout program uses 98 to signal that ^C was pressed
1875 stopNow()
1876 return r
1877
1878 # cmd is a complex command in Bourne-shell syntax
1879 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1880 # Hence it must ultimately be run by a Bourne shell
1881 #
1882 # Mostly it invokes the command wrapped in 'timeout' thus
1883 # timeout 300 'cd . && ...blah blah'
1884 # so it's timeout's job to invoke the Bourne shell
1885 #
1886 # But watch out for the case when there is no timeout program!
1887 # Then, when using the native Python, os.system will invoke the cmd shell
1888
1889 def runCmd( cmd ):
1890 if_verbose( 1, cmd )
1891 r = 0
1892 if config.os == 'mingw32':
1893 # On MinGW, we will always have timeout
1894 assert config.timeout_prog!=''
1895
1896 if config.timeout_prog != '':
1897 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1898 else:
1899 r = os.system(cmd)
1900 return r << 8
1901
1902 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1903 if_verbose( 1, cmd )
1904 r = 0
1905 if config.os == 'mingw32':
1906 # On MinGW, we will always have timeout
1907 assert config.timeout_prog!=''
1908 timeout = int(ceil(config.timeout * timeout_multiplier))
1909
1910 if config.timeout_prog != '':
1911 if config.check_files_written:
1912 fn = name + ".strace"
1913 r = rawSystemWithTimeout(
1914 ["strace", "-o", fn, "-fF",
1915 "-e", "creat,open,chdir,clone,vfork",
1916 config.timeout_prog, str(timeout), cmd])
1917 addTestFilesWritten(name, fn)
1918 rm_no_fail(fn)
1919 else:
1920 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1921 else:
1922 r = os.system(cmd)
1923 return r << 8
1924
1925 def runCmdExitCode( cmd ):
1926 return (runCmd(cmd) >> 8);
1927
1928
1929 # -----------------------------------------------------------------------------
1930 # checking for files being written to by multiple tests
1931
1932 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1933 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1934 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1935 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1936 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1937 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1938 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1939 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1940 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1941 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1942 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1943 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1944 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1945 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1946 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1947 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1948 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1949
1950 # Files that are read or written but shouldn't be:
1951 # * ghci_history shouldn't be read or written by tests
1952 # * things under package.conf.d shouldn't be written by tests
1953 bad_file_usages = {}
1954
1955 # Mapping from tests to the list of files that they write
1956 files_written = {}
1957
1958 # Mapping from tests to the list of files that they write but don't clean
1959 files_written_not_removed = {}
1960
1961 def add_bad_file_usage(name, file):
1962 try:
1963 if not file in bad_file_usages[name]:
1964 bad_file_usages[name].append(file)
1965 except:
1966 bad_file_usages[name] = [file]
1967
1968 def mkPath(curdir, path):
1969 # Given the current full directory is 'curdir', what is the full
1970 # path to 'path'?
1971 return os.path.realpath(os.path.join(curdir, path))
1972
1973 def addTestFilesWritten(name, fn):
1974 if config.use_threads:
1975 with t.lockFilesWritten:
1976 addTestFilesWrittenHelper(name, fn)
1977 else:
1978 addTestFilesWrittenHelper(name, fn)
1979
1980 def addTestFilesWrittenHelper(name, fn):
1981 started = False
1982 working_directories = {}
1983
1984 with open(fn, 'r') as f:
1985 for line in f:
1986 m_pid = re_strace_pid.match(line)
1987 if m_pid:
1988 pid = m_pid.group(1)
1989 content = m_pid.group(2)
1990 elif re_strace_unavailable.match(line):
1991 next
1992 else:
1993 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1994
1995 m_open = re_strace_open.match(content)
1996 m_chdir = re_strace_chdir.match(content)
1997 m_clone = re_strace_clone.match(content)
1998 m_vfork = re_strace_vfork.match(content)
1999
2000 if not started:
2001 working_directories[pid] = os.getcwd()
2002 started = True
2003
2004 if m_open:
2005 file = m_open.group(1)
2006 file = mkPath(working_directories[pid], file)
2007 if file.endswith("ghci_history"):
2008 add_bad_file_usage(name, file)
2009 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
2010 flags = m_open.group(2).split('|')
2011 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
2012 if re.match('package\.conf\.d', file):
2013 add_bad_file_usage(name, file)
2014 else:
2015 try:
2016 if not file in files_written[name]:
2017 files_written[name].append(file)
2018 except:
2019 files_written[name] = [file]
2020 elif 'O_RDONLY' in flags:
2021 pass
2022 else:
2023 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2024 elif m_chdir:
2025 # We optimistically assume that unfinished chdir's are going to succeed
2026 dir = m_chdir.group(1)
2027 working_directories[pid] = mkPath(working_directories[pid], dir)
2028 elif m_clone:
2029 working_directories[m_clone.group(2)] = working_directories[pid]
2030 elif m_vfork:
2031 working_directories[m_vfork.group(2)] = working_directories[pid]
2032 elif re_strace_open_resumed.match(content):
2033 pass
2034 elif re_strace_chdir_resumed.match(content):
2035 pass
2036 elif re_strace_vfork_unfinished.match(content):
2037 pass
2038 elif re_strace_clone_unfinished.match(content):
2039 pass
2040 elif re_strace_ignore_sigchild.match(content):
2041 pass
2042 elif re_strace_ignore_sigvtalarm.match(content):
2043 pass
2044 elif re_strace_ignore_sigint.match(content):
2045 pass
2046 elif re_strace_ignore_sigfpe.match(content):
2047 pass
2048 elif re_strace_ignore_sigsegv.match(content):
2049 pass
2050 elif re_strace_ignore_sigpipe.match(content):
2051 pass
2052 else:
2053 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2054
2055 def checkForFilesWrittenProblems(file):
2056 foundProblem = False
2057
2058 files_written_inverted = {}
2059 for t in files_written.keys():
2060 for f in files_written[t]:
2061 try:
2062 files_written_inverted[f].append(t)
2063 except:
2064 files_written_inverted[f] = [t]
2065
2066 for f in files_written_inverted.keys():
2067 if len(files_written_inverted[f]) > 1:
2068 if not foundProblem:
2069 foundProblem = True
2070 file.write("\n")
2071 file.write("\nSome files are written by multiple tests:\n")
2072 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2073 if foundProblem:
2074 file.write("\n")
2075
2076 # -----
2077
2078 if len(files_written_not_removed) > 0:
2079 file.write("\n")
2080 file.write("\nSome files written but not removed:\n")
2081 tests = files_written_not_removed.keys()
2082 tests.sort()
2083 for t in tests:
2084 for f in files_written_not_removed[t]:
2085 file.write(" " + t + ": " + f + "\n")
2086 file.write("\n")
2087
2088 # -----
2089
2090 if len(bad_file_usages) > 0:
2091 file.write("\n")
2092 file.write("\nSome bad file usages:\n")
2093 tests = bad_file_usages.keys()
2094 tests.sort()
2095 for t in tests:
2096 for f in bad_file_usages[t]:
2097 file.write(" " + t + ": " + f + "\n")
2098 file.write("\n")
2099
2100 # -----------------------------------------------------------------------------
2101 # checking if ghostscript is available for checking the output of hp2ps
2102
2103 def genGSCmd(psfile):
2104 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2105
2106 def gsNotWorking():
2107 global gs_working
2108 print "GhostScript not available for hp2ps tests"
2109
2110 global gs_working
2111 gs_working = 0
2112 if config.have_profiling:
2113 if config.gs != '':
2114 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2115 if resultGood == 0:
2116 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2117 if resultBad != 0:
2118 print "GhostScript available for hp2ps tests"
2119 gs_working = 1;
2120 else:
2121 gsNotWorking();
2122 else:
2123 gsNotWorking();
2124 else:
2125 gsNotWorking();
2126
2127 def rm_no_fail( file ):
2128 try:
2129 os.remove( file )
2130 finally:
2131 return
2132
2133 def add_suffix( name, suffix ):
2134 if suffix == '':
2135 return name
2136 else:
2137 return name + '.' + suffix
2138
2139 def add_hs_lhs_suffix(name):
2140 if getTestOpts().c_src:
2141 return add_suffix(name, 'c')
2142 elif getTestOpts().cmm_src:
2143 return add_suffix(name, 'cmm')
2144 elif getTestOpts().objc_src:
2145 return add_suffix(name, 'm')
2146 elif getTestOpts().objcpp_src:
2147 return add_suffix(name, 'mm')
2148 elif getTestOpts().literate:
2149 return add_suffix(name, 'lhs')
2150 else:
2151 return add_suffix(name, 'hs')
2152
2153 def replace_suffix( name, suffix ):
2154 base, suf = os.path.splitext(name)
2155 return base + '.' + suffix
2156
2157 def in_testdir( name ):
2158 return (getTestOpts().testdir + '/' + name)
2159
2160 def qualify( name, suff ):
2161 return in_testdir(add_suffix(name, suff))
2162
2163
2164 # Finding the sample output. The filename is of the form
2165 #
2166 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2167 #
2168 # and we pick the most specific version available. The <version> is
2169 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2170 # more fine-grained control use if_compiler_lt().
2171 #
2172 def platform_wordsize_qualify( name, suff ):
2173
2174 basepath = qualify(name, suff)
2175
2176 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2177 for (platformSpecific, plat) in [(1, '-' + config.platform),
2178 (1, '-' + config.os),
2179 (0, '')]
2180 for ws in ['-ws-' + config.wordsize, '']
2181 for comp in ['-' + config.compiler_type, '']
2182 for vers in ['-' + config.compiler_maj_version, '']]
2183
2184 dir = glob.glob(basepath + '*')
2185 dir = map (lambda d: normalise_slashes_(d), dir)
2186
2187 for (platformSpecific, f) in paths:
2188 if f in dir:
2189 return (platformSpecific,f)
2190
2191 return (0, basepath)
2192
2193 # Clean up prior to the test, so that we can't spuriously conclude
2194 # that it passed on the basis of old run outputs.
2195 def pretest_cleanup(name):
2196 if getTestOpts().outputdir != None:
2197 odir = in_testdir(getTestOpts().outputdir)
2198 try:
2199 shutil.rmtree(odir)
2200 except:
2201 pass
2202 os.mkdir(odir)
2203
2204 rm_no_fail(qualify(name,'comp.stderr'))
2205 rm_no_fail(qualify(name,'run.stderr'))
2206 rm_no_fail(qualify(name,'run.stdout'))
2207 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2208 # simple_build zaps the following:
2209 # rm_nofail(qualify("o"))
2210 # rm_nofail(qualify(""))
2211 # not interested in the return code
2212
2213 # -----------------------------------------------------------------------------
2214 # Return a list of all the files ending in '.T' below the directory dir.
2215
2216 def findTFiles(roots):
2217 return concat(map(findTFiles_,roots))
2218
2219 def findTFiles_(path):
2220 if os.path.isdir(path):
2221 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2222 return findTFiles(paths)
2223 elif path[-2:] == '.T':
2224 return [path]
2225 else:
2226 return []
2227
2228 # -----------------------------------------------------------------------------
2229 # Output a test summary to the specified file object
2230
2231 def summary(t, file):
2232
2233 file.write('\n')
2234 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2235 file.write('OVERALL SUMMARY for test run started at ' \
2236 + t.start_time + '\n'\
2237 + string.rjust(`t.total_tests`, 8) \
2238 + ' total tests, which gave rise to\n' \
2239 + string.rjust(`t.total_test_cases`, 8) \
2240 + ' test cases, of which\n' \
2241 + string.rjust(`t.n_framework_failures`, 8) \
2242 + ' caused framework failures\n' \
2243 + string.rjust(`t.n_tests_skipped`, 8)
2244 + ' were skipped\n\n' \
2245 + string.rjust(`t.n_expected_passes`, 8)
2246 + ' expected passes\n' \
2247 + string.rjust(`t.n_missing_libs`, 8)
2248 + ' had missing libraries\n' \
2249 + string.rjust(`t.n_expected_failures`, 8) \
2250 + ' expected failures\n' \
2251 + string.rjust(`t.n_unexpected_passes`, 8) \
2252 + ' unexpected passes\n'
2253 + string.rjust(`t.n_unexpected_failures`, 8) \
2254 + ' unexpected failures\n'
2255 + '\n')
2256
2257 if t.n_unexpected_passes > 0:
2258 file.write('Unexpected passes:\n')
2259 printPassingTestInfosSummary(file, t.unexpected_passes)
2260
2261 if t.n_unexpected_failures > 0:
2262 file.write('Unexpected failures:\n')
2263 printFailingTestInfosSummary(file, t.unexpected_failures)
2264
2265 if config.check_files_written:
2266 checkForFilesWrittenProblems(file)
2267
2268 if stopping():
2269 file.write('WARNING: Testsuite run was terminated early\n')
2270
2271 def printUnexpectedTests(file, testInfoss):
2272 unexpected = []
2273 for testInfos in testInfoss:
2274 directories = testInfos.keys()
2275 for directory in directories:
2276 tests = testInfos[directory].keys()
2277 unexpected += tests
2278 if unexpected != []:
2279 file.write('Unexpected results from:\n')
2280 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2281 file.write('\n')
2282
2283 def printPassingTestInfosSummary(file, testInfos):
2284 directories = testInfos.keys()
2285 directories.sort()
2286 maxDirLen = max(map ((lambda x : len(x)), directories))
2287 for directory in directories:
2288 tests = testInfos[directory].keys()
2289 tests.sort()
2290 for test in tests:
2291 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2292 ' (' + join(testInfos[directory][test],',') + ')\n')
2293 file.write('\n')
2294
2295 def printFailingTestInfosSummary(file, testInfos):
2296 directories = testInfos.keys()
2297 directories.sort()
2298 maxDirLen = max(map ((lambda x : len(x)), directories))
2299 for directory in directories:
2300 tests = testInfos[directory].keys()
2301 tests.sort()
2302 for test in tests:
2303 reasons = testInfos[directory][test].keys()
2304 for reason in reasons:
2305 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2306 ' [' + reason + ']' + \
2307 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2308 file.write('\n')
2309
2310 def getStdout(cmd):
2311 if have_subprocess:
2312 p = subprocess.Popen(cmd,
2313 stdout=subprocess.PIPE,
2314 stderr=subprocess.PIPE)
2315 (stdout, stderr) = p.communicate()
2316 r = p.wait()
2317 if r != 0:
2318 raise Exception("Command failed: " + str(cmd))
2319 if stderr != '':
2320 raise Exception("stderr from command: " + str(cmd))
2321 return stdout
2322 else:
2323 raise Exception("Need subprocess to get stdout, but don't have it")