Add a different sort of stats_num_field helper function
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 from math import ceil, trunc
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for the current test only (these get reset to
44 # testdir_testopts after each test).
45
46 global testopts_local
47 if config.use_threads:
48 testopts_local = threading.local()
49 else:
50 class TestOpts_Local:
51 pass
52 testopts_local = TestOpts_Local()
53
54 def getTestOpts():
55 return testopts_local.x
56
57 def setLocalTestOpts(opts):
58 global testopts_local
59 testopts_local.x=opts
60
61 # This can be called at the top of a file of tests, to set default test options
62 # for the following tests.
63 def setTestOpts( f ):
64 global thisdir_settings
65 thisdir_settings = compose(thisdir_settings, f)
66
67 # -----------------------------------------------------------------------------
68 # Canned setup functions for common cases. eg. for a test you might say
69 #
70 # test('test001', normal, compile, [''])
71 #
72 # to run it without any options, but change it to
73 #
74 # test('test001', expect_fail, compile, [''])
75 #
76 # to expect failure for this test.
77
78 def normal( name, opts ):
79 return;
80
81 def skip( name, opts ):
82 opts.skip = 1
83
84 def expect_fail( name, opts ):
85 opts.expect = 'fail';
86
87 def reqlib( lib ):
88 return lambda name, opts, l=lib: _reqlib (name, opts, l )
89
90 # Cache the results of looking to see if we have a library or not.
91 # This makes quite a difference, especially on Windows.
92 have_lib = {}
93
94 def _reqlib( name, opts, lib ):
95 if have_lib.has_key(lib):
96 got_it = have_lib[lib]
97 else:
98 if have_subprocess:
99 # By preference we use subprocess, as the alternative uses
100 # /dev/null which mingw doesn't have.
101 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 stdout=subprocess.PIPE,
103 stderr=subprocess.PIPE)
104 # read from stdout and stderr to avoid blocking due to
105 # buffers filling
106 p.communicate()
107 r = p.wait()
108 else:
109 r = os.system(config.ghc_pkg + ' describe ' + lib
110 + ' > /dev/null 2> /dev/null')
111 got_it = r == 0
112 have_lib[lib] = got_it
113
114 if not got_it:
115 opts.expect = 'missing-lib'
116
117 def req_profiling( name, opts ):
118 if not config.have_profiling:
119 opts.expect = 'fail'
120
121 def req_shared_libs( name, opts ):
122 if not config.have_shared_libs:
123 opts.expect = 'fail'
124
125 def req_interp( name, opts ):
126 if not config.have_interp:
127 opts.expect = 'fail'
128
129 def req_smp( name, opts ):
130 if not config.have_smp:
131 opts.expect = 'fail'
132
133 def expect_broken( bug ):
134 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
135
136 def _expect_broken( name, opts, bug ):
137 opts.expect = 'fail';
138
139 def ignore_output( name, opts ):
140 opts.ignore_output = 1
141
142 def no_stdin( name, opts ):
143 opts.no_stdin = 1
144
145 def combined_output( name, opts ):
146 opts.combined_output = True
147
148 # -----
149
150 def expect_fail_for( ways ):
151 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
152
153 def _expect_fail_for( name, opts, ways ):
154 opts.expect_fail_for = ways
155
156 def expect_broken_for( bug, ways ):
157 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
158
159 def _expect_broken_for( name, opts, bug, ways ):
160 opts.expect_fail_for = ways
161
162 # -----
163
164 def omit_ways( ways ):
165 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
166
167 def _omit_ways( name, opts, ways ):
168 opts.omit_ways = ways
169
170 # -----
171
172 def only_ways( ways ):
173 return lambda name, opts, w=ways: _only_ways( name, opts, w )
174
175 def _only_ways( name, opts, ways ):
176 opts.only_ways = ways
177
178 # -----
179
180 def extra_ways( ways ):
181 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
182
183 def _extra_ways( name, opts, ways ):
184 opts.extra_ways = ways
185
186 # -----
187
188 def omit_compiler_types( compiler_types ):
189 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
190
191 def _omit_compiler_types( name, opts, compiler_types ):
192 if config.compiler_type in compiler_types:
193 opts.skip = 1
194
195 # -----
196
197 def only_compiler_types( compiler_types ):
198 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
199
200 def _only_compiler_types( name, opts, compiler_types ):
201 if config.compiler_type not in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def set_stdin( file ):
207 return lambda name, opts, f=file: _set_stdin(name, opts, f);
208
209 def _set_stdin( name, opts, f ):
210 opts.stdin = f
211
212 # -----
213
214 def exit_code( val ):
215 return lambda name, opts, v=val: _exit_code(name, opts, v);
216
217 def _exit_code( name, opts, v ):
218 opts.exit_code = v
219
220 # -----
221
222 def timeout_multiplier( val ):
223 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
224
225 def _timeout_multiplier( name, opts, v ):
226 opts.timeout_multiplier = v
227
228 # -----
229
230 def extra_run_opts( val ):
231 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
232
233 def _extra_run_opts( name, opts, v ):
234 opts.extra_run_opts = v
235
236 # -----
237
238 def extra_hc_opts( val ):
239 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
240
241 def _extra_hc_opts( name, opts, v ):
242 opts.extra_hc_opts = v
243
244 # -----
245
246 def extra_clean( files ):
247 return lambda name, opts, v=files: _extra_clean(name, opts, v);
248
249 def _extra_clean( name, opts, v ):
250 opts.clean_files = v
251
252 # -----
253
254 def stats_num_field( field, expecteds ):
255 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
256
257 def _stats_num_field( name, opts, field, expecteds ):
258 if field in opts.stats_range_fields:
259 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
260
261 for (b, expected, dev) in expecteds:
262 if b:
263 opts.stats_range_fields[field] = (expected, dev)
264 return
265
266 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
267
268 def stats_range_field( field, expected, dev ):
269 return stats_num_field( field, [(True, expected, dev)] )
270
271 def compiler_stats_num_field( field, expecteds ):
272 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
273
274 def _compiler_stats_num_field( name, opts, field, expecteds ):
275 if field in opts.compiler_stats_range_fields:
276 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
277
278 for (b, expected, dev) in expecteds:
279 if b:
280 opts.compiler_stats_range_fields[field] = (expected, dev)
281 return
282
283 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
284
285 def compiler_stats_range_field( field, expected, dev ):
286 return compiler_stats_num_field( field, [(True, expected, dev)] )
287
288 # -----
289
290 def skip_if_no_ghci(name, opts):
291 if not ('ghci' in config.run_ways):
292 opts.skip = 1
293
294 # ----
295
296 def skip_if_fast(name, opts):
297 if config.fast:
298 opts.skip = 1
299
300 # -----
301
302 def when(b, f):
303 if b:
304 return f
305 else:
306 return normal
307
308 def unless(b, f):
309 return when(not b, f)
310
311 def if_platform( plat, f ):
312 if config.platform == plat:
313 return f
314 else:
315 return normal
316
317 def unless_platform( plat, f ):
318 if config.platform != plat:
319 return f
320 else:
321 return normal
322
323 def if_os( os, f ):
324 if config.os == os:
325 return f
326 else:
327 return normal
328
329 def unless_os( os, f ):
330 if config.os == os:
331 return normal
332 else:
333 return f
334
335 def if_arch( arch, f ):
336 if config.arch == arch:
337 return f
338 else:
339 return normal
340
341 def unless_arch( arch, f ):
342 if config.arch == arch:
343 return normal
344 else:
345 return f
346
347 def wordsize( ws ):
348 return config.wordsize == str(ws)
349
350 def if_unregisterised( f ):
351 if config.unregisterised:
352 return f
353 else:
354 return normal
355
356 def unless_unregisterised( f ):
357 if config.unregisterised:
358 return normal
359 else:
360 return f
361
362 def if_msys( f ):
363 if config.msys:
364 return f
365 else:
366 return normal
367
368 def if_cygwin( f ):
369 if config.cygwin:
370 return f
371 else:
372 return normal
373
374 def when_have_vanilla( f ):
375 if config.have_vanilla:
376 return f
377 else:
378 return normal
379
380 def unless_have_vanilla( f ):
381 if config.have_vanilla:
382 return normal
383 else:
384 return f
385
386 def when_have_dynamic( f ):
387 if config.have_dynamic:
388 return f
389 else:
390 return normal
391
392 def unless_have_dynamic( f ):
393 if config.have_dynamic:
394 return normal
395 else:
396 return f
397
398 def when_have_profiling( f ):
399 if config.have_profiling:
400 return f
401 else:
402 return normal
403
404 def unless_have_profiling( f ):
405 if config.have_profiling:
406 return normal
407 else:
408 return f
409
410 # ---
411
412 def if_ghci_dynamic( f ):
413 if config.ghc_dynamic_by_default:
414 return f
415 else:
416 return normal
417
418 def if_in_tree_compiler( f ):
419 if config.in_tree_compiler:
420 return f
421 else:
422 return normal
423
424 def unless_in_tree_compiler( f ):
425 if config.in_tree_compiler:
426 return normal
427 else:
428 return f
429
430 def if_compiler_type( compiler, f ):
431 if config.compiler_type == compiler:
432 return f
433 else:
434 return normal
435
436 def if_compiler_profiled( f ):
437 if config.compiler_profiled:
438 return f
439 else:
440 return normal
441
442 def unless_compiler_profiled( f ):
443 if config.compiler_profiled:
444 return normal
445 else:
446 return f
447
448 def if_compiler_lt( compiler, version, f ):
449 if config.compiler_type == compiler and \
450 version_lt(config.compiler_version, version):
451 return f
452 else:
453 return normal
454
455 def if_compiler_le( compiler, version, f ):
456 if config.compiler_type == compiler and \
457 version_le(config.compiler_version, version):
458 return f
459 else:
460 return normal
461
462 def if_compiler_gt( compiler, version, f ):
463 if config.compiler_type == compiler and \
464 version_gt(config.compiler_version, version):
465 return f
466 else:
467 return normal
468
469 def if_compiler_ge( compiler, version, f ):
470 if config.compiler_type == compiler and \
471 version_ge(config.compiler_version, version):
472 return f
473 else:
474 return normal
475
476 def if_compiler_debugged( f ):
477 if config.compiler_debugged:
478 return f
479 else:
480 return normal
481
482 def namebase( nb ):
483 return lambda opts, nb=nb: _namebase(opts, nb)
484
485 def _namebase( opts, nb ):
486 opts.with_namebase = nb
487
488 # ---
489
490 def if_tag( tag, f ):
491 if tag in config.compiler_tags:
492 return f
493 else:
494 return normal
495
496 def unless_tag( tag, f ):
497 if not (tag in config.compiler_tags):
498 return f
499 else:
500 return normal
501
502 # ---
503 def high_memory_usage(name, opts):
504 opts.alone = True
505
506 # ---
507 def literate( name, opts ):
508 opts.literate = 1;
509
510 def c_src( name, opts ):
511 opts.c_src = 1;
512
513 def objc_src( name, opts ):
514 opts.objc_src = 1;
515
516 def objcpp_src( name, opts ):
517 opts.objcpp_src = 1;
518
519 def cmm_src( name, opts ):
520 opts.cmm_src = 1;
521
522 def outputdir( odir ):
523 return lambda name, opts, d=odir: _outputdir(name, opts, d)
524
525 def _outputdir( name, opts, odir ):
526 opts.outputdir = odir;
527
528 # ----
529
530 def pre_cmd( cmd ):
531 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
532
533 def _pre_cmd( name, opts, cmd ):
534 opts.pre_cmd = cmd
535
536 # ----
537
538 def clean_cmd( cmd ):
539 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
540
541 def _clean_cmd( name, opts, cmd ):
542 opts.clean_cmd = cmd
543
544 # ----
545
546 def cmd_prefix( prefix ):
547 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
548
549 def _cmd_prefix( name, opts, prefix ):
550 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
551
552 # ----
553
554 def cmd_wrapper( fun ):
555 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
556
557 def _cmd_wrapper( name, opts, fun ):
558 opts.cmd_wrapper = fun
559
560 # ----
561
562 def compile_cmd_prefix( prefix ):
563 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
564
565 def _compile_cmd_prefix( name, opts, prefix ):
566 opts.compile_cmd_prefix = prefix
567
568 # ----
569
570 def normalise_slashes( name, opts ):
571 opts.extra_normaliser = normalise_slashes_
572
573 def normalise_exe( name, opts ):
574 opts.extra_normaliser = normalise_exe_
575
576 def normalise_fun( fun ):
577 return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
578
579 def _normalise_fun( name, opts, f ):
580 opts.extra_normaliser = f
581
582 def normalise_errmsg_fun( fun ):
583 return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
584
585 def _normalise_errmsg_fun( name, opts, f ):
586 opts.extra_errmsg_normaliser = f
587
588 def two_normalisers(f, g):
589 return lambda x, f=f, g=g: f(g(x))
590
591 # ----
592 # Function for composing two opt-fns together
593
594 def composes( fs ):
595 return reduce(lambda f, g: compose(f, g), fs)
596
597 def compose( f, g ):
598 return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
599
600 def _compose( name, opts, f, g ):
601 f(name, opts)
602 g(name, opts)
603
604 # -----------------------------------------------------------------------------
605 # The current directory of tests
606
607 def newTestDir( dir ):
608 global thisdir_settings
609 # reset the options for this test directory
610 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
611
612 def _newTestDir( name, opts, dir ):
613 opts.testdir = dir
614 opts.compiler_always_flags = config.compiler_always_flags
615
616 # -----------------------------------------------------------------------------
617 # Actually doing tests
618
619 parallelTests = []
620 aloneTests = []
621 allTestNames = set([])
622
623 def runTest (opts, name, func, args):
624 ok = 0
625
626 if config.use_threads:
627 t.thread_pool.acquire()
628 try:
629 while config.threads<(t.running_threads+1):
630 t.thread_pool.wait()
631 t.running_threads = t.running_threads+1
632 ok=1
633 t.thread_pool.release()
634 thread.start_new_thread(test_common_thread, (name, opts, func, args))
635 except:
636 if not ok:
637 t.thread_pool.release()
638 else:
639 test_common_work (name, opts, func, args)
640
641 # name :: String
642 # setup :: TestOpts -> IO ()
643 def test (name, setup, func, args):
644 global aloneTests
645 global parallelTests
646 global allTestNames
647 global thisdir_settings
648 if name in allTestNames:
649 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
650 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
651 framework_fail(name, 'bad_name', 'This test has an invalid name')
652
653 # Make a deep copy of the default_testopts, as we need our own copy
654 # of any dictionaries etc inside it. Otherwise, if one test modifies
655 # them, all tests will see the modified version!
656 myTestOpts = copy.deepcopy(default_testopts)
657
658 if type(setup) is types.ListType:
659 setup = composes(setup)
660
661 setup = compose(thisdir_settings, setup)
662 setup(name, myTestOpts)
663
664 thisTest = lambda : runTest(myTestOpts, name, func, args)
665 if myTestOpts.alone:
666 aloneTests.append(thisTest)
667 else:
668 parallelTests.append(thisTest)
669 allTestNames.add(name)
670
671 if config.use_threads:
672 def test_common_thread(name, opts, func, args):
673 t.lock.acquire()
674 try:
675 test_common_work(name,opts,func,args)
676 finally:
677 t.lock.release()
678 t.thread_pool.acquire()
679 t.running_threads = t.running_threads - 1
680 t.thread_pool.notify()
681 t.thread_pool.release()
682
683 def get_package_cache_timestamp():
684 if config.package_conf_cache_file == '':
685 return 0.0
686 else:
687 try:
688 return os.stat(config.package_conf_cache_file).st_mtime
689 except:
690 return 0.0
691
692
693 def test_common_work (name, opts, func, args):
694 try:
695 t.total_tests = t.total_tests+1
696 setLocalTestOpts(opts)
697
698 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
699
700 # All the ways we might run this test
701 if func == compile or func == multimod_compile:
702 all_ways = config.compile_ways
703 elif func == compile_and_run or func == multimod_compile_and_run:
704 all_ways = config.run_ways
705 elif func == ghci_script:
706 if 'ghci' in config.run_ways:
707 all_ways = ['ghci']
708 else:
709 all_ways = []
710 else:
711 all_ways = ['normal']
712
713 # A test itself can request extra ways by setting opts.extra_ways
714 all_ways = all_ways + filter(lambda way: way not in all_ways,
715 opts.extra_ways)
716
717 t.total_test_cases = t.total_test_cases + len(all_ways)
718
719 ok_way = lambda way: \
720 not getTestOpts().skip \
721 and (config.only == [] or name in config.only) \
722 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
723 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
724 and way not in getTestOpts().omit_ways
725
726 # Which ways we are asked to skip
727 do_ways = filter (ok_way,all_ways)
728
729 # In fast mode, we skip all but one way
730 if config.fast and len(do_ways) > 0:
731 do_ways = [do_ways[0]]
732
733 if not config.clean_only:
734 # Run the required tests...
735 for way in do_ways:
736 if stopping():
737 break
738 do_test (name, way, func, args)
739
740 for way in all_ways:
741 if way not in do_ways:
742 skiptest (name,way)
743
744 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
745 clean(map (lambda suff: name + suff,
746 ['', '.exe', '.exe.manifest', '.genscript',
747 '.stderr.normalised', '.stdout.normalised',
748 '.run.stderr', '.run.stdout',
749 '.run.stderr.normalised', '.run.stdout.normalised',
750 '.comp.stderr', '.comp.stdout',
751 '.comp.stderr.normalised', '.comp.stdout.normalised',
752 '.interp.stderr', '.interp.stdout',
753 '.interp.stderr.normalised', '.interp.stdout.normalised',
754 '.stats', '.comp.stats',
755 '.hi', '.o', '.prof', '.exe.prof', '.hc',
756 '_stub.h', '_stub.c', '_stub.o',
757 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
758
759 if func == multi_compile or func == multi_compile_fail:
760 extra_mods = args[1]
761 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
762 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
763
764 clean(getTestOpts().clean_files)
765
766 if getTestOpts().outputdir != None:
767 odir = in_testdir(getTestOpts().outputdir)
768 try:
769 shutil.rmtree(odir)
770 except:
771 pass
772
773 try:
774 shutil.rmtree(in_testdir('.hpc.' + name))
775 except:
776 pass
777
778 try:
779 cleanCmd = getTestOpts().clean_cmd
780 if cleanCmd != None:
781 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
782 if result != 0:
783 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
784 except e:
785 framework_fail(name, 'cleaning', 'clean-command exception')
786
787 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
788
789 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
790 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
791
792 try:
793 for f in files_written[name]:
794 if os.path.exists(f):
795 try:
796 if not f in files_written_not_removed[name]:
797 files_written_not_removed[name].append(f)
798 except:
799 files_written_not_removed[name] = [f]
800 except:
801 pass
802 except Exception, e:
803 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
804
805 def clean(strs):
806 for str in strs:
807 for name in glob.glob(in_testdir(str)):
808 clean_full_path(name)
809
810 def clean_full_path(name):
811 try:
812 # Remove files...
813 os.remove(name)
814 except OSError, e1:
815 try:
816 # ... and empty directories
817 os.rmdir(name)
818 except OSError, e2:
819 # We don't want to fail here, but we do want to know
820 # what went wrong, so print out the exceptions.
821 # ENOENT isn't a problem, though, as we clean files
822 # that don't necessarily exist.
823 if e1.errno != errno.ENOENT:
824 print e1
825 if e2.errno != errno.ENOENT:
826 print e2
827
828 def do_test(name, way, func, args):
829 full_name = name + '(' + way + ')'
830
831 try:
832 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
833 str([t.n_unexpected_passes, \
834 t.n_unexpected_failures, \
835 t.n_framework_failures])
836
837 if config.use_threads:
838 t.lock.release()
839
840 try:
841 preCmd = getTestOpts().pre_cmd
842 if preCmd != None:
843 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
844 if result != 0:
845 framework_fail(name, way, 'pre-command failed: ' + str(result))
846 except e:
847 framework_fail(name, way, 'pre-command exception')
848
849 try:
850 result = apply(func, [name,way] + args)
851 finally:
852 if config.use_threads:
853 t.lock.acquire()
854
855 if getTestOpts().expect != 'pass' and \
856 getTestOpts().expect != 'fail' and \
857 getTestOpts().expect != 'missing-lib':
858 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
859
860 try:
861 passFail = result['passFail']
862 except:
863 passFail = 'No passFail found'
864
865 if passFail == 'pass':
866 if getTestOpts().expect == 'pass' \
867 and way not in getTestOpts().expect_fail_for:
868 t.n_expected_passes = t.n_expected_passes + 1
869 if name in t.expected_passes:
870 t.expected_passes[name].append(way)
871 else:
872 t.expected_passes[name] = [way]
873 else:
874 print '*** unexpected pass for', full_name
875 t.n_unexpected_passes = t.n_unexpected_passes + 1
876 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
877 elif passFail == 'fail':
878 if getTestOpts().expect == 'pass' \
879 and way not in getTestOpts().expect_fail_for:
880 print '*** unexpected failure for', full_name
881 t.n_unexpected_failures = t.n_unexpected_failures + 1
882 reason = result['reason']
883 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
884 else:
885 if getTestOpts().expect == 'missing-lib':
886 t.n_missing_libs = t.n_missing_libs + 1
887 if name in t.missing_libs:
888 t.missing_libs[name].append(way)
889 else:
890 t.missing_libs[name] = [way]
891 else:
892 t.n_expected_failures = t.n_expected_failures + 1
893 if name in t.expected_failures:
894 t.expected_failures[name].append(way)
895 else:
896 t.expected_failures[name] = [way]
897 else:
898 framework_fail(name, way, 'bad result ' + passFail)
899 except KeyboardInterrupt:
900 stopNow()
901 except:
902 framework_fail(name, way, 'do_test exception')
903 traceback.print_exc()
904
905 def addPassingTestInfo (testInfos, directory, name, way):
906 directory = re.sub('^\\.[/\\\\]', '', directory)
907
908 if not directory in testInfos:
909 testInfos[directory] = {}
910
911 if not name in testInfos[directory]:
912 testInfos[directory][name] = []
913
914 testInfos[directory][name].append(way)
915
916 def addFailingTestInfo (testInfos, directory, name, reason, way):
917 directory = re.sub('^\\.[/\\\\]', '', directory)
918
919 if not directory in testInfos:
920 testInfos[directory] = {}
921
922 if not name in testInfos[directory]:
923 testInfos[directory][name] = {}
924
925 if not reason in testInfos[directory][name]:
926 testInfos[directory][name][reason] = []
927
928 testInfos[directory][name][reason].append(way)
929
930 def skiptest (name, way):
931 # print 'Skipping test \"', name, '\"'
932 t.n_tests_skipped = t.n_tests_skipped + 1
933 if name in t.tests_skipped:
934 t.tests_skipped[name].append(way)
935 else:
936 t.tests_skipped[name] = [way]
937
938 def framework_fail( name, way, reason ):
939 full_name = name + '(' + way + ')'
940 print '*** framework failure for', full_name, reason
941 t.n_framework_failures = t.n_framework_failures + 1
942 if name in t.framework_failures:
943 t.framework_failures[name].append(way)
944 else:
945 t.framework_failures[name] = [way]
946
947 def badResult(result):
948 try:
949 if result['passFail'] == 'pass':
950 return False
951 return True
952 except:
953 return True
954
955 def passed():
956 return {'passFail': 'pass'}
957
958 def failBecause(reason):
959 return {'passFail': 'fail', 'reason': reason}
960
961 # -----------------------------------------------------------------------------
962 # Generic command tests
963
964 # A generic command test is expected to run and exit successfully.
965 #
966 # The expected exit code can be changed via exit_code() as normal, and
967 # the expected stdout/stderr are stored in <testname>.stdout and
968 # <testname>.stderr. The output of the command can be ignored
969 # altogether by using run_command_ignore_output instead of
970 # run_command.
971
972 def run_command( name, way, cmd ):
973 return simple_run( name, '', cmd, '' )
974
975 # -----------------------------------------------------------------------------
976 # GHCi tests
977
978 def ghci_script( name, way, script ):
979 # filter out -fforce-recomp from compiler_always_flags, because we're
980 # actually testing the recompilation behaviour in the GHCi tests.
981 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
982 flags.append(getTestOpts().extra_hc_opts)
983 if getTestOpts().outputdir != None:
984 flags.extend(["-outputdir", getTestOpts().outputdir])
985
986 # We pass HC and HC_OPTS as environment variables, so that the
987 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
988 cmd = "HC='" + config.compiler + "' " + \
989 "HC_OPTS='" + join(flags,' ') + "' " + \
990 "'" + config.compiler + "'" + \
991 ' --interactive -v0 -ignore-dot-ghci ' + \
992 join(flags,' ')
993
994 getTestOpts().stdin = script
995 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
996
997 # -----------------------------------------------------------------------------
998 # Compile-only tests
999
1000 def compile( name, way, extra_hc_opts ):
1001 return do_compile( name, way, 0, '', [], extra_hc_opts )
1002
1003 def compile_fail( name, way, extra_hc_opts ):
1004 return do_compile( name, way, 1, '', [], extra_hc_opts )
1005
1006 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1007 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1008
1009 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1010 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1011
1012 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1013 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1014
1015 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1016 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1017
1018 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1019 # print 'Compile only, extra args = ', extra_hc_opts
1020 pretest_cleanup(name)
1021
1022 result = extras_build( way, extra_mods, extra_hc_opts )
1023 if badResult(result):
1024 return result
1025 extra_hc_opts = result['hc_opts']
1026
1027 force = 0
1028 if extra_mods:
1029 force = 1
1030 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1031
1032 if badResult(result):
1033 return result
1034
1035 # the actual stderr should always match the expected, regardless
1036 # of whether we expected the compilation to fail or not (successful
1037 # compilations may generate warnings).
1038
1039 if getTestOpts().with_namebase == None:
1040 namebase = name
1041 else:
1042 namebase = getTestOpts().with_namebase
1043
1044 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1045 actual_stderr_file = qualify(name, 'comp.stderr')
1046
1047 if not compare_outputs('stderr', \
1048 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1049 expected_stderr_file, actual_stderr_file):
1050 return failBecause('stderr mismatch')
1051
1052 # no problems found, this test passed
1053 return passed()
1054
1055 def compile_cmp_asm( name, way, extra_hc_opts ):
1056 print 'Compile only, extra args = ', extra_hc_opts
1057 pretest_cleanup(name)
1058 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1059
1060 if badResult(result):
1061 return result
1062
1063 # the actual stderr should always match the expected, regardless
1064 # of whether we expected the compilation to fail or not (successful
1065 # compilations may generate warnings).
1066
1067 if getTestOpts().with_namebase == None:
1068 namebase = name
1069 else:
1070 namebase = getTestOpts().with_namebase
1071
1072 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1073 actual_asm_file = qualify(name, 's')
1074
1075 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1076 expected_asm_file, actual_asm_file):
1077 return failBecause('asm mismatch')
1078
1079 # no problems found, this test passed
1080 return passed()
1081
1082 # -----------------------------------------------------------------------------
1083 # Compile-and-run tests
1084
1085 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1086 # print 'Compile and run, extra args = ', extra_hc_opts
1087 pretest_cleanup(name)
1088
1089 result = extras_build( way, extra_mods, extra_hc_opts )
1090 if badResult(result):
1091 return result
1092 extra_hc_opts = result['hc_opts']
1093
1094 if way == 'ghci': # interpreted...
1095 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1096 elif way == 'extcore' or way == 'optextcore' :
1097 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1098 else: # compiled...
1099 force = 0
1100 if extra_mods:
1101 force = 1
1102
1103 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1104 if badResult(result):
1105 return result
1106
1107 cmd = './' + name;
1108
1109 # we don't check the compiler's stderr for a compile-and-run test
1110 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1111
1112 def compile_and_run( name, way, extra_hc_opts ):
1113 return compile_and_run__( name, way, '', [], extra_hc_opts)
1114
1115 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1116 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1117
1118 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1119 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1120
1121 def stats( name, way, stats_file ):
1122 opts = getTestOpts()
1123 return checkStats(stats_file, opts.stats_range_fields)
1124
1125 # -----------------------------------------------------------------------------
1126 # Check -t stats info
1127
1128 def checkStats(stats_file, range_fields):
1129 result = passed()
1130 if len(range_fields) > 0:
1131 f = open(in_testdir(stats_file))
1132 contents = f.read()
1133 f.close()
1134
1135 for (field, (expected, dev)) in range_fields.items():
1136 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1137 if m == None:
1138 print 'Failed to find field: ', field
1139 result = failBecause('no such stats field')
1140 val = int(m.group(1))
1141
1142 lowerBound = trunc( expected * ((100 - float(dev))/100));
1143 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)));
1144
1145 if val < lowerBound:
1146 print field, 'value is too low:'
1147 print '(If this is because you have improved GHC, please'
1148 print 'update the test so that GHC doesn\'t regress again)'
1149 result = failBecause('stat too good')
1150 if val > upperBound:
1151 print field, 'value is too high:'
1152 result = failBecause('stat not good enough')
1153
1154 if val < lowerBound or val > upperBound:
1155 valStr = str(val)
1156 valLen = len(valStr)
1157 expectedStr = str(expected)
1158 expectedLen = len(expectedStr)
1159 length = max(map (lambda x : len(str(x)), [expected, lowerBound, upperBound, val]))
1160 def display(descr, val, extra):
1161 print descr, string.rjust(str(val), length), extra
1162 display(' Expected ' + field + ':', expected, '+/-' + str(dev) + '%')
1163 display(' Lower bound ' + field + ':', lowerBound, '')
1164 display(' Upper bound ' + field + ':', upperBound, '')
1165 display(' Actual ' + field + ':', val, '')
1166
1167 return result
1168
1169 # -----------------------------------------------------------------------------
1170 # Build a single-module program
1171
1172 def extras_build( way, extra_mods, extra_hc_opts ):
1173 for modopts in extra_mods:
1174 mod, opts = modopts
1175 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1176 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1177 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1178 if badResult(result):
1179 return result
1180
1181 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1182
1183
1184 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1185 opts = getTestOpts()
1186 errname = add_suffix(name, 'comp.stderr')
1187 rm_no_fail( qualify(errname, '') )
1188
1189 if top_mod != '':
1190 srcname = top_mod
1191 rm_no_fail( qualify(name, '') )
1192 base, suf = os.path.splitext(top_mod)
1193 rm_no_fail( qualify(base, '') )
1194 rm_no_fail( qualify(base, 'exe') )
1195 elif addsuf:
1196 srcname = add_hs_lhs_suffix(name)
1197 rm_no_fail( qualify(name, '') )
1198 else:
1199 srcname = name
1200 rm_no_fail( qualify(name, 'o') )
1201
1202 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1203
1204 to_do = ''
1205 if top_mod != '':
1206 to_do = '--make '
1207 if link:
1208 to_do = to_do + '-o ' + name
1209 elif link:
1210 to_do = '-o ' + name
1211 elif opts.compile_to_hc:
1212 to_do = '-C'
1213 else:
1214 to_do = '-c' # just compile
1215
1216 stats_file = name + '.comp.stats'
1217 if len(opts.compiler_stats_range_fields) > 0:
1218 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1219
1220 # Required by GHC 7.3+, harmless for earlier versions:
1221 if (getTestOpts().c_src or
1222 getTestOpts().objc_src or
1223 getTestOpts().objcpp_src or
1224 getTestOpts().cmm_src):
1225 extra_hc_opts += ' -no-hs-main '
1226
1227 if getTestOpts().compile_cmd_prefix == '':
1228 cmd_prefix = ''
1229 else:
1230 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1231
1232 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1233 if noforce:
1234 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1235 if getTestOpts().outputdir != None:
1236 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1237
1238 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1239 + config.compiler + "' " \
1240 + join(comp_flags,' ') + ' ' \
1241 + to_do + ' ' + srcname + ' ' \
1242 + join(config.way_flags(name)[way],' ') + ' ' \
1243 + extra_hc_opts + ' ' \
1244 + opts.extra_hc_opts + ' ' \
1245 + '>' + errname + ' 2>&1'
1246
1247 result = runCmdFor(name, cmd)
1248
1249 if result != 0 and not should_fail:
1250 actual_stderr = qualify(name, 'comp.stderr')
1251 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1252 if_verbose_dump(1,actual_stderr)
1253
1254 # ToDo: if the sub-shell was killed by ^C, then exit
1255
1256 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields)
1257
1258 if badResult(statsResult):
1259 return statsResult
1260
1261 if should_fail:
1262 if result == 0:
1263 return failBecause('exit code 0')
1264 else:
1265 if result != 0:
1266 return failBecause('exit code non-0')
1267
1268 return passed()
1269
1270 # -----------------------------------------------------------------------------
1271 # Run a program and check its output
1272 #
1273 # If testname.stdin exists, route input from that, else
1274 # from /dev/null. Route output to testname.run.stdout and
1275 # testname.run.stderr. Returns the exit code of the run.
1276
1277 def simple_run( name, way, prog, args ):
1278 opts = getTestOpts()
1279
1280 # figure out what to use for stdin
1281 if opts.stdin != '':
1282 use_stdin = opts.stdin
1283 else:
1284 stdin_file = add_suffix(name, 'stdin')
1285 if os.path.exists(in_testdir(stdin_file)):
1286 use_stdin = stdin_file
1287 else:
1288 use_stdin = '/dev/null'
1289
1290 run_stdout = add_suffix(name,'run.stdout')
1291 run_stderr = add_suffix(name,'run.stderr')
1292
1293 rm_no_fail(qualify(name,'run.stdout'))
1294 rm_no_fail(qualify(name,'run.stderr'))
1295 rm_no_fail(qualify(name, 'hp'))
1296 rm_no_fail(qualify(name,'ps'))
1297 rm_no_fail(qualify(name, 'prof'))
1298
1299 my_rts_flags = rts_flags(way)
1300
1301 stats_file = name + '.stats'
1302 if len(opts.stats_range_fields) > 0:
1303 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1304
1305 if opts.no_stdin:
1306 stdin_comes_from = ''
1307 else:
1308 stdin_comes_from = ' <' + use_stdin
1309
1310 if opts.combined_output:
1311 redirection = ' >' + run_stdout \
1312 + ' 2>&1'
1313 else:
1314 redirection = ' >' + run_stdout \
1315 + ' 2>' + run_stderr
1316
1317 cmd = prog + ' ' + args + ' ' \
1318 + my_rts_flags + ' ' \
1319 + stdin_comes_from \
1320 + redirection
1321
1322 if opts.cmd_wrapper != None:
1323 cmd = opts.cmd_wrapper(cmd);
1324
1325 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1326
1327 # run the command
1328 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1329
1330 exit_code = result >> 8
1331 signal = result & 0xff
1332
1333 # check the exit code
1334 if exit_code != opts.exit_code:
1335 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1336 dump_stdout(name)
1337 dump_stderr(name)
1338 return failBecause('bad exit code')
1339
1340 check_hp = my_rts_flags.find("-h") != -1
1341 check_prof = my_rts_flags.find("-p") != -1
1342
1343 if not opts.ignore_output:
1344 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1345 bad_stdout = not check_stdout_ok(name)
1346 if bad_stderr:
1347 return failBecause('bad stderr')
1348 if bad_stdout:
1349 return failBecause('bad stdout')
1350 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1351 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1352 return failBecause('bad heap profile')
1353 if check_prof and not check_prof_ok(name):
1354 return failBecause('bad profile')
1355
1356 return checkStats(stats_file, opts.stats_range_fields)
1357
1358 def rts_flags(way):
1359 if (way == ''):
1360 return ''
1361 else:
1362 args = config.way_rts_flags[way]
1363
1364 if args == []:
1365 return ''
1366 else:
1367 return '+RTS ' + join(args,' ') + ' -RTS'
1368
1369 # -----------------------------------------------------------------------------
1370 # Run a program in the interpreter and check its output
1371
1372 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1373 outname = add_suffix(name, 'interp.stdout')
1374 errname = add_suffix(name, 'interp.stderr')
1375 rm_no_fail(outname)
1376 rm_no_fail(errname)
1377 rm_no_fail(name)
1378
1379 if (top_mod == ''):
1380 srcname = add_hs_lhs_suffix(name)
1381 else:
1382 srcname = top_mod
1383
1384 scriptname = add_suffix(name, 'genscript')
1385 qscriptname = in_testdir(scriptname)
1386 rm_no_fail(qscriptname)
1387
1388 delimiter = '===== program output begins here\n'
1389
1390 script = open(qscriptname, 'w')
1391 if not compile_only:
1392 # set the prog name and command-line args to match the compiled
1393 # environment.
1394 script.write(':set prog ' + name + '\n')
1395 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1396 # Add marker lines to the stdout and stderr output files, so we
1397 # can separate GHCi's output from the program's.
1398 script.write(':! echo ' + delimiter)
1399 script.write(':! echo 1>&2 ' + delimiter)
1400 # Set stdout to be line-buffered to match the compiled environment.
1401 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1402 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1403 # in the event of an exception as for the compiled program.
1404 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1405 script.close()
1406
1407 # figure out what to use for stdin
1408 if getTestOpts().stdin != '':
1409 stdin_file = in_testdir(getTestOpts().stdin)
1410 else:
1411 stdin_file = qualify(name, 'stdin')
1412
1413 if os.path.exists(stdin_file):
1414 stdin = open(stdin_file, 'r')
1415 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1416
1417 script.close()
1418
1419 flags = copy.copy(getTestOpts().compiler_always_flags)
1420 if getTestOpts().outputdir != None:
1421 flags.extend(["-outputdir", getTestOpts().outputdir])
1422
1423 cmd = "'" + config.compiler + "' " \
1424 + join(flags,' ') + ' ' \
1425 + srcname + ' ' \
1426 + join(config.way_flags(name)[way],' ') + ' ' \
1427 + extra_hc_opts + ' ' \
1428 + getTestOpts().extra_hc_opts + ' ' \
1429 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1430
1431 if getTestOpts().cmd_wrapper != None:
1432 cmd = getTestOpts().cmd_wrapper(cmd);
1433
1434 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1435
1436 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1437
1438 exit_code = result >> 8
1439 signal = result & 0xff
1440
1441 # split the stdout into compilation/program output
1442 split_file(in_testdir(outname), delimiter,
1443 qualify(name, 'comp.stdout'),
1444 qualify(name, 'run.stdout'))
1445 split_file(in_testdir(errname), delimiter,
1446 qualify(name, 'comp.stderr'),
1447 qualify(name, 'run.stderr'))
1448
1449 # check the exit code
1450 if exit_code != getTestOpts().exit_code:
1451 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1452 dump_stdout(name)
1453 dump_stderr(name)
1454 return failBecause('bad exit code')
1455
1456 # ToDo: if the sub-shell was killed by ^C, then exit
1457
1458 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1459 check_stdout_ok(name)):
1460 return passed()
1461 else:
1462 return failBecause('bad stdout or stderr')
1463
1464
1465 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1466 infile = open(in_fn)
1467 out1 = open(out1_fn, 'w')
1468 out2 = open(out2_fn, 'w')
1469
1470 line = infile.readline()
1471 line = re.sub('\r', '', line) # ignore Windows EOL
1472 while (re.sub('^\s*','',line) != delimiter and line != ''):
1473 out1.write(line)
1474 line = infile.readline()
1475 line = re.sub('\r', '', line)
1476 out1.close()
1477
1478 line = infile.readline()
1479 while (line != ''):
1480 out2.write(line)
1481 line = infile.readline()
1482 out2.close()
1483
1484 # -----------------------------------------------------------------------------
1485 # Generate External Core for the given program, then compile the resulting Core
1486 # and compare its output to the expected output
1487
1488 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1489
1490 depsfilename = qualify(name, 'deps')
1491 errname = add_suffix(name, 'comp.stderr')
1492 qerrname = qualify(errname,'')
1493
1494 hcname = qualify(name, 'hc')
1495 oname = qualify(name, 'o')
1496
1497 rm_no_fail( qerrname )
1498 rm_no_fail( qualify(name, '') )
1499
1500 if (top_mod == ''):
1501 srcname = add_hs_lhs_suffix(name)
1502 else:
1503 srcname = top_mod
1504
1505 qcorefilename = qualify(name, 'hcr')
1506 corefilename = add_suffix(name, 'hcr')
1507 rm_no_fail(qcorefilename)
1508
1509 # Generate External Core
1510
1511 if (top_mod == ''):
1512 to_do = ' ' + srcname + ' '
1513 else:
1514 to_do = ' --make ' + top_mod + ' '
1515
1516 flags = copy.copy(getTestOpts().compiler_always_flags)
1517 if getTestOpts().outputdir != None:
1518 flags.extend(["-outputdir", getTestOpts().outputdir])
1519 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1520 + config.compiler + "' " \
1521 + join(flags,' ') + ' ' \
1522 + join(config.way_flags(name)[way],' ') + ' ' \
1523 + extra_hc_opts + ' ' \
1524 + getTestOpts().extra_hc_opts \
1525 + to_do \
1526 + '>' + errname + ' 2>&1'
1527 result = runCmdFor(name, cmd)
1528
1529 exit_code = result >> 8
1530
1531 if exit_code != 0:
1532 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1533 if_verbose_dump(1,qerrname)
1534 return failBecause('ext core exit code non-0')
1535
1536 # Compile the resulting files -- if there's more than one module, we need to read the output
1537 # of the previous compilation in order to find the dependencies
1538 if (top_mod == ''):
1539 to_compile = corefilename
1540 else:
1541 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1542 deps = open(depsfilename).read()
1543 deplist = string.replace(deps, '\n',' ');
1544 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1545 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1546
1547 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1548 if getTestOpts().outputdir != None:
1549 flags.extend(["-outputdir", getTestOpts().outputdir])
1550
1551 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1552 + config.compiler + "' " \
1553 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1554 + to_compile + ' ' \
1555 + extra_hc_opts + ' ' \
1556 + getTestOpts().extra_hc_opts + ' ' \
1557 + flags \
1558 + ' -fglasgow-exts -o ' + name \
1559 + '>' + errname + ' 2>&1'
1560
1561 result = runCmdFor(name, cmd)
1562 exit_code = result >> 8
1563
1564 if exit_code != 0:
1565 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1566 if_verbose_dump(1,qerrname)
1567 return failBecause('ext core exit code non-0')
1568
1569 # Clean up
1570 rm_no_fail ( oname )
1571 rm_no_fail ( hcname )
1572 rm_no_fail ( qcorefilename )
1573 rm_no_fail ( depsfilename )
1574
1575 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1576
1577 # -----------------------------------------------------------------------------
1578 # Utils
1579
1580 def check_stdout_ok( name ):
1581 if getTestOpts().with_namebase == None:
1582 namebase = name
1583 else:
1584 namebase = getTestOpts().with_namebase
1585
1586 actual_stdout_file = qualify(name, 'run.stdout')
1587 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1588
1589 def norm(str):
1590 if platform_specific:
1591 return str
1592 else:
1593 return normalise_output(str)
1594
1595 return compare_outputs('stdout', \
1596 two_normalisers(norm, getTestOpts().extra_normaliser), \
1597 expected_stdout_file, actual_stdout_file)
1598
1599 def dump_stdout( name ):
1600 print 'Stdout:'
1601 print read_no_crs(qualify(name, 'run.stdout'))
1602
1603 def check_stderr_ok( name ):
1604 if getTestOpts().with_namebase == None:
1605 namebase = name
1606 else:
1607 namebase = getTestOpts().with_namebase
1608
1609 actual_stderr_file = qualify(name, 'run.stderr')
1610 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1611
1612 def norm(str):
1613 if platform_specific:
1614 return str
1615 else:
1616 return normalise_errmsg(str)
1617
1618 return compare_outputs('stderr', \
1619 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1620 expected_stderr_file, actual_stderr_file)
1621
1622 def dump_stderr( name ):
1623 print "Stderr:"
1624 print read_no_crs(qualify(name, 'run.stderr'))
1625
1626 def read_no_crs(file):
1627 str = ''
1628 try:
1629 h = open(file)
1630 str = h.read()
1631 h.close
1632 except:
1633 # On Windows, if the program fails very early, it seems the
1634 # files stdout/stderr are redirected to may not get created
1635 pass
1636 return re.sub('\r', '', str)
1637
1638 def write_file(file, str):
1639 h = open(file, 'w')
1640 h.write(str)
1641 h.close
1642
1643 def check_hp_ok(name):
1644
1645 # do not qualify for hp2ps because we should be in the right directory
1646 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1647
1648 hp2psResult = runCmdExitCode(hp2psCmd)
1649
1650 actual_ps_file = qualify(name, 'ps')
1651
1652 if(hp2psResult == 0):
1653 if (os.path.exists(actual_ps_file)):
1654 if gs_working:
1655 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1656 if (gsResult == 0):
1657 return (True)
1658 else:
1659 print "hp2ps output for " + name + "is not valid PostScript"
1660 else: return (True) # assume postscript is valid without ghostscript
1661 else:
1662 print "hp2ps did not generate PostScript for " + name
1663 return (False)
1664 else:
1665 print "hp2ps error when processing heap profile for " + name
1666 return(False)
1667
1668 def check_prof_ok(name):
1669
1670 prof_file = qualify(name,'prof')
1671
1672 if not os.path.exists(prof_file):
1673 print prof_file + " does not exist"
1674 return(False)
1675
1676 if os.path.getsize(qualify(name,'prof')) == 0:
1677 print prof_file + " is empty"
1678 return(False)
1679
1680 if getTestOpts().with_namebase == None:
1681 namebase = name
1682 else:
1683 namebase = getTestOpts().with_namebase
1684
1685 (platform_specific, expected_prof_file) = \
1686 platform_wordsize_qualify(namebase, 'prof.sample')
1687
1688 # sample prof file is not required
1689 if not os.path.exists(expected_prof_file):
1690 return True
1691 else:
1692 return compare_outputs('prof', \
1693 two_normalisers(normalise_whitespace,normalise_prof), \
1694 expected_prof_file, prof_file)
1695
1696 # Compare expected output to actual output, and optionally accept the
1697 # new output. Returns true if output matched or was accepted, false
1698 # otherwise.
1699 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1700 if os.path.exists(expected_file):
1701 expected_raw = read_no_crs(expected_file)
1702 # print "norm:", normaliser(expected_raw)
1703 expected_str = normaliser(expected_raw)
1704 expected_file_for_diff = expected_file
1705 else:
1706 expected_str = ''
1707 expected_file_for_diff = '/dev/null'
1708
1709 actual_raw = read_no_crs(actual_file)
1710 actual_str = normaliser(actual_raw)
1711
1712 if expected_str == actual_str:
1713 return 1
1714 else:
1715 print 'Actual ' + kind + ' output differs from expected:'
1716
1717 if expected_file_for_diff == '/dev/null':
1718 expected_normalised_file = '/dev/null'
1719 else:
1720 expected_normalised_file = expected_file + ".normalised"
1721 write_file(expected_normalised_file, expected_str)
1722
1723 actual_normalised_file = actual_file + ".normalised"
1724 write_file(actual_normalised_file, actual_str)
1725
1726 # Ignore whitespace when diffing. We should only get to this
1727 # point if there are non-whitespace differences
1728 #
1729 # Note we are diffing the *actual* output, not the normalised
1730 # output. The normalised output may have whitespace squashed
1731 # (including newlines) so the diff would be hard to read.
1732 # This does mean that the diff might contain changes that
1733 # would be normalised away.
1734 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1735 ' ' + actual_file )
1736
1737 # If for some reason there were no non-whitespace differences,
1738 # then do a full diff
1739 if r == 0:
1740 r = os.system( 'diff -u ' + expected_file_for_diff + \
1741 ' ' + actual_file )
1742
1743 if config.accept:
1744 print 'Accepting new output.'
1745 write_file(expected_file, actual_raw)
1746 return 1
1747 else:
1748 return 0
1749
1750
1751 def normalise_whitespace( str ):
1752 # Merge contiguous whitespace characters into a single space.
1753 str = re.sub('[ \t\n]+', ' ', str)
1754 return str
1755
1756 def normalise_errmsg( str ):
1757 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1758 # the colon is there because it appears in error messages; this
1759 # hacky solution is used in place of more sophisticated filename
1760 # mangling
1761 str = re.sub('([^\\s])\\.exe', '\\1', str)
1762 # normalise slashes, minimise Windows/Unix filename differences
1763 str = re.sub('\\\\', '/', str)
1764 # The inplace ghc's are called ghc-stage[123] to avoid filename
1765 # collisions, so we need to normalise that to just "ghc"
1766 str = re.sub('ghc-stage[123]', 'ghc', str)
1767 # We sometimes see the name of the integer-gmp package on stderr,
1768 # but this can change (either the implementation name or the
1769 # version number), so we canonicalise it here
1770 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1771 return str
1772
1773 # normalise a .prof file, so that we can reasonably compare it against
1774 # a sample. This doesn't compare any of the actual profiling data,
1775 # only the shape of the profile and the number of entries.
1776 def normalise_prof (str):
1777 # strip everything up to the line beginning "COST CENTRE"
1778 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1779
1780 # strip results for CAFs, these tend to change unpredictably
1781 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1782
1783 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1784 # sometimes under MAIN.
1785 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1786
1787 # We have somthing like this:
1788
1789 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1790 # k Main 204 1 0.0 0.0 0.0 0.0
1791 # foo Main 205 1 0.0 0.0 0.0 0.0
1792 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1793
1794 # then we remove all the specific profiling data, leaving only the
1795 # cost centre name, module, and entries, to end up with this:
1796
1797 # MAIN MAIN 0
1798 # k Main 1
1799 # foo Main 1
1800 # foo.bar Main 1
1801
1802 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1803 return str
1804
1805 def normalise_slashes_( str ):
1806 str = re.sub('\\\\', '/', str)
1807 return str
1808
1809 def normalise_exe_( str ):
1810 str = re.sub('\.exe', '', str)
1811 return str
1812
1813 def normalise_output( str ):
1814 # Remove a .exe extension (for Windows)
1815 # This can occur in error messages generated by the program.
1816 str = re.sub('([^\\s])\\.exe', '\\1', str)
1817 return str
1818
1819 def normalise_asm( str ):
1820 lines = str.split('\n')
1821 # Only keep instructions and labels not starting with a dot.
1822 metadata = re.compile('^[ \t]*\\..*$')
1823 out = []
1824 for line in lines:
1825 # Drop metadata directives (e.g. ".type")
1826 if not metadata.match(line):
1827 line = re.sub('@plt', '', line)
1828 instr = line.lstrip().split()
1829 # Drop empty lines.
1830 if not instr:
1831 continue
1832 # Drop operands, except for call instructions.
1833 elif instr[0] == 'call':
1834 out.append(instr[0] + ' ' + instr[1])
1835 else:
1836 out.append(instr[0])
1837 out = '\n'.join(out)
1838 return out
1839
1840 def if_verbose( n, str ):
1841 if config.verbose >= n:
1842 print str
1843
1844 def if_verbose_dump( n, f ):
1845 if config.verbose >= n:
1846 try:
1847 print open(f).read()
1848 except:
1849 print ''
1850
1851 def rawSystem(cmd_and_args):
1852 # We prefer subprocess.call to os.spawnv as the latter
1853 # seems to send its arguments through a shell or something
1854 # with the Windows (non-cygwin) python. An argument "a b c"
1855 # turns into three arguments ["a", "b", "c"].
1856
1857 # However, subprocess is new in python 2.4, so fall back to
1858 # using spawnv if we don't have it
1859
1860 if have_subprocess:
1861 return subprocess.call(cmd_and_args)
1862 else:
1863 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1864
1865 # Note that this doesn't handle the timeout itself; it is just used for
1866 # commands that have timeout handling built-in.
1867 def rawSystemWithTimeout(cmd_and_args):
1868 r = rawSystem(cmd_and_args)
1869 if r == 98:
1870 # The python timeout program uses 98 to signal that ^C was pressed
1871 stopNow()
1872 return r
1873
1874 # cmd is a complex command in Bourne-shell syntax
1875 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1876 # Hence it must ultimately be run by a Bourne shell
1877 #
1878 # Mostly it invokes the command wrapped in 'timeout' thus
1879 # timeout 300 'cd . && ...blah blah'
1880 # so it's timeout's job to invoke the Bourne shell
1881 #
1882 # But watch out for the case when there is no timeout program!
1883 # Then, when using the native Python, os.system will invoke the cmd shell
1884
1885 def runCmd( cmd ):
1886 if_verbose( 1, cmd )
1887 r = 0
1888 if config.os == 'mingw32':
1889 # On MinGW, we will always have timeout
1890 assert config.timeout_prog!=''
1891
1892 if config.timeout_prog != '':
1893 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1894 else:
1895 r = os.system(cmd)
1896 return r << 8
1897
1898 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1899 if_verbose( 1, cmd )
1900 r = 0
1901 if config.os == 'mingw32':
1902 # On MinGW, we will always have timeout
1903 assert config.timeout_prog!=''
1904 timeout = int(ceil(config.timeout * timeout_multiplier))
1905
1906 if config.timeout_prog != '':
1907 if config.check_files_written:
1908 fn = name + ".strace"
1909 r = rawSystemWithTimeout(
1910 ["strace", "-o", fn, "-fF",
1911 "-e", "creat,open,chdir,clone,vfork",
1912 config.timeout_prog, str(timeout), cmd])
1913 addTestFilesWritten(name, fn)
1914 rm_no_fail(fn)
1915 else:
1916 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1917 else:
1918 r = os.system(cmd)
1919 return r << 8
1920
1921 def runCmdExitCode( cmd ):
1922 return (runCmd(cmd) >> 8);
1923
1924
1925 # -----------------------------------------------------------------------------
1926 # checking for files being written to by multiple tests
1927
1928 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1929 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1930 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1931 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1932 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1933 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1934 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1935 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1936 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1937 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1938 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1939 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1940 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1941 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1942 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1943 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1944 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1945
1946 # Files that are read or written but shouldn't be:
1947 # * ghci_history shouldn't be read or written by tests
1948 # * things under package.conf.d shouldn't be written by tests
1949 bad_file_usages = {}
1950
1951 # Mapping from tests to the list of files that they write
1952 files_written = {}
1953
1954 # Mapping from tests to the list of files that they write but don't clean
1955 files_written_not_removed = {}
1956
1957 def add_bad_file_usage(name, file):
1958 try:
1959 if not file in bad_file_usages[name]:
1960 bad_file_usages[name].append(file)
1961 except:
1962 bad_file_usages[name] = [file]
1963
1964 def mkPath(curdir, path):
1965 # Given the current full directory is 'curdir', what is the full
1966 # path to 'path'?
1967 return os.path.realpath(os.path.join(curdir, path))
1968
1969 def addTestFilesWritten(name, fn):
1970 if config.use_threads:
1971 with t.lockFilesWritten:
1972 addTestFilesWrittenHelper(name, fn)
1973 else:
1974 addTestFilesWrittenHelper(name, fn)
1975
1976 def addTestFilesWrittenHelper(name, fn):
1977 started = False
1978 working_directories = {}
1979
1980 with open(fn, 'r') as f:
1981 for line in f:
1982 m_pid = re_strace_pid.match(line)
1983 if m_pid:
1984 pid = m_pid.group(1)
1985 content = m_pid.group(2)
1986 elif re_strace_unavailable.match(line):
1987 next
1988 else:
1989 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1990
1991 m_open = re_strace_open.match(content)
1992 m_chdir = re_strace_chdir.match(content)
1993 m_clone = re_strace_clone.match(content)
1994 m_vfork = re_strace_vfork.match(content)
1995
1996 if not started:
1997 working_directories[pid] = os.getcwd()
1998 started = True
1999
2000 if m_open:
2001 file = m_open.group(1)
2002 file = mkPath(working_directories[pid], file)
2003 if file.endswith("ghci_history"):
2004 add_bad_file_usage(name, file)
2005 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
2006 flags = m_open.group(2).split('|')
2007 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
2008 if re.match('package\.conf\.d', file):
2009 add_bad_file_usage(name, file)
2010 else:
2011 try:
2012 if not file in files_written[name]:
2013 files_written[name].append(file)
2014 except:
2015 files_written[name] = [file]
2016 elif 'O_RDONLY' in flags:
2017 pass
2018 else:
2019 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2020 elif m_chdir:
2021 # We optimistically assume that unfinished chdir's are going to succeed
2022 dir = m_chdir.group(1)
2023 working_directories[pid] = mkPath(working_directories[pid], dir)
2024 elif m_clone:
2025 working_directories[m_clone.group(2)] = working_directories[pid]
2026 elif m_vfork:
2027 working_directories[m_vfork.group(2)] = working_directories[pid]
2028 elif re_strace_open_resumed.match(content):
2029 pass
2030 elif re_strace_chdir_resumed.match(content):
2031 pass
2032 elif re_strace_vfork_unfinished.match(content):
2033 pass
2034 elif re_strace_clone_unfinished.match(content):
2035 pass
2036 elif re_strace_ignore_sigchild.match(content):
2037 pass
2038 elif re_strace_ignore_sigvtalarm.match(content):
2039 pass
2040 elif re_strace_ignore_sigint.match(content):
2041 pass
2042 elif re_strace_ignore_sigfpe.match(content):
2043 pass
2044 elif re_strace_ignore_sigsegv.match(content):
2045 pass
2046 elif re_strace_ignore_sigpipe.match(content):
2047 pass
2048 else:
2049 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2050
2051 def checkForFilesWrittenProblems(file):
2052 foundProblem = False
2053
2054 files_written_inverted = {}
2055 for t in files_written.keys():
2056 for f in files_written[t]:
2057 try:
2058 files_written_inverted[f].append(t)
2059 except:
2060 files_written_inverted[f] = [t]
2061
2062 for f in files_written_inverted.keys():
2063 if len(files_written_inverted[f]) > 1:
2064 if not foundProblem:
2065 foundProblem = True
2066 file.write("\n")
2067 file.write("\nSome files are written by multiple tests:\n")
2068 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2069 if foundProblem:
2070 file.write("\n")
2071
2072 # -----
2073
2074 if len(files_written_not_removed) > 0:
2075 file.write("\n")
2076 file.write("\nSome files written but not removed:\n")
2077 tests = files_written_not_removed.keys()
2078 tests.sort()
2079 for t in tests:
2080 for f in files_written_not_removed[t]:
2081 file.write(" " + t + ": " + f + "\n")
2082 file.write("\n")
2083
2084 # -----
2085
2086 if len(bad_file_usages) > 0:
2087 file.write("\n")
2088 file.write("\nSome bad file usages:\n")
2089 tests = bad_file_usages.keys()
2090 tests.sort()
2091 for t in tests:
2092 for f in bad_file_usages[t]:
2093 file.write(" " + t + ": " + f + "\n")
2094 file.write("\n")
2095
2096 # -----------------------------------------------------------------------------
2097 # checking if ghostscript is available for checking the output of hp2ps
2098
2099 def genGSCmd(psfile):
2100 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2101
2102 def gsNotWorking():
2103 global gs_working
2104 print "GhostScript not available for hp2ps tests"
2105
2106 global gs_working
2107 gs_working = 0
2108 if config.have_profiling:
2109 if config.gs != '':
2110 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2111 if resultGood == 0:
2112 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2113 if resultBad != 0:
2114 print "GhostScript available for hp2ps tests"
2115 gs_working = 1;
2116 else:
2117 gsNotWorking();
2118 else:
2119 gsNotWorking();
2120 else:
2121 gsNotWorking();
2122
2123 def rm_no_fail( file ):
2124 try:
2125 os.remove( file )
2126 finally:
2127 return
2128
2129 def add_suffix( name, suffix ):
2130 if suffix == '':
2131 return name
2132 else:
2133 return name + '.' + suffix
2134
2135 def add_hs_lhs_suffix(name):
2136 if getTestOpts().c_src:
2137 return add_suffix(name, 'c')
2138 elif getTestOpts().cmm_src:
2139 return add_suffix(name, 'cmm')
2140 elif getTestOpts().objc_src:
2141 return add_suffix(name, 'm')
2142 elif getTestOpts().objcpp_src:
2143 return add_suffix(name, 'mm')
2144 elif getTestOpts().literate:
2145 return add_suffix(name, 'lhs')
2146 else:
2147 return add_suffix(name, 'hs')
2148
2149 def replace_suffix( name, suffix ):
2150 base, suf = os.path.splitext(name)
2151 return base + '.' + suffix
2152
2153 def in_testdir( name ):
2154 return (getTestOpts().testdir + '/' + name)
2155
2156 def qualify( name, suff ):
2157 return in_testdir(add_suffix(name, suff))
2158
2159
2160 # Finding the sample output. The filename is of the form
2161 #
2162 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2163 #
2164 # and we pick the most specific version available. The <version> is
2165 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2166 # more fine-grained control use if_compiler_lt().
2167 #
2168 def platform_wordsize_qualify( name, suff ):
2169
2170 basepath = qualify(name, suff)
2171
2172 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2173 for (platformSpecific, plat) in [(1, '-' + config.platform),
2174 (1, '-' + config.os),
2175 (0, '')]
2176 for ws in ['-ws-' + config.wordsize, '']
2177 for comp in ['-' + config.compiler_type, '']
2178 for vers in ['-' + config.compiler_maj_version, '']]
2179
2180 dir = glob.glob(basepath + '*')
2181 dir = map (lambda d: normalise_slashes_(d), dir)
2182
2183 for (platformSpecific, f) in paths:
2184 if f in dir:
2185 return (platformSpecific,f)
2186
2187 return (0, basepath)
2188
2189 # Clean up prior to the test, so that we can't spuriously conclude
2190 # that it passed on the basis of old run outputs.
2191 def pretest_cleanup(name):
2192 if getTestOpts().outputdir != None:
2193 odir = in_testdir(getTestOpts().outputdir)
2194 try:
2195 shutil.rmtree(odir)
2196 except:
2197 pass
2198 os.mkdir(odir)
2199
2200 rm_no_fail(qualify(name,'comp.stderr'))
2201 rm_no_fail(qualify(name,'run.stderr'))
2202 rm_no_fail(qualify(name,'run.stdout'))
2203 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2204 # simple_build zaps the following:
2205 # rm_nofail(qualify("o"))
2206 # rm_nofail(qualify(""))
2207 # not interested in the return code
2208
2209 # -----------------------------------------------------------------------------
2210 # Return a list of all the files ending in '.T' below the directory dir.
2211
2212 def findTFiles(roots):
2213 return concat(map(findTFiles_,roots))
2214
2215 def findTFiles_(path):
2216 if os.path.isdir(path):
2217 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2218 return findTFiles(paths)
2219 elif path[-2:] == '.T':
2220 return [path]
2221 else:
2222 return []
2223
2224 # -----------------------------------------------------------------------------
2225 # Output a test summary to the specified file object
2226
2227 def summary(t, file):
2228
2229 file.write('\n')
2230 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
2231 file.write('OVERALL SUMMARY for test run started at ' \
2232 + t.start_time + '\n'\
2233 + string.rjust(`t.total_tests`, 8) \
2234 + ' total tests, which gave rise to\n' \
2235 + string.rjust(`t.total_test_cases`, 8) \
2236 + ' test cases, of which\n' \
2237 + string.rjust(`t.n_framework_failures`, 8) \
2238 + ' caused framework failures\n' \
2239 + string.rjust(`t.n_tests_skipped`, 8)
2240 + ' were skipped\n\n' \
2241 + string.rjust(`t.n_expected_passes`, 8)
2242 + ' expected passes\n' \
2243 + string.rjust(`t.n_missing_libs`, 8)
2244 + ' had missing libraries\n' \
2245 + string.rjust(`t.n_expected_failures`, 8) \
2246 + ' expected failures\n' \
2247 + string.rjust(`t.n_unexpected_passes`, 8) \
2248 + ' unexpected passes\n'
2249 + string.rjust(`t.n_unexpected_failures`, 8) \
2250 + ' unexpected failures\n'
2251 + '\n')
2252
2253 if t.n_unexpected_passes > 0:
2254 file.write('Unexpected passes:\n')
2255 printPassingTestInfosSummary(file, t.unexpected_passes)
2256
2257 if t.n_unexpected_failures > 0:
2258 file.write('Unexpected failures:\n')
2259 printFailingTestInfosSummary(file, t.unexpected_failures)
2260
2261 if config.check_files_written:
2262 checkForFilesWrittenProblems(file)
2263
2264 if stopping():
2265 file.write('WARNING: Testsuite run was terminated early\n')
2266
2267 def printUnexpectedTests(file, testInfoss):
2268 unexpected = []
2269 for testInfos in testInfoss:
2270 directories = testInfos.keys()
2271 for directory in directories:
2272 tests = testInfos[directory].keys()
2273 unexpected += tests
2274 if unexpected != []:
2275 file.write('Unexpected results from:\n')
2276 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2277 file.write('\n')
2278
2279 def printPassingTestInfosSummary(file, testInfos):
2280 directories = testInfos.keys()
2281 directories.sort()
2282 maxDirLen = max(map ((lambda x : len(x)), directories))
2283 for directory in directories:
2284 tests = testInfos[directory].keys()
2285 tests.sort()
2286 for test in tests:
2287 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2288 ' (' + join(testInfos[directory][test],',') + ')\n')
2289 file.write('\n')
2290
2291 def printFailingTestInfosSummary(file, testInfos):
2292 directories = testInfos.keys()
2293 directories.sort()
2294 maxDirLen = max(map ((lambda x : len(x)), directories))
2295 for directory in directories:
2296 tests = testInfos[directory].keys()
2297 tests.sort()
2298 for test in tests:
2299 reasons = testInfos[directory][test].keys()
2300 for reason in reasons:
2301 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2302 ' [' + reason + ']' + \
2303 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2304 file.write('\n')
2305
2306 def getStdout(cmd):
2307 if have_subprocess:
2308 p = subprocess.Popen(cmd,
2309 stdout=subprocess.PIPE,
2310 stderr=subprocess.PIPE)
2311 (stdout, stderr) = p.communicate()
2312 r = p.wait()
2313 if r != 0:
2314 raise Exception("Command failed: " + str(cmd))
2315 if stderr != '':
2316 raise Exception("stderr from command: " + str(cmd))
2317 return stdout
2318 else:
2319 raise Exception("Need subprocess to get stdout, but don't have it")