Fix cabal01 on Windows
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 # This allows us to use the "with X:" syntax with python 2.5:
6 from __future__ import with_statement
7
8 import shutil
9 import sys
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import copy
16 import glob
17 import types
18 import math
19
20 have_subprocess = False
21 try:
22 import subprocess
23 have_subprocess = True
24 except:
25 print "Warning: subprocess not found, will fall back to spawnv"
26
27 from string import join
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 import thread
34
35 global wantToStop
36 wantToStop = False
37 def stopNow():
38 global wantToStop
39 wantToStop = True
40 def stopping():
41 return wantToStop
42
43 # Options valid for all the tests in the current "directory". After
44 # each test, we reset the options to these. To change the options for
45 # multiple tests, the function setTestOpts() below can be used to alter
46 # these options.
47 global thisdir_testopts
48 thisdir_testopts = TestOptions()
49
50 def getThisDirTestOpts():
51 return thisdir_testopts
52
53 # Options valid for the current test only (these get reset to
54 # testdir_testopts after each test).
55
56 global testopts_local
57 if config.use_threads:
58 testopts_local = threading.local()
59 else:
60 class TestOpts_Local:
61 pass
62 testopts_local = TestOpts_Local()
63
64 def getTestOpts():
65 return testopts_local.x
66
67 def setLocalTestOpts(opts):
68 global testopts_local
69 testopts_local.x=opts
70
71 # This can be called at the top of a file of tests, to set default test options
72 # for the following tests.
73 def setTestOpts( f ):
74 f( thisdir_testopts );
75
76 # -----------------------------------------------------------------------------
77 # Canned setup functions for common cases. eg. for a test you might say
78 #
79 # test('test001', normal, compile, [''])
80 #
81 # to run it without any options, but change it to
82 #
83 # test('test001', expect_fail, compile, [''])
84 #
85 # to expect failure for this test.
86
87 def normal( opts ):
88 return;
89
90 def skip( opts ):
91 opts.skip = 1
92
93 def expect_fail( opts ):
94 opts.expect = 'fail';
95
96 def reqlib( lib ):
97 return lambda opts, l=lib: _reqlib (opts, l )
98
99 # Cache the results of looking to see if we have a library or not.
100 # This makes quite a difference, especially on Windows.
101 have_lib = {}
102
103 def _reqlib( opts, lib ):
104 if have_lib.has_key(lib):
105 got_it = have_lib[lib]
106 else:
107 if have_subprocess:
108 # By preference we use subprocess, as the alternative uses
109 # /dev/null which mingw doesn't have.
110 p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' describe ' + lib
119 + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def expect_broken( bug ):
143 return lambda opts, b=bug: _expect_broken (opts, b )
144
145 def _expect_broken( opts, bug ):
146 opts.expect = 'fail';
147
148 def ignore_output( opts ):
149 opts.ignore_output = 1
150
151 def no_stdin( opts ):
152 opts.no_stdin = 1
153
154 def combined_output( opts ):
155 opts.combined_output = True
156
157 # -----
158
159 def expect_fail_for( ways ):
160 return lambda opts, w=ways: _expect_fail_for( opts, w )
161
162 def _expect_fail_for( opts, ways ):
163 opts.expect_fail_for = ways
164
165 def expect_broken_for( bug, ways ):
166 return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )
167
168 def _expect_broken_for( opts, bug, ways ):
169 opts.expect_fail_for = ways
170
171 # -----
172
173 def omit_ways( ways ):
174 return lambda opts, w=ways: _omit_ways( opts, w )
175
176 def _omit_ways( opts, ways ):
177 opts.omit_ways = ways
178
179 # -----
180
181 def only_ways( ways ):
182 return lambda opts, w=ways: _only_ways( opts, w )
183
184 def _only_ways( opts, ways ):
185 opts.only_ways = ways
186
187 # -----
188
189 def extra_ways( ways ):
190 return lambda opts, w=ways: _extra_ways( opts, w )
191
192 def _extra_ways( opts, ways ):
193 opts.extra_ways = ways
194
195 # -----
196
197 def omit_compiler_types( compiler_types ):
198 return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)
199
200 def _omit_compiler_types( opts, compiler_types ):
201 if config.compiler_type in compiler_types:
202 opts.skip = 1
203
204 # -----
205
206 def only_compiler_types( compiler_types ):
207 return lambda opts, c=compiler_types: _only_compiler_types(opts, c)
208
209 def _only_compiler_types( opts, compiler_types ):
210 if config.compiler_type not in compiler_types:
211 opts.skip = 1
212
213 # -----
214
215 def set_stdin( file ):
216 return lambda opts, f=file: _set_stdin(opts, f);
217
218 def _set_stdin( opts, f ):
219 opts.stdin = f
220
221 # -----
222
223 def exit_code( val ):
224 return lambda opts, v=val: _exit_code(opts, v);
225
226 def _exit_code( opts, v ):
227 opts.exit_code = v
228
229 # -----
230
231 def timeout_multiplier( val ):
232 return lambda opts, v=val: _timeout_multiplier(opts, v)
233
234 def _timeout_multiplier( opts, v ):
235 opts.timeout_multiplier = v
236
237 # -----
238
239 def extra_run_opts( val ):
240 return lambda opts, v=val: _extra_run_opts(opts, v);
241
242 def _extra_run_opts( opts, v ):
243 opts.extra_run_opts = v
244
245 # -----
246
247 def extra_hc_opts( val ):
248 return lambda opts, v=val: _extra_hc_opts(opts, v);
249
250 def _extra_hc_opts( opts, v ):
251 opts.extra_hc_opts = v
252
253 # -----
254
255 def extra_clean( files ):
256 return lambda opts, v=files: _extra_clean(opts, v);
257
258 def _extra_clean( opts, v ):
259 opts.clean_files = v
260
261 # -----
262
263 def stats_num_field( field, min, max ):
264 return lambda opts, f=field, x=min, y=max: _stats_num_field(opts, f, x, y);
265
266 def _stats_num_field( opts, f, x, y ):
267 # copy the dictionary, as the config gets shared between all tests
268 opts.stats_num_fields = opts.stats_num_fields.copy()
269 opts.stats_num_fields[f] = (x, y)
270
271 def compiler_stats_num_field( field, min, max ):
272 return lambda opts, f=field, x=min, y=max: _compiler_stats_num_field(opts, f, x, y);
273
274 def _compiler_stats_num_field( opts, f, x, y ):
275 # copy the dictionary, as the config gets shared between all tests
276 opts.compiler_stats_num_fields = opts.compiler_stats_num_fields.copy()
277 opts.compiler_stats_num_fields[f] = (x, y)
278
279 # -----
280
281 def stats_range_field( field, min, max ):
282 return lambda opts, f=field, x=min, y=max: _stats_range_field(opts, f, x, y);
283
284 def _stats_range_field( opts, f, x, y ):
285 # copy the dictionary, as the config gets shared between all tests
286 opts.stats_range_fields = opts.stats_range_fields.copy()
287 opts.stats_range_fields[f] = (x, y)
288
289 def compiler_stats_range_field( field, min, max ):
290 return lambda opts, f=field, x=min, y=max: _compiler_stats_range_field(opts, f, x, y);
291
292 def _compiler_stats_range_field( opts, f, x, y ):
293 # copy the dictionary, as the config gets shared between all tests
294 opts.compiler_stats_range_fields = opts.compiler_stats_range_fields.copy()
295 opts.compiler_stats_range_fields[f] = (x, y)
296
297 # -----
298
299 def skip_if_no_ghci(opts):
300 if not ('ghci' in config.run_ways):
301 opts.skip = 1
302
303 # ----
304
305 def skip_if_fast(opts):
306 if config.fast:
307 opts.skip = 1
308
309 # -----
310
311 def if_platform( plat, f ):
312 if config.platform == plat:
313 return f
314 else:
315 return normal
316
317 def unless_platform( plat, f ):
318 if config.platform != plat:
319 return f
320 else:
321 return normal
322
323 def if_os( os, f ):
324 if config.os == os:
325 return f
326 else:
327 return normal
328
329 def unless_os( os, f ):
330 if config.os == os:
331 return normal
332 else:
333 return f
334
335 def if_arch( arch, f ):
336 if config.arch == arch:
337 return f
338 else:
339 return normal
340
341 def unless_arch( arch, f ):
342 if config.arch == arch:
343 return normal
344 else:
345 return f
346
347 def if_wordsize( ws, f ):
348 if config.wordsize == str(ws):
349 return f
350 else:
351 return normal
352
353 def unless_wordsize( ws, f ):
354 if config.wordsize == str(ws):
355 return normal
356 else:
357 return f
358
359 def if_unregisterised( f ):
360 if config.unregisterised:
361 return f
362 else:
363 return normal
364
365 def unless_unregisterised( f ):
366 if config.unregisterised:
367 return normal
368 else:
369 return f
370
371 def if_msys( f ):
372 if config.msys:
373 return f
374 else:
375 return normal
376
377 def if_cygwin( f ):
378 if config.cygwin:
379 return f
380 else:
381 return normal
382
383 def when_have_vanilla( f ):
384 if config.have_vanilla:
385 return f
386 else:
387 return normal
388
389 def unless_have_vanilla( f ):
390 if config.have_vanilla:
391 return normal
392 else:
393 return f
394
395 def when_have_dynamic( f ):
396 if config.have_dynamic:
397 return f
398 else:
399 return normal
400
401 def unless_have_dynamic( f ):
402 if config.have_dynamic:
403 return normal
404 else:
405 return f
406
407 def when_have_profiling( f ):
408 if config.have_profiling:
409 return f
410 else:
411 return normal
412
413 def unless_have_profiling( f ):
414 if config.have_profiling:
415 return normal
416 else:
417 return f
418
419 # ---
420
421 def if_ghci_dynamic( f ):
422 if config.ghc_dynamic_by_default:
423 return f
424 else:
425 return normal
426
427 def if_in_tree_compiler( f ):
428 if config.in_tree_compiler:
429 return f
430 else:
431 return normal
432
433 def unless_in_tree_compiler( f ):
434 if config.in_tree_compiler:
435 return normal
436 else:
437 return f
438
439 def if_compiler_type( compiler, f ):
440 if config.compiler_type == compiler:
441 return f
442 else:
443 return normal
444
445 def if_compiler_profiled( f ):
446 if config.compiler_profiled:
447 return f
448 else:
449 return normal
450
451 def unless_compiler_profiled( f ):
452 if config.compiler_profiled:
453 return normal
454 else:
455 return f
456
457 def if_compiler_lt( compiler, version, f ):
458 if config.compiler_type == compiler and \
459 version_lt(config.compiler_version, version):
460 return f
461 else:
462 return normal
463
464 def if_compiler_le( compiler, version, f ):
465 if config.compiler_type == compiler and \
466 version_le(config.compiler_version, version):
467 return f
468 else:
469 return normal
470
471 def if_compiler_gt( compiler, version, f ):
472 if config.compiler_type == compiler and \
473 version_gt(config.compiler_version, version):
474 return f
475 else:
476 return normal
477
478 def if_compiler_ge( compiler, version, f ):
479 if config.compiler_type == compiler and \
480 version_ge(config.compiler_version, version):
481 return f
482 else:
483 return normal
484
485 def if_compiler_debugged( f ):
486 if config.compiler_debugged:
487 return f
488 else:
489 return normal
490
491 def namebase( nb ):
492 return lambda opts, nb=nb: _namebase(opts, nb)
493
494 def _namebase( opts, nb ):
495 opts.with_namebase = nb
496
497 # ---
498
499 def if_tag( tag, f ):
500 if tag in config.compiler_tags:
501 return f
502 else:
503 return normal
504
505 def unless_tag( tag, f ):
506 if not (tag in config.compiler_tags):
507 return f
508 else:
509 return normal
510
511 # ---
512 def high_memory_usage(opts):
513 opts.alone = True
514
515 # ---
516 def literate( opts ):
517 opts.literate = 1;
518
519 def c_src( opts ):
520 opts.c_src = 1;
521
522 def objc_src( opts ):
523 opts.objc_src = 1;
524
525 def objcpp_src( opts ):
526 opts.objcpp_src = 1;
527
528 def cmm_src( opts ):
529 opts.cmm_src = 1;
530
531 def outputdir( odir ):
532 return lambda opts, d=odir: _outputdir(opts, d)
533
534 def _outputdir( opts, odir ):
535 opts.outputdir = odir;
536
537 # ----
538
539 def pre_cmd( cmd ):
540 return lambda opts, c=cmd: _pre_cmd(opts, cmd)
541
542 def _pre_cmd( opts, cmd ):
543 opts.pre_cmd = cmd
544
545 # ----
546
547 def clean_cmd( cmd ):
548 return lambda opts, c=cmd: _clean_cmd(opts, cmd)
549
550 def _clean_cmd( opts, cmd ):
551 opts.clean_cmd = cmd
552
553 # ----
554
555 def cmd_prefix( prefix ):
556 return lambda opts, p=prefix: _cmd_prefix(opts, prefix)
557
558 def _cmd_prefix( opts, prefix ):
559 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
560
561 # ----
562
563 def cmd_wrapper( fun ):
564 return lambda opts, f=fun: _cmd_wrapper(opts, fun)
565
566 def _cmd_wrapper( opts, fun ):
567 opts.cmd_wrapper = fun
568
569 # ----
570
571 def compile_cmd_prefix( prefix ):
572 return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)
573
574 def _compile_cmd_prefix( opts, prefix ):
575 opts.compile_cmd_prefix = prefix
576
577 # ----
578
579 def normalise_slashes( opts ):
580 opts.extra_normaliser = normalise_slashes_
581
582 def normalise_exe( opts ):
583 opts.extra_normaliser = normalise_exe_
584
585 def normalise_fun( fun ):
586 return lambda opts, f=fun: _normalise_fun(opts, f)
587
588 def _normalise_fun( opts, f ):
589 opts.extra_normaliser = f
590
591 def normalise_errmsg_fun( fun ):
592 return lambda opts, f=fun: _normalise_errmsg_fun(opts, f)
593
594 def _normalise_errmsg_fun( opts, f ):
595 opts.extra_errmsg_normaliser = f
596
597 def two_normalisers(f, g):
598 return lambda x, f=f, g=g: f(g(x))
599
600 # ----
601 # Function for composing two opt-fns together
602
603 def composes( fs ):
604 return reduce(lambda f, g: compose(f, g), fs)
605
606 def compose( f, g ):
607 return lambda opts, f=f, g=g: _compose(opts,f,g)
608
609 def _compose( opts, f, g ):
610 f(opts)
611 g(opts)
612
613 # -----------------------------------------------------------------------------
614 # The current directory of tests
615
616 def newTestDir( dir ):
617 global thisdir_testopts
618 # reset the options for this test directory
619 thisdir_testopts = copy.copy(default_testopts)
620 thisdir_testopts.testdir = dir
621 thisdir_testopts.compiler_always_flags = config.compiler_always_flags
622
623 # -----------------------------------------------------------------------------
624 # Actually doing tests
625
626 parallelTests = []
627 aloneTests = []
628 allTestNames = set([])
629
630 def runTest (opts, name, func, args):
631 ok = 0
632
633 if config.use_threads:
634 t.thread_pool.acquire()
635 try:
636 while config.threads<(t.running_threads+1):
637 t.thread_pool.wait()
638 t.running_threads = t.running_threads+1
639 ok=1
640 t.thread_pool.release()
641 thread.start_new_thread(test_common_thread, (name, opts, func, args))
642 except:
643 if not ok:
644 t.thread_pool.release()
645 else:
646 test_common_work (name, opts, func, args)
647
648 # name :: String
649 # setup :: TestOpts -> IO ()
650 def test (name, setup, func, args):
651 global aloneTests
652 global parallelTests
653 global allTestNames
654 if name in allTestNames:
655 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
656 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
657 framework_fail(name, 'bad_name', 'This test has an invalid name')
658 myTestOpts = copy.copy(thisdir_testopts)
659
660 if type(setup) is types.ListType:
661 setup = composes(setup)
662
663 setup(myTestOpts)
664
665 thisTest = lambda : runTest(myTestOpts, name, func, args)
666 if myTestOpts.alone:
667 aloneTests.append(thisTest)
668 else:
669 parallelTests.append(thisTest)
670 allTestNames.add(name)
671
672 if config.use_threads:
673 def test_common_thread(name, opts, func, args):
674 t.lock.acquire()
675 try:
676 test_common_work(name,opts,func,args)
677 finally:
678 t.lock.release()
679 t.thread_pool.acquire()
680 t.running_threads = t.running_threads - 1
681 t.thread_pool.notify()
682 t.thread_pool.release()
683
684 def get_package_cache_timestamp():
685 if config.package_conf_cache_file == '':
686 return 0.0
687 else:
688 try:
689 return os.stat(config.package_conf_cache_file).st_mtime
690 except:
691 return 0.0
692
693
694 def test_common_work (name, opts, func, args):
695 try:
696 t.total_tests = t.total_tests+1
697 setLocalTestOpts(opts)
698
699 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
700
701 # All the ways we might run this test
702 if func == compile or func == multimod_compile:
703 all_ways = config.compile_ways
704 elif func == compile_and_run or func == multimod_compile_and_run:
705 all_ways = config.run_ways
706 elif func == ghci_script:
707 if 'ghci' in config.run_ways:
708 all_ways = ['ghci']
709 else:
710 all_ways = []
711 else:
712 all_ways = ['normal']
713
714 # A test itself can request extra ways by setting opts.extra_ways
715 all_ways = all_ways + filter(lambda way: way not in all_ways,
716 opts.extra_ways)
717
718 t.total_test_cases = t.total_test_cases + len(all_ways)
719
720 ok_way = lambda way: \
721 not getTestOpts().skip \
722 and (config.only == [] or name in config.only) \
723 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
724 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
725 and way not in getTestOpts().omit_ways
726
727 # Which ways we are asked to skip
728 do_ways = filter (ok_way,all_ways)
729
730 # In fast mode, we skip all but one way
731 if config.fast and len(do_ways) > 0:
732 do_ways = [do_ways[0]]
733
734 if not config.clean_only:
735 # Run the required tests...
736 for way in do_ways:
737 if stopping():
738 break
739 do_test (name, way, func, args)
740
741 for way in all_ways:
742 if way not in do_ways:
743 skiptest (name,way)
744
745 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
746 clean(map (lambda suff: name + suff,
747 ['', '.exe', '.exe.manifest', '.genscript',
748 '.stderr.normalised', '.stdout.normalised',
749 '.run.stderr', '.run.stdout',
750 '.run.stderr.normalised', '.run.stdout.normalised',
751 '.comp.stderr', '.comp.stdout',
752 '.comp.stderr.normalised', '.comp.stdout.normalised',
753 '.interp.stderr', '.interp.stdout',
754 '.interp.stderr.normalised', '.interp.stdout.normalised',
755 '.stats', '.comp.stats',
756 '.hi', '.o', '.prof', '.exe.prof', '.hc',
757 '_stub.h', '_stub.c', '_stub.o',
758 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
759
760 if func == multi_compile or func == multi_compile_fail:
761 extra_mods = args[1]
762 clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
763 clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))
764
765 clean(getTestOpts().clean_files)
766
767 if getTestOpts().outputdir != None:
768 odir = in_testdir(getTestOpts().outputdir)
769 try:
770 shutil.rmtree(odir)
771 except:
772 pass
773
774 try:
775 shutil.rmtree(in_testdir('.hpc.' + name))
776 except:
777 pass
778
779 try:
780 cleanCmd = getTestOpts().clean_cmd
781 if cleanCmd != None:
782 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
783 if result != 0:
784 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
785 except e:
786 framework_fail(name, 'cleaning', 'clean-command exception')
787
788 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
789
790 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
791 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
792
793 try:
794 for f in files_written[name]:
795 if os.path.exists(f):
796 try:
797 if not f in files_written_not_removed[name]:
798 files_written_not_removed[name].append(f)
799 except:
800 files_written_not_removed[name] = [f]
801 except:
802 pass
803 except Exception, e:
804 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
805
806 def clean(strs):
807 for str in strs:
808 for name in glob.glob(in_testdir(str)):
809 clean_full_path(name)
810
811 def clean_full_path(name):
812 try:
813 # Remove files...
814 os.remove(name)
815 except OSError, e1:
816 try:
817 # ... and empty directories
818 os.rmdir(name)
819 except OSError, e2:
820 # We don't want to fail here, but we do want to know
821 # what went wrong, so print out the exceptions.
822 # ENOENT isn't a problem, though, as we clean files
823 # that don't necessarily exist.
824 if e1.errno != errno.ENOENT:
825 print e1
826 if e2.errno != errno.ENOENT:
827 print e2
828
829 def do_test(name, way, func, args):
830 full_name = name + '(' + way + ')'
831
832 try:
833 print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
834 str([t.n_unexpected_passes, \
835 t.n_unexpected_failures, \
836 t.n_framework_failures])
837
838 if config.use_threads:
839 t.lock.release()
840
841 try:
842 preCmd = getTestOpts().pre_cmd
843 if preCmd != None:
844 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
845 if result != 0:
846 framework_fail(name, way, 'pre-command failed: ' + str(result))
847 except e:
848 framework_fail(name, way, 'pre-command exception')
849
850 try:
851 result = apply(func, [name,way] + args)
852 finally:
853 if config.use_threads:
854 t.lock.acquire()
855
856 if getTestOpts().expect != 'pass' and \
857 getTestOpts().expect != 'fail' and \
858 getTestOpts().expect != 'missing-lib':
859 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
860
861 try:
862 passFail = result['passFail']
863 except:
864 passFail = 'No passFail found'
865
866 if passFail == 'pass':
867 if getTestOpts().expect == 'pass' \
868 and way not in getTestOpts().expect_fail_for:
869 t.n_expected_passes = t.n_expected_passes + 1
870 if name in t.expected_passes:
871 t.expected_passes[name].append(way)
872 else:
873 t.expected_passes[name] = [way]
874 else:
875 print '*** unexpected pass for', full_name
876 t.n_unexpected_passes = t.n_unexpected_passes + 1
877 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
878 elif passFail == 'fail':
879 if getTestOpts().expect == 'pass' \
880 and way not in getTestOpts().expect_fail_for:
881 print '*** unexpected failure for', full_name
882 t.n_unexpected_failures = t.n_unexpected_failures + 1
883 reason = result['reason']
884 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
885 else:
886 if getTestOpts().expect == 'missing-lib':
887 t.n_missing_libs = t.n_missing_libs + 1
888 if name in t.missing_libs:
889 t.missing_libs[name].append(way)
890 else:
891 t.missing_libs[name] = [way]
892 else:
893 t.n_expected_failures = t.n_expected_failures + 1
894 if name in t.expected_failures:
895 t.expected_failures[name].append(way)
896 else:
897 t.expected_failures[name] = [way]
898 else:
899 framework_fail(name, way, 'bad result ' + passFail)
900 except KeyboardInterrupt:
901 stopNow()
902 except:
903 framework_fail(name, way, 'do_test exception')
904 traceback.print_exc()
905
906 def addPassingTestInfo (testInfos, directory, name, way):
907 directory = re.sub('^\\.[/\\\\]', '', directory)
908
909 if not directory in testInfos:
910 testInfos[directory] = {}
911
912 if not name in testInfos[directory]:
913 testInfos[directory][name] = []
914
915 testInfos[directory][name].append(way)
916
917 def addFailingTestInfo (testInfos, directory, name, reason, way):
918 directory = re.sub('^\\.[/\\\\]', '', directory)
919
920 if not directory in testInfos:
921 testInfos[directory] = {}
922
923 if not name in testInfos[directory]:
924 testInfos[directory][name] = {}
925
926 if not reason in testInfos[directory][name]:
927 testInfos[directory][name][reason] = []
928
929 testInfos[directory][name][reason].append(way)
930
931 def skiptest (name, way):
932 # print 'Skipping test \"', name, '\"'
933 t.n_tests_skipped = t.n_tests_skipped + 1
934 if name in t.tests_skipped:
935 t.tests_skipped[name].append(way)
936 else:
937 t.tests_skipped[name] = [way]
938
939 def framework_fail( name, way, reason ):
940 full_name = name + '(' + way + ')'
941 print '*** framework failure for', full_name, reason
942 t.n_framework_failures = t.n_framework_failures + 1
943 if name in t.framework_failures:
944 t.framework_failures[name].append(way)
945 else:
946 t.framework_failures[name] = [way]
947
948 def badResult(result):
949 try:
950 if result['passFail'] == 'pass':
951 return False
952 return True
953 except:
954 return True
955
956 def passed():
957 return {'passFail': 'pass'}
958
959 def failBecause(reason):
960 return {'passFail': 'fail', 'reason': reason}
961
962 # -----------------------------------------------------------------------------
963 # Generic command tests
964
965 # A generic command test is expected to run and exit successfully.
966 #
967 # The expected exit code can be changed via exit_code() as normal, and
968 # the expected stdout/stderr are stored in <testname>.stdout and
969 # <testname>.stderr. The output of the command can be ignored
970 # altogether by using run_command_ignore_output instead of
971 # run_command.
972
973 def run_command( name, way, cmd ):
974 return simple_run( name, '', cmd, '' )
975
976 # -----------------------------------------------------------------------------
977 # GHCi tests
978
979 def ghci_script( name, way, script ):
980 # filter out -fforce-recomp from compiler_always_flags, because we're
981 # actually testing the recompilation behaviour in the GHCi tests.
982 flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
983 flags.append(getTestOpts().extra_hc_opts)
984 if getTestOpts().outputdir != None:
985 flags.extend(["-outputdir", getTestOpts().outputdir])
986
987 # We pass HC and HC_OPTS as environment variables, so that the
988 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
989 cmd = "HC='" + config.compiler + "' " + \
990 "HC_OPTS='" + join(flags,' ') + "' " + \
991 "'" + config.compiler + "'" + \
992 ' --interactive -v0 -ignore-dot-ghci ' + \
993 join(flags,' ')
994
995 getTestOpts().stdin = script
996 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
997
998 # -----------------------------------------------------------------------------
999 # Compile-only tests
1000
1001 def compile( name, way, extra_hc_opts ):
1002 return do_compile( name, way, 0, '', [], extra_hc_opts )
1003
1004 def compile_fail( name, way, extra_hc_opts ):
1005 return do_compile( name, way, 1, '', [], extra_hc_opts )
1006
1007 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1008 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1009
1010 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1011 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1012
1013 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1014 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1015
1016 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1017 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1018
1019 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1020 # print 'Compile only, extra args = ', extra_hc_opts
1021 pretest_cleanup(name)
1022
1023 result = extras_build( way, extra_mods, extra_hc_opts )
1024 if badResult(result):
1025 return result
1026 extra_hc_opts = result['hc_opts']
1027
1028 force = 0
1029 if extra_mods:
1030 force = 1
1031 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)
1032
1033 if badResult(result):
1034 return result
1035
1036 # the actual stderr should always match the expected, regardless
1037 # of whether we expected the compilation to fail or not (successful
1038 # compilations may generate warnings).
1039
1040 if getTestOpts().with_namebase == None:
1041 namebase = name
1042 else:
1043 namebase = getTestOpts().with_namebase
1044
1045 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1046 actual_stderr_file = qualify(name, 'comp.stderr')
1047
1048 if not compare_outputs('stderr', \
1049 two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
1050 expected_stderr_file, actual_stderr_file):
1051 return failBecause('stderr mismatch')
1052
1053 # no problems found, this test passed
1054 return passed()
1055
1056 def compile_cmp_asm( name, way, extra_hc_opts ):
1057 print 'Compile only, extra args = ', extra_hc_opts
1058 pretest_cleanup(name)
1059 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1060
1061 if badResult(result):
1062 return result
1063
1064 # the actual stderr should always match the expected, regardless
1065 # of whether we expected the compilation to fail or not (successful
1066 # compilations may generate warnings).
1067
1068 if getTestOpts().with_namebase == None:
1069 namebase = name
1070 else:
1071 namebase = getTestOpts().with_namebase
1072
1073 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1074 actual_asm_file = qualify(name, 's')
1075
1076 if not compare_outputs('asm', two_normalisers(normalise_errmsg, normalise_asm), \
1077 expected_asm_file, actual_asm_file):
1078 return failBecause('asm mismatch')
1079
1080 # no problems found, this test passed
1081 return passed()
1082
1083 # -----------------------------------------------------------------------------
1084 # Compile-and-run tests
1085
1086 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1087 # print 'Compile and run, extra args = ', extra_hc_opts
1088 pretest_cleanup(name)
1089
1090 result = extras_build( way, extra_mods, extra_hc_opts )
1091 if badResult(result):
1092 return result
1093 extra_hc_opts = result['hc_opts']
1094
1095 if way == 'ghci': # interpreted...
1096 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1097 elif way == 'extcore' or way == 'optextcore' :
1098 return extcore_run( name, way, extra_hc_opts, 0, top_mod )
1099 else: # compiled...
1100 force = 0
1101 if extra_mods:
1102 force = 1
1103
1104 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1105 if badResult(result):
1106 return result
1107
1108 cmd = './' + name;
1109
1110 # we don't check the compiler's stderr for a compile-and-run test
1111 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1112
1113 def compile_and_run( name, way, extra_hc_opts ):
1114 return compile_and_run__( name, way, '', [], extra_hc_opts)
1115
1116 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1117 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1118
1119 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1120 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1121
1122 def stats( name, way, stats_file ):
1123 opts = getTestOpts()
1124 return checkStats(stats_file, opts.stats_range_fields
1125 , opts.stats_num_fields)
1126
1127 # -----------------------------------------------------------------------------
1128 # Check -t stats info
1129
1130 def checkStats(stats_file, range_fields, num_fields):
1131 result = passed()
1132 if len(num_fields) + len(range_fields) > 0:
1133 f = open(in_testdir(stats_file))
1134 contents = f.read()
1135 f.close()
1136
1137 for (field, (expected, dev)) in range_fields.items():
1138 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1139 if m == None:
1140 print 'Failed to find field: ', field
1141 result = failBecause('no such stats field')
1142 val = int(m.group(1))
1143
1144 min = expected * ((100 - float(dev))/100);
1145 max = expected * ((100 + float(dev))/100);
1146
1147 if val < min:
1148 print field, val, 'is more than ' + repr(dev) + '%'
1149 print 'less than the exepected value', expected
1150 print 'If this is because you have improved GHC, please'
1151 print 'update the test so that GHC doesn\'t regress again'
1152 result = failBecause('stat too good')
1153 if val > max:
1154 print field, val, 'is more than ' + repr(dev) + '% greater than the expected value,', expected, max
1155 result = failBecause('stat not good enough')
1156
1157 # ToDo: remove all uses of this, and delete it
1158 for (field, (min, max)) in num_fields.items():
1159 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1160 if m == None:
1161 print 'Failed to find field: ', field
1162 result = failBecause('no such stats field')
1163 val = int(m.group(1))
1164
1165 if val < min:
1166 print field, val, 'is less than minimum allowed', min
1167 print 'If this is because you have improved GHC, please'
1168 print 'update the test so that GHC doesn\'t regress again'
1169 result = failBecause('stat too good')
1170 if val > max:
1171 print field, val, 'is more than maximum allowed', max
1172 result = failBecause('stat not good enough')
1173
1174 return result
1175
1176 # -----------------------------------------------------------------------------
1177 # Build a single-module program
1178
1179 def extras_build( way, extra_mods, extra_hc_opts ):
1180 for modopts in extra_mods:
1181 mod, opts = modopts
1182 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1183 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1184 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1185 if badResult(result):
1186 return result
1187
1188 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1189
1190
1191 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
1192 opts = getTestOpts()
1193 errname = add_suffix(name, 'comp.stderr')
1194 rm_no_fail( qualify(errname, '') )
1195
1196 if top_mod != '':
1197 srcname = top_mod
1198 rm_no_fail( qualify(name, '') )
1199 base, suf = os.path.splitext(top_mod)
1200 rm_no_fail( qualify(base, '') )
1201 rm_no_fail( qualify(base, 'exe') )
1202 elif addsuf:
1203 srcname = add_hs_lhs_suffix(name)
1204 rm_no_fail( qualify(name, '') )
1205 else:
1206 srcname = name
1207 rm_no_fail( qualify(name, 'o') )
1208
1209 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1210
1211 to_do = ''
1212 if top_mod != '':
1213 to_do = '--make '
1214 if link:
1215 to_do = to_do + '-o ' + name
1216 elif link:
1217 to_do = '-o ' + name
1218 elif opts.compile_to_hc:
1219 to_do = '-C'
1220 else:
1221 to_do = '-c' # just compile
1222
1223 stats_file = name + '.comp.stats'
1224 if len(opts.compiler_stats_num_fields) + len(opts.compiler_stats_range_fields) > 0:
1225 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1226
1227 # Required by GHC 7.3+, harmless for earlier versions:
1228 if (getTestOpts().c_src or
1229 getTestOpts().objc_src or
1230 getTestOpts().objcpp_src or
1231 getTestOpts().cmm_src):
1232 extra_hc_opts += ' -no-hs-main '
1233
1234 if getTestOpts().compile_cmd_prefix == '':
1235 cmd_prefix = ''
1236 else:
1237 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1238
1239 comp_flags = copy.copy(getTestOpts().compiler_always_flags)
1240 if noforce:
1241 comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)
1242 if getTestOpts().outputdir != None:
1243 comp_flags.extend(["-outputdir", getTestOpts().outputdir])
1244
1245 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1246 + config.compiler + "' " \
1247 + join(comp_flags,' ') + ' ' \
1248 + to_do + ' ' + srcname + ' ' \
1249 + join(config.way_flags(name)[way],' ') + ' ' \
1250 + extra_hc_opts + ' ' \
1251 + opts.extra_hc_opts + ' ' \
1252 + '>' + errname + ' 2>&1'
1253
1254 result = runCmdFor(name, cmd)
1255
1256 if result != 0 and not should_fail:
1257 actual_stderr = qualify(name, 'comp.stderr')
1258 if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
1259 if_verbose_dump(1,actual_stderr)
1260
1261 # ToDo: if the sub-shell was killed by ^C, then exit
1262
1263 statsResult = checkStats(stats_file, opts.compiler_stats_range_fields
1264 , opts.compiler_stats_num_fields)
1265
1266 if badResult(statsResult):
1267 return statsResult
1268
1269 if should_fail:
1270 if result == 0:
1271 return failBecause('exit code 0')
1272 else:
1273 if result != 0:
1274 return failBecause('exit code non-0')
1275
1276 return passed()
1277
1278 # -----------------------------------------------------------------------------
1279 # Run a program and check its output
1280 #
1281 # If testname.stdin exists, route input from that, else
1282 # from /dev/null. Route output to testname.run.stdout and
1283 # testname.run.stderr. Returns the exit code of the run.
1284
1285 def simple_run( name, way, prog, args ):
1286 opts = getTestOpts()
1287
1288 # figure out what to use for stdin
1289 if opts.stdin != '':
1290 use_stdin = opts.stdin
1291 else:
1292 stdin_file = add_suffix(name, 'stdin')
1293 if os.path.exists(in_testdir(stdin_file)):
1294 use_stdin = stdin_file
1295 else:
1296 use_stdin = '/dev/null'
1297
1298 run_stdout = add_suffix(name,'run.stdout')
1299 run_stderr = add_suffix(name,'run.stderr')
1300
1301 rm_no_fail(qualify(name,'run.stdout'))
1302 rm_no_fail(qualify(name,'run.stderr'))
1303 rm_no_fail(qualify(name, 'hp'))
1304 rm_no_fail(qualify(name,'ps'))
1305 rm_no_fail(qualify(name, 'prof'))
1306
1307 my_rts_flags = rts_flags(way)
1308
1309 stats_file = name + '.stats'
1310 if len(opts.stats_num_fields) + len(opts.stats_range_fields) > 0:
1311 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1312
1313 if opts.no_stdin:
1314 stdin_comes_from = ''
1315 else:
1316 stdin_comes_from = ' <' + use_stdin
1317
1318 if opts.combined_output:
1319 redirection = ' >' + run_stdout \
1320 + ' 2>&1'
1321 else:
1322 redirection = ' >' + run_stdout \
1323 + ' 2>' + run_stderr
1324
1325 cmd = prog + ' ' + args + ' ' \
1326 + my_rts_flags + ' ' \
1327 + stdin_comes_from \
1328 + redirection
1329
1330 if opts.cmd_wrapper != None:
1331 cmd = opts.cmd_wrapper(cmd);
1332
1333 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1334
1335 # run the command
1336 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1337
1338 exit_code = result >> 8
1339 signal = result & 0xff
1340
1341 # check the exit code
1342 if exit_code != opts.exit_code:
1343 print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
1344 dump_stdout(name)
1345 dump_stderr(name)
1346 return failBecause('bad exit code')
1347
1348 check_hp = my_rts_flags.find("-h") != -1
1349 check_prof = my_rts_flags.find("-p") != -1
1350
1351 if not opts.ignore_output:
1352 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1353 bad_stdout = not check_stdout_ok(name)
1354 if bad_stderr:
1355 return failBecause('bad stderr')
1356 if bad_stdout:
1357 return failBecause('bad stdout')
1358 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1359 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1360 return failBecause('bad heap profile')
1361 if check_prof and not check_prof_ok(name):
1362 return failBecause('bad profile')
1363
1364 return checkStats(stats_file, opts.stats_range_fields
1365 , opts.stats_num_fields)
1366
1367 def rts_flags(way):
1368 if (way == ''):
1369 return ''
1370 else:
1371 args = config.way_rts_flags[way]
1372
1373 if args == []:
1374 return ''
1375 else:
1376 return '+RTS ' + join(args,' ') + ' -RTS'
1377
1378 # -----------------------------------------------------------------------------
1379 # Run a program in the interpreter and check its output
1380
1381 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1382 outname = add_suffix(name, 'interp.stdout')
1383 errname = add_suffix(name, 'interp.stderr')
1384 rm_no_fail(outname)
1385 rm_no_fail(errname)
1386 rm_no_fail(name)
1387
1388 if (top_mod == ''):
1389 srcname = add_hs_lhs_suffix(name)
1390 else:
1391 srcname = top_mod
1392
1393 scriptname = add_suffix(name, 'genscript')
1394 qscriptname = in_testdir(scriptname)
1395 rm_no_fail(qscriptname)
1396
1397 delimiter = '===== program output begins here\n'
1398
1399 script = open(qscriptname, 'w')
1400 if not compile_only:
1401 # set the prog name and command-line args to match the compiled
1402 # environment.
1403 script.write(':set prog ' + name + '\n')
1404 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1405 # Add marker lines to the stdout and stderr output files, so we
1406 # can separate GHCi's output from the program's.
1407 script.write(':! echo ' + delimiter)
1408 script.write(':! echo 1>&2 ' + delimiter)
1409 # Set stdout to be line-buffered to match the compiled environment.
1410 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1411 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1412 # in the event of an exception as for the compiled program.
1413 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1414 script.close()
1415
1416 # figure out what to use for stdin
1417 if getTestOpts().stdin != '':
1418 stdin_file = in_testdir(getTestOpts().stdin)
1419 else:
1420 stdin_file = qualify(name, 'stdin')
1421
1422 if os.path.exists(stdin_file):
1423 stdin = open(stdin_file, 'r')
1424 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1425
1426 script.close()
1427
1428 flags = copy.copy(getTestOpts().compiler_always_flags)
1429 if getTestOpts().outputdir != None:
1430 flags.extend(["-outputdir", getTestOpts().outputdir])
1431
1432 cmd = "'" + config.compiler + "' " \
1433 + join(flags,' ') + ' ' \
1434 + srcname + ' ' \
1435 + join(config.way_flags(name)[way],' ') + ' ' \
1436 + extra_hc_opts + ' ' \
1437 + getTestOpts().extra_hc_opts + ' ' \
1438 + '<' + scriptname + ' 1>' + outname + ' 2>' + errname
1439
1440 if getTestOpts().cmd_wrapper != None:
1441 cmd = getTestOpts().cmd_wrapper(cmd);
1442
1443 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1444
1445 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1446
1447 exit_code = result >> 8
1448 signal = result & 0xff
1449
1450 # split the stdout into compilation/program output
1451 split_file(in_testdir(outname), delimiter,
1452 qualify(name, 'comp.stdout'),
1453 qualify(name, 'run.stdout'))
1454 split_file(in_testdir(errname), delimiter,
1455 qualify(name, 'comp.stderr'),
1456 qualify(name, 'run.stderr'))
1457
1458 # check the exit code
1459 if exit_code != getTestOpts().exit_code:
1460 print 'Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')'
1461 dump_stdout(name)
1462 dump_stderr(name)
1463 return failBecause('bad exit code')
1464
1465 # ToDo: if the sub-shell was killed by ^C, then exit
1466
1467 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1468 check_stdout_ok(name)):
1469 return passed()
1470 else:
1471 return failBecause('bad stdout or stderr')
1472
1473
1474 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1475 infile = open(in_fn)
1476 out1 = open(out1_fn, 'w')
1477 out2 = open(out2_fn, 'w')
1478
1479 line = infile.readline()
1480 line = re.sub('\r', '', line) # ignore Windows EOL
1481 while (re.sub('^\s*','',line) != delimiter and line != ''):
1482 out1.write(line)
1483 line = infile.readline()
1484 line = re.sub('\r', '', line)
1485 out1.close()
1486
1487 line = infile.readline()
1488 while (line != ''):
1489 out2.write(line)
1490 line = infile.readline()
1491 out2.close()
1492
1493 # -----------------------------------------------------------------------------
1494 # Generate External Core for the given program, then compile the resulting Core
1495 # and compare its output to the expected output
1496
1497 def extcore_run( name, way, extra_hc_opts, compile_only, top_mod ):
1498
1499 depsfilename = qualify(name, 'deps')
1500 errname = add_suffix(name, 'comp.stderr')
1501 qerrname = qualify(errname,'')
1502
1503 hcname = qualify(name, 'hc')
1504 oname = qualify(name, 'o')
1505
1506 rm_no_fail( qerrname )
1507 rm_no_fail( qualify(name, '') )
1508
1509 if (top_mod == ''):
1510 srcname = add_hs_lhs_suffix(name)
1511 else:
1512 srcname = top_mod
1513
1514 qcorefilename = qualify(name, 'hcr')
1515 corefilename = add_suffix(name, 'hcr')
1516 rm_no_fail(qcorefilename)
1517
1518 # Generate External Core
1519
1520 if (top_mod == ''):
1521 to_do = ' ' + srcname + ' '
1522 else:
1523 to_do = ' --make ' + top_mod + ' '
1524
1525 flags = copy.copy(getTestOpts().compiler_always_flags)
1526 if getTestOpts().outputdir != None:
1527 flags.extend(["-outputdir", getTestOpts().outputdir])
1528 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1529 + config.compiler + "' " \
1530 + join(flags,' ') + ' ' \
1531 + join(config.way_flags(name)[way],' ') + ' ' \
1532 + extra_hc_opts + ' ' \
1533 + getTestOpts().extra_hc_opts \
1534 + to_do \
1535 + '>' + errname + ' 2>&1'
1536 result = runCmdFor(name, cmd)
1537
1538 exit_code = result >> 8
1539
1540 if exit_code != 0:
1541 if_verbose(1,'Compiling to External Core failed (status ' + `result` + ') errors were:')
1542 if_verbose_dump(1,qerrname)
1543 return failBecause('ext core exit code non-0')
1544
1545 # Compile the resulting files -- if there's more than one module, we need to read the output
1546 # of the previous compilation in order to find the dependencies
1547 if (top_mod == ''):
1548 to_compile = corefilename
1549 else:
1550 result = runCmdFor(name, 'grep Compiling ' + qerrname + ' | awk \'{print $4}\' > ' + depsfilename)
1551 deps = open(depsfilename).read()
1552 deplist = string.replace(deps, '\n',' ');
1553 deplist2 = string.replace(deplist,'.lhs,', '.hcr');
1554 to_compile = string.replace(deplist2,'.hs,', '.hcr');
1555
1556 flags = join(filter(lambda f: f != '-fext-core',config.way_flags(name)[way]),' ')
1557 if getTestOpts().outputdir != None:
1558 flags.extend(["-outputdir", getTestOpts().outputdir])
1559
1560 cmd = 'cd ' + getTestOpts().testdir + " && '" \
1561 + config.compiler + "' " \
1562 + join(getTestOpts().compiler_always_flags,' ') + ' ' \
1563 + to_compile + ' ' \
1564 + extra_hc_opts + ' ' \
1565 + getTestOpts().extra_hc_opts + ' ' \
1566 + flags \
1567 + ' -fglasgow-exts -o ' + name \
1568 + '>' + errname + ' 2>&1'
1569
1570 result = runCmdFor(name, cmd)
1571 exit_code = result >> 8
1572
1573 if exit_code != 0:
1574 if_verbose(1,'Compiling External Core file(s) failed (status ' + `result` + ') errors were:')
1575 if_verbose_dump(1,qerrname)
1576 return failBecause('ext core exit code non-0')
1577
1578 # Clean up
1579 rm_no_fail ( oname )
1580 rm_no_fail ( hcname )
1581 rm_no_fail ( qcorefilename )
1582 rm_no_fail ( depsfilename )
1583
1584 return simple_run ( name, way, './'+name, getTestOpts().extra_run_opts )
1585
1586 # -----------------------------------------------------------------------------
1587 # Utils
1588
1589 def check_stdout_ok( name ):
1590 if getTestOpts().with_namebase == None:
1591 namebase = name
1592 else:
1593 namebase = getTestOpts().with_namebase
1594
1595 actual_stdout_file = qualify(name, 'run.stdout')
1596 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1597
1598 def norm(str):
1599 if platform_specific:
1600 return str
1601 else:
1602 return normalise_output(str)
1603
1604 return compare_outputs('stdout', \
1605 two_normalisers(norm, getTestOpts().extra_normaliser), \
1606 expected_stdout_file, actual_stdout_file)
1607
1608 def dump_stdout( name ):
1609 print 'Stdout:'
1610 print read_no_crs(qualify(name, 'run.stdout'))
1611
1612 def check_stderr_ok( name ):
1613 if getTestOpts().with_namebase == None:
1614 namebase = name
1615 else:
1616 namebase = getTestOpts().with_namebase
1617
1618 actual_stderr_file = qualify(name, 'run.stderr')
1619 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1620
1621 def norm(str):
1622 if platform_specific:
1623 return str
1624 else:
1625 return normalise_errmsg(str)
1626
1627 return compare_outputs('stderr', \
1628 two_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1629 expected_stderr_file, actual_stderr_file)
1630
1631 def dump_stderr( name ):
1632 print "Stderr:"
1633 print read_no_crs(qualify(name, 'run.stderr'))
1634
1635 def read_no_crs(file):
1636 str = ''
1637 try:
1638 h = open(file)
1639 str = h.read()
1640 h.close
1641 except:
1642 # On Windows, if the program fails very early, it seems the
1643 # files stdout/stderr are redirected to may not get created
1644 pass
1645 return re.sub('\r', '', str)
1646
1647 def write_file(file, str):
1648 h = open(file, 'w')
1649 h.write(str)
1650 h.close
1651
1652 def check_hp_ok(name):
1653
1654 # do not qualify for hp2ps because we should be in the right directory
1655 hp2psCmd = "cd " + getTestOpts().testdir + " && '" + config.hp2ps + "' " + name
1656
1657 hp2psResult = runCmdExitCode(hp2psCmd)
1658
1659 actual_ps_file = qualify(name, 'ps')
1660
1661 if(hp2psResult == 0):
1662 if (os.path.exists(actual_ps_file)):
1663 if gs_working:
1664 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1665 if (gsResult == 0):
1666 return (True)
1667 else:
1668 print "hp2ps output for " + name + "is not valid PostScript"
1669 else: return (True) # assume postscript is valid without ghostscript
1670 else:
1671 print "hp2ps did not generate PostScript for " + name
1672 return (False)
1673 else:
1674 print "hp2ps error when processing heap profile for " + name
1675 return(False)
1676
1677 def check_prof_ok(name):
1678
1679 prof_file = qualify(name,'prof')
1680
1681 if not os.path.exists(prof_file):
1682 print prof_file + " does not exist"
1683 return(False)
1684
1685 if os.path.getsize(qualify(name,'prof')) == 0:
1686 print prof_file + " is empty"
1687 return(False)
1688
1689 if getTestOpts().with_namebase == None:
1690 namebase = name
1691 else:
1692 namebase = getTestOpts().with_namebase
1693
1694 (platform_specific, expected_prof_file) = \
1695 platform_wordsize_qualify(namebase, 'prof.sample')
1696
1697 # sample prof file is not required
1698 if not os.path.exists(expected_prof_file):
1699 return True
1700 else:
1701 return compare_outputs('prof', \
1702 two_normalisers(normalise_whitespace,normalise_prof), \
1703 expected_prof_file, prof_file)
1704
1705 # Compare expected output to actual output, and optionally accept the
1706 # new output. Returns true if output matched or was accepted, false
1707 # otherwise.
1708 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1709 if os.path.exists(expected_file):
1710 expected_raw = read_no_crs(expected_file)
1711 # print "norm:", normaliser(expected_raw)
1712 expected_str = normaliser(expected_raw)
1713 expected_file_for_diff = expected_file
1714 else:
1715 expected_str = ''
1716 expected_file_for_diff = '/dev/null'
1717
1718 actual_raw = read_no_crs(actual_file)
1719 actual_str = normaliser(actual_raw)
1720
1721 if expected_str == actual_str:
1722 return 1
1723 else:
1724 print 'Actual ' + kind + ' output differs from expected:'
1725
1726 if expected_file_for_diff == '/dev/null':
1727 expected_normalised_file = '/dev/null'
1728 else:
1729 expected_normalised_file = expected_file + ".normalised"
1730 write_file(expected_normalised_file, expected_str)
1731
1732 actual_normalised_file = actual_file + ".normalised"
1733 write_file(actual_normalised_file, actual_str)
1734
1735 # Ignore whitespace when diffing. We should only get to this
1736 # point if there are non-whitespace differences
1737 #
1738 # Note we are diffing the *actual* output, not the normalised
1739 # output. The normalised output may have whitespace squashed
1740 # (including newlines) so the diff would be hard to read.
1741 # This does mean that the diff might contain changes that
1742 # would be normalised away.
1743 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1744 ' ' + actual_file )
1745
1746 # If for some reason there were no non-whitespace differences,
1747 # then do a full diff
1748 if r == 0:
1749 r = os.system( 'diff -u ' + expected_file_for_diff + \
1750 ' ' + actual_file )
1751
1752 if config.accept:
1753 print 'Accepting new output.'
1754 write_file(expected_file, actual_raw)
1755 return 1
1756 else:
1757 return 0
1758
1759
1760 def normalise_whitespace( str ):
1761 # Merge contiguous whitespace characters into a single space.
1762 str = re.sub('[ \t\n]+', ' ', str)
1763 return str
1764
1765 def normalise_errmsg( str ):
1766 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1767 # the colon is there because it appears in error messages; this
1768 # hacky solution is used in place of more sophisticated filename
1769 # mangling
1770 str = re.sub('([^\\s])\\.exe', '\\1', str)
1771 # normalise slashes, minimise Windows/Unix filename differences
1772 str = re.sub('\\\\', '/', str)
1773 # The inplace ghc's are called ghc-stage[123] to avoid filename
1774 # collisions, so we need to normalise that to just "ghc"
1775 str = re.sub('ghc-stage[123]', 'ghc', str)
1776 # We sometimes see the name of the integer-gmp package on stderr,
1777 # but this can change (either the implementation name or the
1778 # version number), so we canonicalise it here
1779 str = re.sub('integer-[a-z]+', 'integer-impl', str)
1780 return str
1781
1782 # normalise a .prof file, so that we can reasonably compare it against
1783 # a sample. This doesn't compare any of the actual profiling data,
1784 # only the shape of the profile and the number of entries.
1785 def normalise_prof (str):
1786 # strip everything up to the line beginning "COST CENTRE"
1787 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1788
1789 # strip results for CAFs, these tend to change unpredictably
1790 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1791
1792 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1793 # sometimes under MAIN.
1794 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1795
1796 # We have somthing like this:
1797
1798 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1799 # k Main 204 1 0.0 0.0 0.0 0.0
1800 # foo Main 205 1 0.0 0.0 0.0 0.0
1801 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1802
1803 # then we remove all the specific profiling data, leaving only the
1804 # cost centre name, module, and entries, to end up with this:
1805
1806 # MAIN MAIN 0
1807 # k Main 1
1808 # foo Main 1
1809 # foo.bar Main 1
1810
1811 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1812 return str
1813
1814 def normalise_slashes_( str ):
1815 str = re.sub('\\\\', '/', str)
1816 return str
1817
1818 def normalise_exe_( str ):
1819 str = re.sub('\.exe', '', str)
1820 return str
1821
1822 def normalise_output( str ):
1823 # Remove a .exe extension (for Windows)
1824 # This can occur in error messages generated by the program.
1825 str = re.sub('([^\\s])\\.exe', '\\1', str)
1826 return str
1827
1828 def normalise_asm( str ):
1829 lines = str.split('\n')
1830 # Only keep instructions and labels not starting with a dot.
1831 metadata = re.compile('^[ \t]*\\..*$')
1832 out = []
1833 for line in lines:
1834 # Drop metadata directives (e.g. ".type")
1835 if not metadata.match(line):
1836 line = re.sub('@plt', '', line)
1837 instr = line.lstrip().split()
1838 # Drop empty lines.
1839 if not instr:
1840 continue
1841 # Drop operands, except for call instructions.
1842 elif instr[0] == 'call':
1843 out.append(instr[0] + ' ' + instr[1])
1844 else:
1845 out.append(instr[0])
1846 out = '\n'.join(out)
1847 return out
1848
1849 def if_verbose( n, str ):
1850 if config.verbose >= n:
1851 print str
1852
1853 def if_verbose_dump( n, f ):
1854 if config.verbose >= n:
1855 try:
1856 print open(f).read()
1857 except:
1858 print ''
1859
1860 def rawSystem(cmd_and_args):
1861 # We prefer subprocess.call to os.spawnv as the latter
1862 # seems to send its arguments through a shell or something
1863 # with the Windows (non-cygwin) python. An argument "a b c"
1864 # turns into three arguments ["a", "b", "c"].
1865
1866 # However, subprocess is new in python 2.4, so fall back to
1867 # using spawnv if we don't have it
1868
1869 if have_subprocess:
1870 return subprocess.call(cmd_and_args)
1871 else:
1872 return os.spawnv(os.P_WAIT, cmd_and_args[0], cmd_and_args)
1873
1874 # Note that this doesn't handle the timeout itself; it is just used for
1875 # commands that have timeout handling built-in.
1876 def rawSystemWithTimeout(cmd_and_args):
1877 r = rawSystem(cmd_and_args)
1878 if r == 98:
1879 # The python timeout program uses 98 to signal that ^C was pressed
1880 stopNow()
1881 return r
1882
1883 # cmd is a complex command in Bourne-shell syntax
1884 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1885 # Hence it must ultimately be run by a Bourne shell
1886 #
1887 # Mostly it invokes the command wrapped in 'timeout' thus
1888 # timeout 300 'cd . && ...blah blah'
1889 # so it's timeout's job to invoke the Bourne shell
1890 #
1891 # But watch out for the case when there is no timeout program!
1892 # Then, when using the native Python, os.system will invoke the cmd shell
1893
1894 def runCmd( cmd ):
1895 if_verbose( 1, cmd )
1896 r = 0
1897 if config.os == 'mingw32':
1898 # On MinGW, we will always have timeout
1899 assert config.timeout_prog!=''
1900
1901 if config.timeout_prog != '':
1902 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1903 else:
1904 r = os.system(cmd)
1905 return r << 8
1906
1907 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1908 if_verbose( 1, cmd )
1909 r = 0
1910 if config.os == 'mingw32':
1911 # On MinGW, we will always have timeout
1912 assert config.timeout_prog!=''
1913 timeout = int(math.ceil(config.timeout * timeout_multiplier))
1914
1915 if config.timeout_prog != '':
1916 if config.check_files_written:
1917 fn = name + ".strace"
1918 r = rawSystemWithTimeout(
1919 ["strace", "-o", fn, "-fF",
1920 "-e", "creat,open,chdir,clone,vfork",
1921 config.timeout_prog, str(timeout), cmd])
1922 addTestFilesWritten(name, fn)
1923 rm_no_fail(fn)
1924 else:
1925 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1926 else:
1927 r = os.system(cmd)
1928 return r << 8
1929
1930 def runCmdExitCode( cmd ):
1931 return (runCmd(cmd) >> 8);
1932
1933
1934 # -----------------------------------------------------------------------------
1935 # checking for files being written to by multiple tests
1936
1937 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1938 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1939 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1940 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1941 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1942 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1943 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1944 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1945 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1946 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1947 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1948 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1949 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1950 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1951 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1952 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1953 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1954
1955 # Files that are read or written but shouldn't be:
1956 # * ghci_history shouldn't be read or written by tests
1957 # * things under package.conf.d shouldn't be written by tests
1958 bad_file_usages = {}
1959
1960 # Mapping from tests to the list of files that they write
1961 files_written = {}
1962
1963 # Mapping from tests to the list of files that they write but don't clean
1964 files_written_not_removed = {}
1965
1966 def add_bad_file_usage(name, file):
1967 try:
1968 if not file in bad_file_usages[name]:
1969 bad_file_usages[name].append(file)
1970 except:
1971 bad_file_usages[name] = [file]
1972
1973 def mkPath(curdir, path):
1974 # Given the current full directory is 'curdir', what is the full
1975 # path to 'path'?
1976 return os.path.realpath(os.path.join(curdir, path))
1977
1978 def addTestFilesWritten(name, fn):
1979 if config.use_threads:
1980 with t.lockFilesWritten:
1981 addTestFilesWrittenHelper(name, fn)
1982 else:
1983 addTestFilesWrittenHelper(name, fn)
1984
1985 def addTestFilesWrittenHelper(name, fn):
1986 started = False
1987 working_directories = {}
1988
1989 with open(fn, 'r') as f:
1990 for line in f:
1991 m_pid = re_strace_pid.match(line)
1992 if m_pid:
1993 pid = m_pid.group(1)
1994 content = m_pid.group(2)
1995 elif re_strace_unavailable.match(line):
1996 next
1997 else:
1998 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1999
2000 m_open = re_strace_open.match(content)
2001 m_chdir = re_strace_chdir.match(content)
2002 m_clone = re_strace_clone.match(content)
2003 m_vfork = re_strace_vfork.match(content)
2004
2005 if not started:
2006 working_directories[pid] = os.getcwd()
2007 started = True
2008
2009 if m_open:
2010 file = m_open.group(1)
2011 file = mkPath(working_directories[pid], file)
2012 if file.endswith("ghci_history"):
2013 add_bad_file_usage(name, file)
2014 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
2015 flags = m_open.group(2).split('|')
2016 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
2017 if re.match('package\.conf\.d', file):
2018 add_bad_file_usage(name, file)
2019 else:
2020 try:
2021 if not file in files_written[name]:
2022 files_written[name].append(file)
2023 except:
2024 files_written[name] = [file]
2025 elif 'O_RDONLY' in flags:
2026 pass
2027 else:
2028 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
2029 elif m_chdir:
2030 # We optimistically assume that unfinished chdir's are going to succeed
2031 dir = m_chdir.group(1)
2032 working_directories[pid] = mkPath(working_directories[pid], dir)
2033 elif m_clone:
2034 working_directories[m_clone.group(2)] = working_directories[pid]
2035 elif m_vfork:
2036 working_directories[m_vfork.group(2)] = working_directories[pid]
2037 elif re_strace_open_resumed.match(content):
2038 pass
2039 elif re_strace_chdir_resumed.match(content):
2040 pass
2041 elif re_strace_vfork_unfinished.match(content):
2042 pass
2043 elif re_strace_clone_unfinished.match(content):
2044 pass
2045 elif re_strace_ignore_sigchild.match(content):
2046 pass
2047 elif re_strace_ignore_sigvtalarm.match(content):
2048 pass
2049 elif re_strace_ignore_sigint.match(content):
2050 pass
2051 elif re_strace_ignore_sigfpe.match(content):
2052 pass
2053 elif re_strace_ignore_sigsegv.match(content):
2054 pass
2055 elif re_strace_ignore_sigpipe.match(content):
2056 pass
2057 else:
2058 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2059
2060 def checkForFilesWrittenProblems(file):
2061 foundProblem = False
2062
2063 files_written_inverted = {}
2064 for t in files_written.keys():
2065 for f in files_written[t]:
2066 try:
2067 files_written_inverted[f].append(t)
2068 except:
2069 files_written_inverted[f] = [t]
2070
2071 for f in files_written_inverted.keys():
2072 if len(files_written_inverted[f]) > 1:
2073 if not foundProblem:
2074 foundProblem = True
2075 file.write("\n")
2076 file.write("\nSome files are written by multiple tests:\n")
2077 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2078 if foundProblem:
2079 file.write("\n")
2080
2081 # -----
2082
2083 if len(files_written_not_removed) > 0:
2084 file.write("\n")
2085 file.write("\nSome files written but not removed:\n")
2086 tests = files_written_not_removed.keys()
2087 tests.sort()
2088 for t in tests:
2089 for f in files_written_not_removed[t]:
2090 file.write(" " + t + ": " + f + "\n")
2091 file.write("\n")
2092
2093 # -----
2094
2095 if len(bad_file_usages) > 0:
2096 file.write("\n")
2097 file.write("\nSome bad file usages:\n")
2098 tests = bad_file_usages.keys()
2099 tests.sort()
2100 for t in tests:
2101 for f in bad_file_usages[t]:
2102 file.write(" " + t + ": " + f + "\n")
2103 file.write("\n")
2104
2105 # -----------------------------------------------------------------------------
2106 # checking if ghostscript is available for checking the output of hp2ps
2107
2108 def genGSCmd(psfile):
2109 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2110
2111 def gsNotWorking():
2112 global gs_working
2113 print "GhostScript not available for hp2ps tests"
2114
2115 global gs_working
2116 gs_working = 0
2117 if config.have_profiling:
2118 if config.gs != '':
2119 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2120 if resultGood == 0:
2121 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2122 if resultBad != 0:
2123 print "GhostScript available for hp2ps tests"
2124 gs_working = 1;
2125 else:
2126 gsNotWorking();
2127 else:
2128 gsNotWorking();
2129 else:
2130 gsNotWorking();
2131
2132 def rm_no_fail( file ):
2133 try:
2134 os.remove( file )
2135 finally:
2136 return
2137
2138 def add_suffix( name, suffix ):
2139 if suffix == '':
2140 return name
2141 else:
2142 return name + '.' + suffix
2143
2144 def add_hs_lhs_suffix(name):
2145 if getTestOpts().c_src:
2146 return add_suffix(name, 'c')
2147 elif getTestOpts().cmm_src:
2148 return add_suffix(name, 'cmm')
2149 elif getTestOpts().objc_src:
2150 return add_suffix(name, 'm')
2151 elif getTestOpts().objcpp_src:
2152 return add_suffix(name, 'mm')
2153 elif getTestOpts().literate:
2154 return add_suffix(name, 'lhs')
2155 else:
2156 return add_suffix(name, 'hs')
2157
2158 def replace_suffix( name, suffix ):
2159 base, suf = os.path.splitext(name)
2160 return base + '.' + suffix
2161
2162 def in_testdir( name ):
2163 return (getTestOpts().testdir + '/' + name)
2164
2165 def qualify( name, suff ):
2166 return in_testdir(add_suffix(name, suff))
2167
2168
2169 # Finding the sample output. The filename is of the form
2170 #
2171 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2172 #
2173 # and we pick the most specific version available. The <version> is
2174 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2175 # more fine-grained control use if_compiler_lt().
2176 #
2177 def platform_wordsize_qualify( name, suff ):
2178
2179 basepath = qualify(name, suff)
2180
2181 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2182 for (platformSpecific, plat) in [(1, '-' + config.platform),
2183 (1, '-' + config.os),
2184 (0, '')]
2185 for ws in ['-ws-' + config.wordsize, '']
2186 for comp in ['-' + config.compiler_type, '']
2187 for vers in ['-' + config.compiler_maj_version, '']]
2188
2189 dir = glob.glob(basepath + '*')
2190 dir = map (lambda d: normalise_slashes_(d), dir)
2191
2192 for (platformSpecific, f) in paths:
2193 if f in dir:
2194 return (platformSpecific,f)
2195
2196 return (0, basepath)
2197
2198 # Clean up prior to the test, so that we can't spuriously conclude
2199 # that it passed on the basis of old run outputs.
2200 def pretest_cleanup(name):
2201 if getTestOpts().outputdir != None:
2202 odir = in_testdir(getTestOpts().outputdir)
2203 try:
2204 shutil.rmtree(odir)
2205 except:
2206 pass
2207 os.mkdir(odir)
2208
2209 rm_no_fail(qualify(name,'comp.stderr'))
2210 rm_no_fail(qualify(name,'run.stderr'))
2211 rm_no_fail(qualify(name,'run.stdout'))
2212 rm_no_fail(qualify(name,'tix')) # remove the old tix file
2213 # simple_build zaps the following:
2214 # rm_nofail(qualify("o"))
2215 # rm_nofail(qualify(""))
2216 # not interested in the return code
2217
2218 # -----------------------------------------------------------------------------
2219 # Return a list of all the files ending in '.T' below the directory dir.
2220
2221 def findTFiles(roots):
2222 return concat(map(findTFiles_,roots))
2223
2224 def findTFiles_(path):
2225 if os.path.isdir(path):
2226 paths = map(lambda x, p=path: p + '/' + x, os.listdir(path))
2227 return findTFiles(paths)
2228 elif path[-2:] == '.T':
2229 return [path]
2230 else:
2231 return []
2232
2233 # -----------------------------------------------------------------------------
2234 # Output a test summary to the specified file object
2235
2236 def summary(t, file):
2237
2238 file.write('\n')
2239 file.write('OVERALL SUMMARY for test run started at ' \
2240 + t.start_time + '\n'\
2241 + string.rjust(`t.total_tests`, 8) \
2242 + ' total tests, which gave rise to\n' \
2243 + string.rjust(`t.total_test_cases`, 8) \
2244 + ' test cases, of which\n' \
2245 + string.rjust(`t.n_framework_failures`, 8) \
2246 + ' caused framework failures\n' \
2247 + string.rjust(`t.n_tests_skipped`, 8)
2248 + ' were skipped\n\n' \
2249 + string.rjust(`t.n_expected_passes`, 8)
2250 + ' expected passes\n' \
2251 + string.rjust(`t.n_missing_libs`, 8)
2252 + ' had missing libraries\n' \
2253 + string.rjust(`t.n_expected_failures`, 8) \
2254 + ' expected failures\n' \
2255 + string.rjust(`t.n_unexpected_passes`, 8) \
2256 + ' unexpected passes\n'
2257 + string.rjust(`t.n_unexpected_failures`, 8) \
2258 + ' unexpected failures\n'
2259 + '\n')
2260
2261 if t.n_unexpected_passes > 0:
2262 file.write('Unexpected passes:\n')
2263 printPassingTestInfosSummary(file, t.unexpected_passes)
2264
2265 if t.n_unexpected_failures > 0:
2266 file.write('Unexpected failures:\n')
2267 printFailingTestInfosSummary(file, t.unexpected_failures)
2268
2269 if config.check_files_written:
2270 checkForFilesWrittenProblems(file)
2271
2272 if stopping():
2273 file.write('WARNING: Testsuite run was terminated early\n')
2274
2275 def printPassingTestInfosSummary(file, testInfos):
2276 directories = testInfos.keys()
2277 directories.sort()
2278 maxDirLen = max(map ((lambda x : len(x)), directories))
2279 for directory in directories:
2280 tests = testInfos[directory].keys()
2281 tests.sort()
2282 for test in tests:
2283 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2284 ' (' + join(testInfos[directory][test],',') + ')\n')
2285 file.write('\n')
2286
2287 def printFailingTestInfosSummary(file, testInfos):
2288 directories = testInfos.keys()
2289 directories.sort()
2290 maxDirLen = max(map ((lambda x : len(x)), directories))
2291 for directory in directories:
2292 tests = testInfos[directory].keys()
2293 tests.sort()
2294 for test in tests:
2295 reasons = testInfos[directory][test].keys()
2296 for reason in reasons:
2297 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2298 ' [' + reason + ']' + \
2299 ' (' + join(testInfos[directory][test][reason],',') + ')\n')
2300 file.write('\n')
2301
2302 def getStdout(cmd):
2303 if have_subprocess:
2304 p = subprocess.Popen(cmd,
2305 stdout=subprocess.PIPE,
2306 stderr=subprocess.PIPE)
2307 (stdout, stderr) = p.communicate()
2308 r = p.wait()
2309 if r != 0:
2310 raise Exception("Command failed: " + str(cmd))
2311 if stderr != '':
2312 raise Exception("stderr from command: " + str(cmd))
2313 return stdout
2314 else:
2315 raise Exception("Need subprocess to get stdout, but don't have it")