testsuite: use same flags for ghci way and scripts
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 from __future__ import print_function
6
7 import shutil
8 import sys
9 import os
10 import errno
11 import string
12 import re
13 import traceback
14 import time
15 import datetime
16 import copy
17 import glob
18 from math import ceil, trunc
19 import collections
20
21 have_subprocess = False
22 try:
23 import subprocess
24 have_subprocess = True
25 except:
26 print("Warning: subprocess not found, will fall back to spawnv")
27
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 try:
34 import thread
35 except ImportError: # Python 3
36 import _thread as thread
37
38 global wantToStop
39 wantToStop = False
40 def stopNow():
41 global wantToStop
42 wantToStop = True
43 def stopping():
44 return wantToStop
45
46 # Options valid for the current test only (these get reset to
47 # testdir_testopts after each test).
48
49 global testopts_local
50 if config.use_threads:
51 testopts_local = threading.local()
52 else:
53 class TestOpts_Local:
54 pass
55 testopts_local = TestOpts_Local()
56
57 def getTestOpts():
58 return testopts_local.x
59
60 def setLocalTestOpts(opts):
61 global testopts_local
62 testopts_local.x=opts
63
64 def isStatsTest():
65 opts = getTestOpts()
66 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
67
68
69 # This can be called at the top of a file of tests, to set default test options
70 # for the following tests.
71 def setTestOpts( f ):
72 global thisdir_settings
73 thisdir_settings = [thisdir_settings, f]
74
75 # -----------------------------------------------------------------------------
76 # Canned setup functions for common cases. eg. for a test you might say
77 #
78 # test('test001', normal, compile, [''])
79 #
80 # to run it without any options, but change it to
81 #
82 # test('test001', expect_fail, compile, [''])
83 #
84 # to expect failure for this test.
85
86 def normal( name, opts ):
87 return;
88
89 def skip( name, opts ):
90 opts.skip = 1
91
92 def expect_fail( name, opts ):
93 opts.expect = 'fail';
94
95 def reqlib( lib ):
96 return lambda name, opts, l=lib: _reqlib (name, opts, l )
97
98 # Cache the results of looking to see if we have a library or not.
99 # This makes quite a difference, especially on Windows.
100 have_lib = {}
101
102 def _reqlib( name, opts, lib ):
103 if lib in have_lib:
104 got_it = have_lib[lib]
105 else:
106 if have_subprocess:
107 # By preference we use subprocess, as the alternative uses
108 # /dev/null which mingw doesn't have.
109 cmd = strip_quotes(config.ghc_pkg)
110 p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' --no-user-package-db describe '
119 + lib + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( name, opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( name, opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( name, opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( name, opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def ignore_output( name, opts ):
143 opts.ignore_output = 1
144
145 def no_stdin( name, opts ):
146 opts.no_stdin = 1
147
148 def combined_output( name, opts ):
149 opts.combined_output = True
150
151 # -----
152
153 def expect_fail_for( ways ):
154 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
155
156 def _expect_fail_for( name, opts, ways ):
157 opts.expect_fail_for = ways
158
159 def expect_broken( bug ):
160 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
161
162 def _expect_broken( name, opts, bug ):
163 record_broken(name, opts, bug)
164 opts.expect = 'fail';
165
166 def expect_broken_for( bug, ways ):
167 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
168
169 def _expect_broken_for( name, opts, bug, ways ):
170 record_broken(name, opts, bug)
171 opts.expect_fail_for = ways
172
173 def record_broken(name, opts, bug):
174 global brokens
175 me = (bug, opts.testdir, name)
176 if not me in brokens:
177 brokens.append(me)
178
179 # -----
180
181 def omit_ways( ways ):
182 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
183
184 def _omit_ways( name, opts, ways ):
185 opts.omit_ways = ways
186
187 # -----
188
189 def only_ways( ways ):
190 return lambda name, opts, w=ways: _only_ways( name, opts, w )
191
192 def _only_ways( name, opts, ways ):
193 opts.only_ways = ways
194
195 # -----
196
197 def extra_ways( ways ):
198 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
199
200 def _extra_ways( name, opts, ways ):
201 opts.extra_ways = ways
202
203 # -----
204
205 def omit_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
207
208 def _omit_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def only_compiler_types( compiler_types ):
215 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
216
217 def _only_compiler_types( name, opts, compiler_types ):
218 if config.compiler_type not in compiler_types:
219 opts.skip = 1
220
221 # -----
222
223 def set_stdin( file ):
224 return lambda name, opts, f=file: _set_stdin(name, opts, f);
225
226 def _set_stdin( name, opts, f ):
227 opts.stdin = f
228
229 # -----
230
231 def exit_code( val ):
232 return lambda name, opts, v=val: _exit_code(name, opts, v);
233
234 def _exit_code( name, opts, v ):
235 opts.exit_code = v
236
237 def signal_exit_code( val ):
238 if opsys('solaris2'):
239 return exit_code( val );
240 else:
241 # When application running on Linux receives fatal error
242 # signal, then its exit code is encoded as 128 + signal
243 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
244 # I assume that Mac OS X behaves in the same way at least Mac
245 # OS X builder behavior suggests this.
246 return exit_code( val+128 );
247
248 # -----
249
250 def timeout_multiplier( val ):
251 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
252
253 def _timeout_multiplier( name, opts, v ):
254 opts.timeout_multiplier = v
255
256 # -----
257
258 def extra_run_opts( val ):
259 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
260
261 def _extra_run_opts( name, opts, v ):
262 opts.extra_run_opts = v
263
264 # -----
265
266 def extra_hc_opts( val ):
267 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
268
269 def _extra_hc_opts( name, opts, v ):
270 opts.extra_hc_opts = v
271
272 # -----
273
274 def extra_clean( files ):
275 return lambda name, opts, v=files: _extra_clean(name, opts, v);
276
277 def _extra_clean( name, opts, v ):
278 opts.clean_files = v
279
280 # -----
281
282 def stats_num_field( field, expecteds ):
283 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
284
285 def _stats_num_field( name, opts, field, expecteds ):
286 if field in opts.stats_range_fields:
287 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
288
289 if type(expecteds) is list:
290 for (b, expected, dev) in expecteds:
291 if b:
292 opts.stats_range_fields[field] = (expected, dev)
293 return
294 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
295
296 else:
297 (expected, dev) = expecteds
298 opts.stats_range_fields[field] = (expected, dev)
299
300 def compiler_stats_num_field( field, expecteds ):
301 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
302
303 def _compiler_stats_num_field( name, opts, field, expecteds ):
304 if field in opts.compiler_stats_range_fields:
305 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
306
307 # Compiler performance numbers change when debugging is on, making the results
308 # useless and confusing. Therefore, skip if debugging is on.
309 if compiler_debugged():
310 skip(name, opts)
311
312 for (b, expected, dev) in expecteds:
313 if b:
314 opts.compiler_stats_range_fields[field] = (expected, dev)
315 return
316
317 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
318
319 # -----
320
321 def when(b, f):
322 # When list_brokens is on, we want to see all expect_broken calls,
323 # so we always do f
324 if b or config.list_broken:
325 return f
326 else:
327 return normal
328
329 def unless(b, f):
330 return when(not b, f)
331
332 def doing_ghci():
333 return 'ghci' in config.run_ways
334
335 def ghci_dynamic( ):
336 return config.ghc_dynamic
337
338 def fast():
339 return config.fast
340
341 def platform( plat ):
342 return config.platform == plat
343
344 def opsys( os ):
345 return config.os == os
346
347 def arch( arch ):
348 return config.arch == arch
349
350 def wordsize( ws ):
351 return config.wordsize == str(ws)
352
353 def msys( ):
354 return config.msys
355
356 def cygwin( ):
357 return config.cygwin
358
359 def have_vanilla( ):
360 return config.have_vanilla
361
362 def have_dynamic( ):
363 return config.have_dynamic
364
365 def have_profiling( ):
366 return config.have_profiling
367
368 def in_tree_compiler( ):
369 return config.in_tree_compiler
370
371 def compiler_type( compiler ):
372 return config.compiler_type == compiler
373
374 def compiler_lt( compiler, version ):
375 return config.compiler_type == compiler and \
376 version_lt(config.compiler_version, version)
377
378 def compiler_le( compiler, version ):
379 return config.compiler_type == compiler and \
380 version_le(config.compiler_version, version)
381
382 def compiler_gt( compiler, version ):
383 return config.compiler_type == compiler and \
384 version_gt(config.compiler_version, version)
385
386 def compiler_ge( compiler, version ):
387 return config.compiler_type == compiler and \
388 version_ge(config.compiler_version, version)
389
390 def unregisterised( ):
391 return config.unregisterised
392
393 def compiler_profiled( ):
394 return config.compiler_profiled
395
396 def compiler_debugged( ):
397 return config.compiler_debugged
398
399 def tag( t ):
400 return t in config.compiler_tags
401
402 # ---
403
404 def namebase( nb ):
405 return lambda opts, nb=nb: _namebase(opts, nb)
406
407 def _namebase( opts, nb ):
408 opts.with_namebase = nb
409
410 # ---
411
412 def high_memory_usage(name, opts):
413 opts.alone = True
414
415 # If a test is for a multi-CPU race, then running the test alone
416 # increases the chance that we'll actually see it.
417 def multi_cpu_race(name, opts):
418 opts.alone = True
419
420 # ---
421 def literate( name, opts ):
422 opts.literate = 1;
423
424 def c_src( name, opts ):
425 opts.c_src = 1;
426
427 def objc_src( name, opts ):
428 opts.objc_src = 1;
429
430 def objcpp_src( name, opts ):
431 opts.objcpp_src = 1;
432
433 def cmm_src( name, opts ):
434 opts.cmm_src = 1;
435
436 def outputdir( odir ):
437 return lambda name, opts, d=odir: _outputdir(name, opts, d)
438
439 def _outputdir( name, opts, odir ):
440 opts.outputdir = odir;
441
442 # ----
443
444 def pre_cmd( cmd ):
445 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
446
447 def _pre_cmd( name, opts, cmd ):
448 opts.pre_cmd = cmd
449
450 # ----
451
452 def clean_cmd( cmd ):
453 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
454
455 def _clean_cmd( name, opts, cmd ):
456 opts.clean_cmd = cmd
457
458 # ----
459
460 def cmd_prefix( prefix ):
461 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
462
463 def _cmd_prefix( name, opts, prefix ):
464 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
465
466 # ----
467
468 def cmd_wrapper( fun ):
469 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
470
471 def _cmd_wrapper( name, opts, fun ):
472 opts.cmd_wrapper = fun
473
474 # ----
475
476 def compile_cmd_prefix( prefix ):
477 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
478
479 def _compile_cmd_prefix( name, opts, prefix ):
480 opts.compile_cmd_prefix = prefix
481
482 # ----
483
484 def check_stdout( f ):
485 return lambda name, opts, f=f: _check_stdout(name, opts, f)
486
487 def _check_stdout( name, opts, f ):
488 opts.check_stdout = f
489
490 # ----
491
492 def normalise_slashes( name, opts ):
493 _normalise_fun(name, opts, normalise_slashes_)
494
495 def normalise_exe( name, opts ):
496 _normalise_fun(name, opts, normalise_exe_)
497
498 def normalise_fun( *fs ):
499 return lambda name, opts: _normalise_fun(name, opts, fs)
500
501 def _normalise_fun( name, opts, *fs ):
502 opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
503
504 def normalise_errmsg_fun( *fs ):
505 return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
506
507 def _normalise_errmsg_fun( name, opts, *fs ):
508 opts.extra_errmsg_normaliser = join_normalisers(opts.extra_errmsg_normaliser, fs)
509
510 def normalise_version_( *pkgs ):
511 def normalise_version__( str ):
512 return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
513 '\\1-<VERSION>', str)
514 return normalise_version__
515
516 def normalise_version( *pkgs ):
517 def normalise_version__( name, opts ):
518 _normalise_fun(name, opts, normalise_version_(*pkgs))
519 _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
520 return normalise_version__
521
522 def join_normalisers(*a):
523 """
524 Compose functions, flattening sequences.
525
526 join_normalisers(f1,[f2,f3],f4)
527
528 is the same as
529
530 lambda x: f1(f2(f3(f4(x))))
531 """
532
533 def flatten(l):
534 """
535 Taken from http://stackoverflow.com/a/2158532/946226
536 """
537 for el in l:
538 if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
539 for sub in flatten(el):
540 yield sub
541 else:
542 yield el
543
544 a = flatten(a)
545
546 fn = lambda x:x # identity function
547 for f in a:
548 assert callable(f)
549 fn = lambda x,f=f,fn=fn: fn(f(x))
550 return fn
551
552 # ----
553 # Function for composing two opt-fns together
554
555 def executeSetups(fs, name, opts):
556 if type(fs) is list:
557 # If we have a list of setups, then execute each one
558 for f in fs:
559 executeSetups(f, name, opts)
560 else:
561 # fs is a single function, so just apply it
562 fs(name, opts)
563
564 # -----------------------------------------------------------------------------
565 # The current directory of tests
566
567 def newTestDir( dir ):
568 global thisdir_settings
569 # reset the options for this test directory
570 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
571
572 def _newTestDir( name, opts, dir ):
573 opts.testdir = dir
574 opts.compiler_always_flags = config.compiler_always_flags
575
576 # -----------------------------------------------------------------------------
577 # Actually doing tests
578
579 parallelTests = []
580 aloneTests = []
581 allTestNames = set([])
582
583 def runTest (opts, name, func, args):
584 ok = 0
585
586 if config.use_threads:
587 t.thread_pool.acquire()
588 try:
589 while config.threads<(t.running_threads+1):
590 t.thread_pool.wait()
591 t.running_threads = t.running_threads+1
592 ok=1
593 t.thread_pool.release()
594 thread.start_new_thread(test_common_thread, (name, opts, func, args))
595 except:
596 if not ok:
597 t.thread_pool.release()
598 else:
599 test_common_work (name, opts, func, args)
600
601 # name :: String
602 # setup :: TestOpts -> IO ()
603 def test (name, setup, func, args):
604 global aloneTests
605 global parallelTests
606 global allTestNames
607 global thisdir_settings
608 if name in allTestNames:
609 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
610 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
611 framework_fail(name, 'bad_name', 'This test has an invalid name')
612
613 # Make a deep copy of the default_testopts, as we need our own copy
614 # of any dictionaries etc inside it. Otherwise, if one test modifies
615 # them, all tests will see the modified version!
616 myTestOpts = copy.deepcopy(default_testopts)
617
618 executeSetups([thisdir_settings, setup], name, myTestOpts)
619
620 thisTest = lambda : runTest(myTestOpts, name, func, args)
621 if myTestOpts.alone:
622 aloneTests.append(thisTest)
623 else:
624 parallelTests.append(thisTest)
625 allTestNames.add(name)
626
627 if config.use_threads:
628 def test_common_thread(name, opts, func, args):
629 t.lock.acquire()
630 try:
631 test_common_work(name,opts,func,args)
632 finally:
633 t.lock.release()
634 t.thread_pool.acquire()
635 t.running_threads = t.running_threads - 1
636 t.thread_pool.notify()
637 t.thread_pool.release()
638
639 def get_package_cache_timestamp():
640 if config.package_conf_cache_file == '':
641 return 0.0
642 else:
643 try:
644 return os.stat(config.package_conf_cache_file).st_mtime
645 except:
646 return 0.0
647
648
649 def test_common_work (name, opts, func, args):
650 try:
651 t.total_tests = t.total_tests+1
652 setLocalTestOpts(opts)
653
654 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
655
656 # All the ways we might run this test
657 if func == compile or func == multimod_compile:
658 all_ways = config.compile_ways
659 elif func == compile_and_run or func == multimod_compile_and_run:
660 all_ways = config.run_ways
661 elif func == ghci_script:
662 if 'ghci' in config.run_ways:
663 all_ways = ['ghci']
664 else:
665 all_ways = []
666 else:
667 all_ways = ['normal']
668
669 # A test itself can request extra ways by setting opts.extra_ways
670 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
671
672 t.total_test_cases = t.total_test_cases + len(all_ways)
673
674 ok_way = lambda way: \
675 not getTestOpts().skip \
676 and (config.only == [] or name in config.only) \
677 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
678 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
679 and (not (config.skip_perf_tests and isStatsTest())) \
680 and way not in getTestOpts().omit_ways
681
682 # Which ways we are asked to skip
683 do_ways = list(filter (ok_way,all_ways))
684
685 # In fast mode, we skip all but one way
686 if config.fast and len(do_ways) > 0:
687 do_ways = [do_ways[0]]
688
689 if not config.clean_only:
690 # Run the required tests...
691 for way in do_ways:
692 if stopping():
693 break
694 do_test (name, way, func, args)
695
696 for way in all_ways:
697 if way not in do_ways:
698 skiptest (name,way)
699
700 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
701 pretest_cleanup(name)
702 clean([name + suff for suff in [
703 '', '.exe', '.exe.manifest', '.genscript',
704 '.stderr.normalised', '.stdout.normalised',
705 '.run.stderr.normalised', '.run.stdout.normalised',
706 '.comp.stderr.normalised', '.comp.stdout.normalised',
707 '.interp.stderr.normalised', '.interp.stdout.normalised',
708 '.stats', '.comp.stats',
709 '.hi', '.o', '.prof', '.exe.prof', '.hc',
710 '_stub.h', '_stub.c', '_stub.o',
711 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
712
713 if func == multi_compile or func == multi_compile_fail:
714 extra_mods = args[1]
715 clean([replace_suffix(fx[0],'o') for fx in extra_mods])
716 clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])
717
718
719 clean(getTestOpts().clean_files)
720
721 if getTestOpts().outputdir != None:
722 odir = in_testdir(getTestOpts().outputdir)
723 try:
724 shutil.rmtree(odir)
725 except:
726 pass
727
728 try:
729 shutil.rmtree(in_testdir('.hpc.' + name))
730 except:
731 pass
732
733 try:
734 cleanCmd = getTestOpts().clean_cmd
735 if cleanCmd != None:
736 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
737 if result != 0:
738 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
739 except:
740 framework_fail(name, 'cleaning', 'clean-command exception')
741
742 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
743
744 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
745 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
746
747 try:
748 for f in files_written[name]:
749 if os.path.exists(f):
750 try:
751 if not f in files_written_not_removed[name]:
752 files_written_not_removed[name].append(f)
753 except:
754 files_written_not_removed[name] = [f]
755 except:
756 pass
757 except Exception as e:
758 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
759
760 def clean(strs):
761 for str in strs:
762 for name in glob.glob(in_testdir(str)):
763 clean_full_path(name)
764
765 def clean_full_path(name):
766 try:
767 # Remove files...
768 os.remove(name)
769 except OSError as e1:
770 try:
771 # ... and empty directories
772 os.rmdir(name)
773 except OSError as e2:
774 # We don't want to fail here, but we do want to know
775 # what went wrong, so print out the exceptions.
776 # ENOENT isn't a problem, though, as we clean files
777 # that don't necessarily exist.
778 if e1.errno != errno.ENOENT:
779 print(e1)
780 if e2.errno != errno.ENOENT:
781 print(e2)
782
783 def do_test(name, way, func, args):
784 full_name = name + '(' + way + ')'
785
786 try:
787 if_verbose(2, "=====> %s %d of %d %s " % \
788 (full_name, t.total_tests, len(allTestNames), \
789 [t.n_unexpected_passes, \
790 t.n_unexpected_failures, \
791 t.n_framework_failures]))
792
793 if config.use_threads:
794 t.lock.release()
795
796 try:
797 preCmd = getTestOpts().pre_cmd
798 if preCmd != None:
799 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
800 if result != 0:
801 framework_fail(name, way, 'pre-command failed: ' + str(result))
802 except:
803 framework_fail(name, way, 'pre-command exception')
804
805 try:
806 result = func(*[name,way] + args)
807 finally:
808 if config.use_threads:
809 t.lock.acquire()
810
811 if getTestOpts().expect != 'pass' and \
812 getTestOpts().expect != 'fail' and \
813 getTestOpts().expect != 'missing-lib':
814 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
815
816 try:
817 passFail = result['passFail']
818 except:
819 passFail = 'No passFail found'
820
821 if passFail == 'pass':
822 if getTestOpts().expect == 'pass' \
823 and way not in getTestOpts().expect_fail_for:
824 t.n_expected_passes = t.n_expected_passes + 1
825 if name in t.expected_passes:
826 t.expected_passes[name].append(way)
827 else:
828 t.expected_passes[name] = [way]
829 else:
830 if_verbose(1, '*** unexpected pass for %s' % full_name)
831 t.n_unexpected_passes = t.n_unexpected_passes + 1
832 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
833 elif passFail == 'fail':
834 if getTestOpts().expect == 'pass' \
835 and way not in getTestOpts().expect_fail_for:
836 reason = result['reason']
837 tag = result.get('tag')
838 if tag == 'stat':
839 if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
840 t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
841 addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
842 else:
843 if_verbose(1, '*** unexpected failure for %s' % full_name)
844 t.n_unexpected_failures = t.n_unexpected_failures + 1
845 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
846 else:
847 if getTestOpts().expect == 'missing-lib':
848 t.n_missing_libs = t.n_missing_libs + 1
849 if name in t.missing_libs:
850 t.missing_libs[name].append(way)
851 else:
852 t.missing_libs[name] = [way]
853 else:
854 t.n_expected_failures = t.n_expected_failures + 1
855 if name in t.expected_failures:
856 t.expected_failures[name].append(way)
857 else:
858 t.expected_failures[name] = [way]
859 else:
860 framework_fail(name, way, 'bad result ' + passFail)
861 except KeyboardInterrupt:
862 stopNow()
863 except:
864 framework_fail(name, way, 'do_test exception')
865 traceback.print_exc()
866
867 def addPassingTestInfo (testInfos, directory, name, way):
868 directory = re.sub('^\\.[/\\\\]', '', directory)
869
870 if not directory in testInfos:
871 testInfos[directory] = {}
872
873 if not name in testInfos[directory]:
874 testInfos[directory][name] = []
875
876 testInfos[directory][name].append(way)
877
878 def addFailingTestInfo (testInfos, directory, name, reason, way):
879 directory = re.sub('^\\.[/\\\\]', '', directory)
880
881 if not directory in testInfos:
882 testInfos[directory] = {}
883
884 if not name in testInfos[directory]:
885 testInfos[directory][name] = {}
886
887 if not reason in testInfos[directory][name]:
888 testInfos[directory][name][reason] = []
889
890 testInfos[directory][name][reason].append(way)
891
892 def skiptest (name, way):
893 # print 'Skipping test \"', name, '\"'
894 t.n_tests_skipped = t.n_tests_skipped + 1
895 if name in t.tests_skipped:
896 t.tests_skipped[name].append(way)
897 else:
898 t.tests_skipped[name] = [way]
899
900 def framework_fail( name, way, reason ):
901 full_name = name + '(' + way + ')'
902 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
903 t.n_framework_failures = t.n_framework_failures + 1
904 if name in t.framework_failures:
905 t.framework_failures[name].append(way)
906 else:
907 t.framework_failures[name] = [way]
908
909 def badResult(result):
910 try:
911 if result['passFail'] == 'pass':
912 return False
913 return True
914 except:
915 return True
916
917 def passed():
918 return {'passFail': 'pass'}
919
920 def failBecause(reason, tag=None):
921 return {'passFail': 'fail', 'reason': reason, 'tag': tag}
922
923 # -----------------------------------------------------------------------------
924 # Generic command tests
925
926 # A generic command test is expected to run and exit successfully.
927 #
928 # The expected exit code can be changed via exit_code() as normal, and
929 # the expected stdout/stderr are stored in <testname>.stdout and
930 # <testname>.stderr. The output of the command can be ignored
931 # altogether by using run_command_ignore_output instead of
932 # run_command.
933
934 def run_command( name, way, cmd ):
935 return simple_run( name, '', cmd, '' )
936
937 # -----------------------------------------------------------------------------
938 # GHCi tests
939
940 def ghci_script_without_flag(flag):
941 def apply(name, way, script):
942 overrides = [f for f in getTestOpts().compiler_always_flags if f != flag]
943 return ghci_script_override_default_flags(overrides)(name, way, script)
944
945 return apply
946
947 def ghci_script_override_default_flags(overrides):
948 def apply(name, way, script):
949 return ghci_script(name, way, script, overrides)
950
951 return apply
952
953 def ghci_script( name, way, script, override_flags = None ):
954 # filter out -fforce-recomp from compiler_always_flags, because we're
955 # actually testing the recompilation behaviour in the GHCi tests.
956 flags = ' '.join(get_compiler_flags(override_flags, noforce=True))
957
958 way_flags = ' '.join(config.way_flags(name)['ghci'])
959
960 # We pass HC and HC_OPTS as environment variables, so that the
961 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
962 cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
963 ).format(flags=flags, way_flags=way_flags)
964
965 getTestOpts().stdin = script
966 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
967
968 # -----------------------------------------------------------------------------
969 # Compile-only tests
970
971 def compile_override_default_flags(overrides):
972 def apply(name, way, extra_opts):
973 return do_compile(name, way, 0, '', [], extra_opts, overrides)
974
975 return apply
976
977 def compile_fail_override_default_flags(overrides):
978 def apply(name, way, extra_opts):
979 return do_compile(name, way, 1, '', [], extra_opts, overrides)
980
981 return apply
982
983 def compile_without_flag(flag):
984 def apply(name, way, extra_opts):
985 overrides = [f for f in getTestOpts().compiler_always_flags if f != flag]
986 return compile_override_default_flags(overrides)(name, way, extra_opts)
987
988 return apply
989
990 def compile_fail_without_flag(flag):
991 def apply(name, way, extra_opts):
992 overrides = [f for f in getTestOpts.compiler_always_flags if f != flag]
993 return compile_fail_override_default_flags(overrides)(name, way, extra_opts)
994
995 return apply
996
997 def compile( name, way, extra_hc_opts ):
998 return do_compile( name, way, 0, '', [], extra_hc_opts )
999
1000 def compile_fail( name, way, extra_hc_opts ):
1001 return do_compile( name, way, 1, '', [], extra_hc_opts )
1002
1003 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1004 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1005
1006 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1007 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1008
1009 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1010 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1011
1012 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1013 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1014
1015 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts, override_flags = None ):
1016 # print 'Compile only, extra args = ', extra_hc_opts
1017 pretest_cleanup(name)
1018
1019 result = extras_build( way, extra_mods, extra_hc_opts )
1020 if badResult(result):
1021 return result
1022 extra_hc_opts = result['hc_opts']
1023
1024 force = 0
1025 if extra_mods:
1026 force = 1
1027 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force, override_flags )
1028
1029 if badResult(result):
1030 return result
1031
1032 # the actual stderr should always match the expected, regardless
1033 # of whether we expected the compilation to fail or not (successful
1034 # compilations may generate warnings).
1035
1036 if getTestOpts().with_namebase == None:
1037 namebase = name
1038 else:
1039 namebase = getTestOpts().with_namebase
1040
1041 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1042 actual_stderr_file = qualify(name, 'comp.stderr')
1043
1044 if not compare_outputs('stderr',
1045 join_normalisers(getTestOpts().extra_errmsg_normaliser,
1046 normalise_errmsg,
1047 normalise_whitespace),
1048 expected_stderr_file, actual_stderr_file):
1049 return failBecause('stderr mismatch')
1050
1051 # no problems found, this test passed
1052 return passed()
1053
1054 def compile_cmp_asm( name, way, extra_hc_opts ):
1055 print('Compile only, extra args = ', extra_hc_opts)
1056 pretest_cleanup(name)
1057 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1058
1059 if badResult(result):
1060 return result
1061
1062 # the actual stderr should always match the expected, regardless
1063 # of whether we expected the compilation to fail or not (successful
1064 # compilations may generate warnings).
1065
1066 if getTestOpts().with_namebase == None:
1067 namebase = name
1068 else:
1069 namebase = getTestOpts().with_namebase
1070
1071 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1072 actual_asm_file = qualify(name, 's')
1073
1074 if not compare_outputs('asm', join_normalisers(normalise_errmsg, normalise_asm), \
1075 expected_asm_file, actual_asm_file):
1076 return failBecause('asm mismatch')
1077
1078 # no problems found, this test passed
1079 return passed()
1080
1081 # -----------------------------------------------------------------------------
1082 # Compile-and-run tests
1083
1084 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1085 # print 'Compile and run, extra args = ', extra_hc_opts
1086 pretest_cleanup(name)
1087
1088 result = extras_build( way, extra_mods, extra_hc_opts )
1089 if badResult(result):
1090 return result
1091 extra_hc_opts = result['hc_opts']
1092
1093 if way == 'ghci': # interpreted...
1094 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1095 else: # compiled...
1096 force = 0
1097 if extra_mods:
1098 force = 1
1099
1100 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1101 if badResult(result):
1102 return result
1103
1104 cmd = './' + name;
1105
1106 # we don't check the compiler's stderr for a compile-and-run test
1107 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1108
1109 def compile_and_run( name, way, extra_hc_opts ):
1110 return compile_and_run__( name, way, '', [], extra_hc_opts)
1111
1112 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1113 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1114
1115 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1116 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1117
1118 def stats( name, way, stats_file ):
1119 opts = getTestOpts()
1120 return checkStats(name, way, stats_file, opts.stats_range_fields)
1121
1122 # -----------------------------------------------------------------------------
1123 # Check -t stats info
1124
1125 def checkStats(name, way, stats_file, range_fields):
1126 full_name = name + '(' + way + ')'
1127
1128 result = passed()
1129 if len(range_fields) > 0:
1130 f = open(in_testdir(stats_file))
1131 contents = f.read()
1132 f.close()
1133
1134 for (field, (expected, dev)) in range_fields.items():
1135 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1136 if m == None:
1137 print('Failed to find field: ', field)
1138 result = failBecause('no such stats field')
1139 val = int(m.group(1))
1140
1141 lowerBound = trunc( expected * ((100 - float(dev))/100))
1142 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1143
1144 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1145
1146 if val < lowerBound:
1147 print(field, 'value is too low:')
1148 print('(If this is because you have improved GHC, please')
1149 print('update the test so that GHC doesn\'t regress again)')
1150 result = failBecause('stat too good', tag='stat')
1151 if val > upperBound:
1152 print(field, 'value is too high:')
1153 result = failBecause('stat not good enough', tag='stat')
1154
1155 if val < lowerBound or val > upperBound or config.verbose >= 4:
1156 valStr = str(val)
1157 valLen = len(valStr)
1158 expectedStr = str(expected)
1159 expectedLen = len(expectedStr)
1160 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1161
1162 def display(descr, val, extra):
1163 print(descr, str(val).rjust(length), extra)
1164
1165 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1166 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1167 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1168 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1169 if val != expected:
1170 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1171
1172 return result
1173
1174 # -----------------------------------------------------------------------------
1175 # Build a single-module program
1176
1177 def extras_build( way, extra_mods, extra_hc_opts ):
1178 for modopts in extra_mods:
1179 mod, opts = modopts
1180 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1181 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1182 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1183 if badResult(result):
1184 return result
1185
1186 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1187
1188
1189 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce, override_flags = None ):
1190 opts = getTestOpts()
1191 errname = add_suffix(name, 'comp.stderr')
1192 rm_no_fail( qualify(errname, '') )
1193
1194 if top_mod != '':
1195 srcname = top_mod
1196 rm_no_fail( qualify(name, '') )
1197 base, suf = os.path.splitext(top_mod)
1198 rm_no_fail( qualify(base, '') )
1199 rm_no_fail( qualify(base, 'exe') )
1200 elif addsuf:
1201 srcname = add_hs_lhs_suffix(name)
1202 rm_no_fail( qualify(name, '') )
1203 else:
1204 srcname = name
1205 rm_no_fail( qualify(name, 'o') )
1206
1207 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1208
1209 to_do = ''
1210 if top_mod != '':
1211 to_do = '--make '
1212 if link:
1213 to_do = to_do + '-o ' + name
1214 elif link:
1215 to_do = '-o ' + name
1216 elif opts.compile_to_hc:
1217 to_do = '-C'
1218 else:
1219 to_do = '-c' # just compile
1220
1221 stats_file = name + '.comp.stats'
1222 if len(opts.compiler_stats_range_fields) > 0:
1223 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1224
1225 # Required by GHC 7.3+, harmless for earlier versions:
1226 if (getTestOpts().c_src or
1227 getTestOpts().objc_src or
1228 getTestOpts().objcpp_src or
1229 getTestOpts().cmm_src):
1230 extra_hc_opts += ' -no-hs-main '
1231
1232 if getTestOpts().compile_cmd_prefix == '':
1233 cmd_prefix = ''
1234 else:
1235 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1236
1237 flags = ' '.join(get_compiler_flags(override_flags, noforce) +
1238 config.way_flags(name)[way])
1239
1240 cmd = ('cd {opts.testdir} && {cmd_prefix} '
1241 '{{compiler}} {to_do} {srcname} {flags} {extra_hc_opts} '
1242 '> {errname} 2>&1'
1243 ).format(**locals())
1244
1245 result = runCmdFor(name, cmd)
1246
1247 if result != 0 and not should_fail:
1248 actual_stderr = qualify(name, 'comp.stderr')
1249 if_verbose(1,'Compile failed (status ' + repr(result) + ') errors were:')
1250 if_verbose_dump(1,actual_stderr)
1251
1252 # ToDo: if the sub-shell was killed by ^C, then exit
1253
1254 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1255
1256 if badResult(statsResult):
1257 return statsResult
1258
1259 if should_fail:
1260 if result == 0:
1261 return failBecause('exit code 0')
1262 else:
1263 if result != 0:
1264 return failBecause('exit code non-0')
1265
1266 return passed()
1267
1268 # -----------------------------------------------------------------------------
1269 # Run a program and check its output
1270 #
1271 # If testname.stdin exists, route input from that, else
1272 # from /dev/null. Route output to testname.run.stdout and
1273 # testname.run.stderr. Returns the exit code of the run.
1274
1275 def simple_run( name, way, prog, args ):
1276 opts = getTestOpts()
1277
1278 # figure out what to use for stdin
1279 if opts.stdin != '':
1280 use_stdin = opts.stdin
1281 else:
1282 stdin_file = add_suffix(name, 'stdin')
1283 if os.path.exists(in_testdir(stdin_file)):
1284 use_stdin = stdin_file
1285 else:
1286 use_stdin = '/dev/null'
1287
1288 run_stdout = add_suffix(name,'run.stdout')
1289 run_stderr = add_suffix(name,'run.stderr')
1290
1291 rm_no_fail(qualify(name,'run.stdout'))
1292 rm_no_fail(qualify(name,'run.stderr'))
1293 rm_no_fail(qualify(name, 'hp'))
1294 rm_no_fail(qualify(name,'ps'))
1295 rm_no_fail(qualify(name, 'prof'))
1296
1297 my_rts_flags = rts_flags(way)
1298
1299 stats_file = name + '.stats'
1300 if len(opts.stats_range_fields) > 0:
1301 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1302
1303 if opts.no_stdin:
1304 stdin_comes_from = ''
1305 else:
1306 stdin_comes_from = ' <' + use_stdin
1307
1308 if opts.combined_output:
1309 redirection = ' > {} 2>&1'.format(run_stdout)
1310 redirection_append = ' >> {} 2>&1'.format(run_stdout)
1311 else:
1312 redirection = ' > {} 2> {}'.format(run_stdout, run_stderr)
1313 redirection_append = ' >> {} 2>> {}'.format(run_stdout, run_stderr)
1314
1315 cmd = prog + ' ' + args + ' ' \
1316 + my_rts_flags + ' ' \
1317 + stdin_comes_from \
1318 + redirection
1319
1320 if opts.cmd_wrapper != None:
1321 cmd = opts.cmd_wrapper(cmd) + redirection_append
1322
1323 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1324
1325 # run the command
1326 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1327
1328 exit_code = result >> 8
1329 signal = result & 0xff
1330
1331 # check the exit code
1332 if exit_code != opts.exit_code:
1333 print('Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')')
1334 dump_stdout(name)
1335 dump_stderr(name)
1336 return failBecause('bad exit code')
1337
1338 check_hp = my_rts_flags.find("-h") != -1
1339 check_prof = my_rts_flags.find("-p") != -1
1340
1341 if not opts.ignore_output:
1342 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1343 bad_stdout = not check_stdout_ok(name)
1344 if bad_stderr:
1345 return failBecause('bad stderr')
1346 if bad_stdout:
1347 return failBecause('bad stdout')
1348 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1349 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1350 return failBecause('bad heap profile')
1351 if check_prof and not check_prof_ok(name):
1352 return failBecause('bad profile')
1353
1354 return checkStats(name, way, stats_file, opts.stats_range_fields)
1355
1356 def rts_flags(way):
1357 if (way == ''):
1358 return ''
1359 else:
1360 args = config.way_rts_flags[way]
1361
1362 if args == []:
1363 return ''
1364 else:
1365 return '+RTS ' + ' '.join(args) + ' -RTS'
1366
1367 # -----------------------------------------------------------------------------
1368 # Run a program in the interpreter and check its output
1369
1370 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1371 outname = add_suffix(name, 'interp.stdout')
1372 errname = add_suffix(name, 'interp.stderr')
1373 rm_no_fail(outname)
1374 rm_no_fail(errname)
1375 rm_no_fail(name)
1376
1377 if (top_mod == ''):
1378 srcname = add_hs_lhs_suffix(name)
1379 else:
1380 srcname = top_mod
1381
1382 scriptname = add_suffix(name, 'genscript')
1383 qscriptname = in_testdir(scriptname)
1384 rm_no_fail(qscriptname)
1385
1386 delimiter = '===== program output begins here\n'
1387
1388 script = open(qscriptname, 'w')
1389 if not compile_only:
1390 # set the prog name and command-line args to match the compiled
1391 # environment.
1392 script.write(':set prog ' + name + '\n')
1393 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1394 # Add marker lines to the stdout and stderr output files, so we
1395 # can separate GHCi's output from the program's.
1396 script.write(':! echo ' + delimiter)
1397 script.write(':! echo 1>&2 ' + delimiter)
1398 # Set stdout to be line-buffered to match the compiled environment.
1399 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1400 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1401 # in the event of an exception as for the compiled program.
1402 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1403 script.close()
1404
1405 # figure out what to use for stdin
1406 if getTestOpts().stdin != '':
1407 stdin_file = in_testdir(getTestOpts().stdin)
1408 else:
1409 stdin_file = qualify(name, 'stdin')
1410
1411 if os.path.exists(stdin_file):
1412 stdin = open(stdin_file, 'r')
1413 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1414
1415 script.close()
1416
1417 flags = ' '.join(get_compiler_flags(override_flags=None, noforce=False) +
1418 config.way_flags(name)[way])
1419
1420 if getTestOpts().combined_output:
1421 redirection = ' > {} 2>&1'.format(outname)
1422 redirection_append = ' >> {} 2>&1'.format(outname)
1423 else:
1424 redirection = ' > {} 2> {}'.format(outname, errname)
1425 redirection_append = ' >> {} 2>> {}'.format(outname, errname)
1426
1427 cmd = ('{{compiler}} {srcname} {flags} {extra_hc_opts} '
1428 '< {scriptname} {redirection}'
1429 ).format(**locals())
1430
1431 if getTestOpts().cmd_wrapper != None:
1432 cmd = getTestOpts().cmd_wrapper(cmd) + redirection_append;
1433
1434 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1435
1436 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1437
1438 exit_code = result >> 8
1439 signal = result & 0xff
1440
1441 # split the stdout into compilation/program output
1442 split_file(in_testdir(outname), delimiter,
1443 qualify(name, 'comp.stdout'),
1444 qualify(name, 'run.stdout'))
1445 split_file(in_testdir(errname), delimiter,
1446 qualify(name, 'comp.stderr'),
1447 qualify(name, 'run.stderr'))
1448
1449 # check the exit code
1450 if exit_code != getTestOpts().exit_code:
1451 print('Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1452 dump_stdout(name)
1453 dump_stderr(name)
1454 return failBecause('bad exit code')
1455
1456 # ToDo: if the sub-shell was killed by ^C, then exit
1457
1458 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1459 check_stdout_ok(name)):
1460 return passed()
1461 else:
1462 return failBecause('bad stdout or stderr')
1463
1464
1465 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1466 infile = open(in_fn)
1467 out1 = open(out1_fn, 'w')
1468 out2 = open(out2_fn, 'w')
1469
1470 line = infile.readline()
1471 line = re.sub('\r', '', line) # ignore Windows EOL
1472 while (re.sub('^\s*','',line) != delimiter and line != ''):
1473 out1.write(line)
1474 line = infile.readline()
1475 line = re.sub('\r', '', line)
1476 out1.close()
1477
1478 line = infile.readline()
1479 while (line != ''):
1480 out2.write(line)
1481 line = infile.readline()
1482 out2.close()
1483
1484 # -----------------------------------------------------------------------------
1485 # Utils
1486 def get_compiler_flags(override_flags, noforce):
1487 opts = getTestOpts()
1488
1489 if override_flags is not None:
1490 flags = copy.copy(override_flags)
1491 else:
1492 flags = copy.copy(opts.compiler_always_flags)
1493
1494 if noforce:
1495 flags = [f for f in flags if f != '-fforce-recomp']
1496
1497 flags.append(opts.extra_hc_opts)
1498
1499 if opts.outputdir != None:
1500 flags.extend(["-outputdir", opts.outputdir])
1501
1502 return flags
1503
1504 def check_stdout_ok( name ):
1505 if getTestOpts().with_namebase == None:
1506 namebase = name
1507 else:
1508 namebase = getTestOpts().with_namebase
1509
1510 actual_stdout_file = qualify(name, 'run.stdout')
1511 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1512
1513 def norm(str):
1514 if platform_specific:
1515 return str
1516 else:
1517 return normalise_output(str)
1518
1519 extra_norm = join_normalisers(norm, getTestOpts().extra_normaliser)
1520
1521 check_stdout = getTestOpts().check_stdout
1522 if check_stdout:
1523 return check_stdout(actual_stdout_file, extra_norm)
1524
1525 return compare_outputs('stdout', \
1526 extra_norm, \
1527 expected_stdout_file, actual_stdout_file)
1528
1529 def dump_stdout( name ):
1530 print('Stdout:')
1531 print(read_no_crs(qualify(name, 'run.stdout')))
1532
1533 def check_stderr_ok( name ):
1534 if getTestOpts().with_namebase == None:
1535 namebase = name
1536 else:
1537 namebase = getTestOpts().with_namebase
1538
1539 actual_stderr_file = qualify(name, 'run.stderr')
1540 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1541
1542 def norm(str):
1543 if platform_specific:
1544 return str
1545 else:
1546 return normalise_errmsg(str)
1547
1548 return compare_outputs('stderr', \
1549 join_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1550 expected_stderr_file, actual_stderr_file)
1551
1552 def dump_stderr( name ):
1553 print("Stderr:")
1554 print(read_no_crs(qualify(name, 'run.stderr')))
1555
1556 def read_no_crs(file):
1557 str = ''
1558 try:
1559 h = open(file)
1560 str = h.read()
1561 h.close
1562 except:
1563 # On Windows, if the program fails very early, it seems the
1564 # files stdout/stderr are redirected to may not get created
1565 pass
1566 return re.sub('\r', '', str)
1567
1568 def write_file(file, str):
1569 h = open(file, 'w')
1570 h.write(str)
1571 h.close
1572
1573 def check_hp_ok(name):
1574
1575 # do not qualify for hp2ps because we should be in the right directory
1576 hp2psCmd = "cd " + getTestOpts().testdir + " && {hp2ps} " + name
1577
1578 hp2psResult = runCmdExitCode(hp2psCmd)
1579
1580 actual_ps_file = qualify(name, 'ps')
1581
1582 if(hp2psResult == 0):
1583 if (os.path.exists(actual_ps_file)):
1584 if gs_working:
1585 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1586 if (gsResult == 0):
1587 return (True)
1588 else:
1589 print("hp2ps output for " + name + "is not valid PostScript")
1590 else: return (True) # assume postscript is valid without ghostscript
1591 else:
1592 print("hp2ps did not generate PostScript for " + name)
1593 return (False)
1594 else:
1595 print("hp2ps error when processing heap profile for " + name)
1596 return(False)
1597
1598 def check_prof_ok(name):
1599
1600 prof_file = qualify(name,'prof')
1601
1602 if not os.path.exists(prof_file):
1603 print(prof_file + " does not exist")
1604 return(False)
1605
1606 if os.path.getsize(qualify(name,'prof')) == 0:
1607 print(prof_file + " is empty")
1608 return(False)
1609
1610 if getTestOpts().with_namebase == None:
1611 namebase = name
1612 else:
1613 namebase = getTestOpts().with_namebase
1614
1615 (platform_specific, expected_prof_file) = \
1616 platform_wordsize_qualify(namebase, 'prof.sample')
1617
1618 # sample prof file is not required
1619 if not os.path.exists(expected_prof_file):
1620 return True
1621 else:
1622 return compare_outputs('prof', \
1623 join_normalisers(normalise_whitespace,normalise_prof), \
1624 expected_prof_file, prof_file)
1625
1626 # Compare expected output to actual output, and optionally accept the
1627 # new output. Returns true if output matched or was accepted, false
1628 # otherwise.
1629 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1630 if os.path.exists(expected_file):
1631 expected_raw = read_no_crs(expected_file)
1632 # print "norm:", normaliser(expected_raw)
1633 expected_str = normaliser(expected_raw)
1634 expected_file_for_diff = expected_file
1635 else:
1636 expected_str = ''
1637 expected_file_for_diff = '/dev/null'
1638
1639 actual_raw = read_no_crs(actual_file)
1640 actual_str = normaliser(actual_raw)
1641
1642 if expected_str == actual_str:
1643 return 1
1644 else:
1645 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1646
1647 if expected_file_for_diff == '/dev/null':
1648 expected_normalised_file = '/dev/null'
1649 else:
1650 expected_normalised_file = expected_file + ".normalised"
1651 write_file(expected_normalised_file, expected_str)
1652
1653 actual_normalised_file = actual_file + ".normalised"
1654 write_file(actual_normalised_file, actual_str)
1655
1656 # Ignore whitespace when diffing. We should only get to this
1657 # point if there are non-whitespace differences
1658 #
1659 # Note we are diffing the *actual* output, not the normalised
1660 # output. The normalised output may have whitespace squashed
1661 # (including newlines) so the diff would be hard to read.
1662 # This does mean that the diff might contain changes that
1663 # would be normalised away.
1664 if (config.verbose >= 1):
1665 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1666 ' ' + actual_file )
1667
1668 # If for some reason there were no non-whitespace differences,
1669 # then do a full diff
1670 if r == 0:
1671 r = os.system( 'diff -u ' + expected_file_for_diff + \
1672 ' ' + actual_file )
1673
1674 if config.accept:
1675 if_verbose(1, 'Accepting new output.')
1676 write_file(expected_file, actual_raw)
1677 return 1
1678 else:
1679 return 0
1680
1681
1682 def normalise_whitespace( str ):
1683 # Merge contiguous whitespace characters into a single space.
1684 str = re.sub('[ \t\n]+', ' ', str)
1685 return str
1686
1687 def normalise_errmsg( str ):
1688 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1689 # the colon is there because it appears in error messages; this
1690 # hacky solution is used in place of more sophisticated filename
1691 # mangling
1692 str = re.sub('([^\\s])\\.exe', '\\1', str)
1693 # normalise slashes, minimise Windows/Unix filename differences
1694 str = re.sub('\\\\', '/', str)
1695 # The inplace ghc's are called ghc-stage[123] to avoid filename
1696 # collisions, so we need to normalise that to just "ghc"
1697 str = re.sub('ghc-stage[123]', 'ghc', str)
1698 # Error messages simetimes contain integer implementation package
1699 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1700 return str
1701
1702 # normalise a .prof file, so that we can reasonably compare it against
1703 # a sample. This doesn't compare any of the actual profiling data,
1704 # only the shape of the profile and the number of entries.
1705 def normalise_prof (str):
1706 # strip everything up to the line beginning "COST CENTRE"
1707 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1708
1709 # strip results for CAFs, these tend to change unpredictably
1710 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1711
1712 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1713 # sometimes under MAIN.
1714 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1715
1716 # We have somthing like this:
1717
1718 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1719 # k Main 204 1 0.0 0.0 0.0 0.0
1720 # foo Main 205 1 0.0 0.0 0.0 0.0
1721 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1722
1723 # then we remove all the specific profiling data, leaving only the
1724 # cost centre name, module, and entries, to end up with this:
1725
1726 # MAIN MAIN 0
1727 # k Main 1
1728 # foo Main 1
1729 # foo.bar Main 1
1730
1731 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1732 return str
1733
1734 def normalise_slashes_( str ):
1735 str = re.sub('\\\\', '/', str)
1736 return str
1737
1738 def normalise_exe_( str ):
1739 str = re.sub('\.exe', '', str)
1740 return str
1741
1742 def normalise_output( str ):
1743 # Remove a .exe extension (for Windows)
1744 # This can occur in error messages generated by the program.
1745 str = re.sub('([^\\s])\\.exe', '\\1', str)
1746 return str
1747
1748 def normalise_asm( str ):
1749 lines = str.split('\n')
1750 # Only keep instructions and labels not starting with a dot.
1751 metadata = re.compile('^[ \t]*\\..*$')
1752 out = []
1753 for line in lines:
1754 # Drop metadata directives (e.g. ".type")
1755 if not metadata.match(line):
1756 line = re.sub('@plt', '', line)
1757 instr = line.lstrip().split()
1758 # Drop empty lines.
1759 if not instr:
1760 continue
1761 # Drop operands, except for call instructions.
1762 elif instr[0] == 'call':
1763 out.append(instr[0] + ' ' + instr[1])
1764 else:
1765 out.append(instr[0])
1766 out = '\n'.join(out)
1767 return out
1768
1769 def if_verbose( n, s ):
1770 if config.verbose >= n:
1771 print(s)
1772
1773 def if_verbose_dump( n, f ):
1774 if config.verbose >= n:
1775 try:
1776 print(open(f).read())
1777 except:
1778 print('')
1779
1780 def rawSystem(cmd_and_args):
1781 # We prefer subprocess.call to os.spawnv as the latter
1782 # seems to send its arguments through a shell or something
1783 # with the Windows (non-cygwin) python. An argument "a b c"
1784 # turns into three arguments ["a", "b", "c"].
1785
1786 # However, subprocess is new in python 2.4, so fall back to
1787 # using spawnv if we don't have it
1788 cmd = cmd_and_args[0]
1789 if have_subprocess:
1790 return subprocess.call([strip_quotes(cmd)] + cmd_and_args[1:])
1791 else:
1792 return os.spawnv(os.P_WAIT, cmd, cmd_and_args)
1793
1794 # When running under native msys Python, any invocations of non-msys binaries,
1795 # including timeout.exe, will have their arguments munged according to some
1796 # heuristics, which leads to malformed command lines (#9626). The easiest way
1797 # to avoid problems is to invoke through /usr/bin/cmd which sidesteps argument
1798 # munging because it is a native msys application.
1799 def passThroughCmd(cmd_and_args):
1800 args = []
1801 # cmd needs a Windows-style path for its first argument.
1802 args.append(cmd_and_args[0].replace('/', '\\'))
1803 # Other arguments need to be quoted to deal with spaces.
1804 args.extend(['"%s"' % arg for arg in cmd_and_args[1:]])
1805 return ["cmd", "/c", " ".join(args)]
1806
1807 # Note that this doesn't handle the timeout itself; it is just used for
1808 # commands that have timeout handling built-in.
1809 def rawSystemWithTimeout(cmd_and_args):
1810 if config.os == 'mingw32' and sys.executable.startswith('/usr'):
1811 # This is only needed when running under msys python.
1812 cmd_and_args = passThroughCmd(cmd_and_args)
1813 r = rawSystem(cmd_and_args)
1814 if r == 98:
1815 # The python timeout program uses 98 to signal that ^C was pressed
1816 stopNow()
1817 return r
1818
1819 # cmd is a complex command in Bourne-shell syntax
1820 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1821 # Hence it must ultimately be run by a Bourne shell
1822 #
1823 # Mostly it invokes the command wrapped in 'timeout' thus
1824 # timeout 300 'cd . && ...blah blah'
1825 # so it's timeout's job to invoke the Bourne shell
1826 #
1827 # But watch out for the case when there is no timeout program!
1828 # Then, when using the native Python, os.system will invoke the cmd shell
1829
1830 def runCmd( cmd ):
1831 if_verbose( 3, cmd )
1832 r = 0
1833 if config.os == 'mingw32':
1834 # On MinGW, we will always have timeout
1835 assert config.timeout_prog!=''
1836
1837 if config.timeout_prog != '':
1838 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1839 else:
1840 r = os.system(cmd)
1841 return r << 8
1842
1843 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1844 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1845 cmd = cmd.format(**config.__dict__)
1846
1847 if_verbose( 3, cmd )
1848 r = 0
1849 if config.os == 'mingw32':
1850 # On MinGW, we will always have timeout
1851 assert config.timeout_prog!=''
1852 timeout = int(ceil(config.timeout * timeout_multiplier))
1853
1854 if config.timeout_prog != '':
1855 if config.check_files_written:
1856 fn = name + ".strace"
1857 r = rawSystemWithTimeout(
1858 ["strace", "-o", fn, "-fF",
1859 "-e", "creat,open,chdir,clone,vfork",
1860 config.timeout_prog, str(timeout), cmd])
1861 addTestFilesWritten(name, fn)
1862 rm_no_fail(fn)
1863 else:
1864 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1865 else:
1866 r = os.system(cmd)
1867 return r << 8
1868
1869 def runCmdExitCode( cmd ):
1870 return (runCmd(cmd) >> 8);
1871
1872
1873 # -----------------------------------------------------------------------------
1874 # checking for files being written to by multiple tests
1875
1876 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1877 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1878 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1879 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1880 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1881 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1882 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1883 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1884 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1885 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1886 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1887 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1888 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1889 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1890 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1891 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1892 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1893
1894 # Files that are read or written but shouldn't be:
1895 # * ghci_history shouldn't be read or written by tests
1896 # * things under package.conf.d shouldn't be written by tests
1897 bad_file_usages = {}
1898
1899 # Mapping from tests to the list of files that they write
1900 files_written = {}
1901
1902 # Mapping from tests to the list of files that they write but don't clean
1903 files_written_not_removed = {}
1904
1905 def add_bad_file_usage(name, file):
1906 try:
1907 if not file in bad_file_usages[name]:
1908 bad_file_usages[name].append(file)
1909 except:
1910 bad_file_usages[name] = [file]
1911
1912 def mkPath(curdir, path):
1913 # Given the current full directory is 'curdir', what is the full
1914 # path to 'path'?
1915 return os.path.realpath(os.path.join(curdir, path))
1916
1917 def addTestFilesWritten(name, fn):
1918 if config.use_threads:
1919 with t.lockFilesWritten:
1920 addTestFilesWrittenHelper(name, fn)
1921 else:
1922 addTestFilesWrittenHelper(name, fn)
1923
1924 def addTestFilesWrittenHelper(name, fn):
1925 started = False
1926 working_directories = {}
1927
1928 with open(fn, 'r') as f:
1929 for line in f:
1930 m_pid = re_strace_pid.match(line)
1931 if m_pid:
1932 pid = m_pid.group(1)
1933 content = m_pid.group(2)
1934 elif re_strace_unavailable.match(line):
1935 next
1936 else:
1937 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1938
1939 m_open = re_strace_open.match(content)
1940 m_chdir = re_strace_chdir.match(content)
1941 m_clone = re_strace_clone.match(content)
1942 m_vfork = re_strace_vfork.match(content)
1943
1944 if not started:
1945 working_directories[pid] = os.getcwd()
1946 started = True
1947
1948 if m_open:
1949 file = m_open.group(1)
1950 file = mkPath(working_directories[pid], file)
1951 if file.endswith("ghci_history"):
1952 add_bad_file_usage(name, file)
1953 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1954 flags = m_open.group(2).split('|')
1955 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1956 if re.match('package\.conf\.d', file):
1957 add_bad_file_usage(name, file)
1958 else:
1959 try:
1960 if not file in files_written[name]:
1961 files_written[name].append(file)
1962 except:
1963 files_written[name] = [file]
1964 elif 'O_RDONLY' in flags:
1965 pass
1966 else:
1967 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1968 elif m_chdir:
1969 # We optimistically assume that unfinished chdir's are going to succeed
1970 dir = m_chdir.group(1)
1971 working_directories[pid] = mkPath(working_directories[pid], dir)
1972 elif m_clone:
1973 working_directories[m_clone.group(2)] = working_directories[pid]
1974 elif m_vfork:
1975 working_directories[m_vfork.group(2)] = working_directories[pid]
1976 elif re_strace_open_resumed.match(content):
1977 pass
1978 elif re_strace_chdir_resumed.match(content):
1979 pass
1980 elif re_strace_vfork_unfinished.match(content):
1981 pass
1982 elif re_strace_clone_unfinished.match(content):
1983 pass
1984 elif re_strace_ignore_sigchild.match(content):
1985 pass
1986 elif re_strace_ignore_sigvtalarm.match(content):
1987 pass
1988 elif re_strace_ignore_sigint.match(content):
1989 pass
1990 elif re_strace_ignore_sigfpe.match(content):
1991 pass
1992 elif re_strace_ignore_sigsegv.match(content):
1993 pass
1994 elif re_strace_ignore_sigpipe.match(content):
1995 pass
1996 else:
1997 framework_fail(name, 'strace', "Can't understand strace line: " + line)
1998
1999 def checkForFilesWrittenProblems(file):
2000 foundProblem = False
2001
2002 files_written_inverted = {}
2003 for t in files_written.keys():
2004 for f in files_written[t]:
2005 try:
2006 files_written_inverted[f].append(t)
2007 except:
2008 files_written_inverted[f] = [t]
2009
2010 for f in files_written_inverted.keys():
2011 if len(files_written_inverted[f]) > 1:
2012 if not foundProblem:
2013 foundProblem = True
2014 file.write("\n")
2015 file.write("\nSome files are written by multiple tests:\n")
2016 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2017 if foundProblem:
2018 file.write("\n")
2019
2020 # -----
2021
2022 if len(files_written_not_removed) > 0:
2023 file.write("\n")
2024 file.write("\nSome files written but not removed:\n")
2025 tests = list(files_written_not_removed.keys())
2026 tests.sort()
2027 for t in tests:
2028 for f in files_written_not_removed[t]:
2029 file.write(" " + t + ": " + f + "\n")
2030 file.write("\n")
2031
2032 # -----
2033
2034 if len(bad_file_usages) > 0:
2035 file.write("\n")
2036 file.write("\nSome bad file usages:\n")
2037 tests = list(bad_file_usages.keys())
2038 tests.sort()
2039 for t in tests:
2040 for f in bad_file_usages[t]:
2041 file.write(" " + t + ": " + f + "\n")
2042 file.write("\n")
2043
2044 # -----------------------------------------------------------------------------
2045 # checking if ghostscript is available for checking the output of hp2ps
2046
2047 def genGSCmd(psfile):
2048 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2049
2050 def gsNotWorking():
2051 global gs_working
2052 print("GhostScript not available for hp2ps tests")
2053
2054 global gs_working
2055 gs_working = 0
2056 if config.have_profiling:
2057 if config.gs != '':
2058 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2059 if resultGood == 0:
2060 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps'));
2061 if resultBad != 0:
2062 print("GhostScript available for hp2ps tests")
2063 gs_working = 1;
2064 else:
2065 gsNotWorking();
2066 else:
2067 gsNotWorking();
2068 else:
2069 gsNotWorking();
2070
2071 def rm_no_fail( file ):
2072 try:
2073 os.remove( file )
2074 finally:
2075 return
2076
2077 def add_suffix( name, suffix ):
2078 if suffix == '':
2079 return name
2080 else:
2081 return name + '.' + suffix
2082
2083 def add_hs_lhs_suffix(name):
2084 if getTestOpts().c_src:
2085 return add_suffix(name, 'c')
2086 elif getTestOpts().cmm_src:
2087 return add_suffix(name, 'cmm')
2088 elif getTestOpts().objc_src:
2089 return add_suffix(name, 'm')
2090 elif getTestOpts().objcpp_src:
2091 return add_suffix(name, 'mm')
2092 elif getTestOpts().literate:
2093 return add_suffix(name, 'lhs')
2094 else:
2095 return add_suffix(name, 'hs')
2096
2097 def replace_suffix( name, suffix ):
2098 base, suf = os.path.splitext(name)
2099 return base + '.' + suffix
2100
2101 def in_testdir( name ):
2102 return (getTestOpts().testdir + '/' + name)
2103
2104 def qualify( name, suff ):
2105 return in_testdir(add_suffix(name, suff))
2106
2107
2108 # Finding the sample output. The filename is of the form
2109 #
2110 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2111 #
2112 # and we pick the most specific version available. The <version> is
2113 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2114 # more fine-grained control use if_compiler_lt().
2115 #
2116 def platform_wordsize_qualify( name, suff ):
2117
2118 basepath = qualify(name, suff)
2119
2120 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2121 for (platformSpecific, plat) in [(1, '-' + config.platform),
2122 (1, '-' + config.os),
2123 (0, '')]
2124 for ws in ['-ws-' + config.wordsize, '']
2125 for comp in ['-' + config.compiler_type, '']
2126 for vers in ['-' + config.compiler_maj_version, '']]
2127
2128 dir = glob.glob(basepath + '*')
2129 dir = [normalise_slashes_(d) for d in dir]
2130
2131 for (platformSpecific, f) in paths:
2132 if f in dir:
2133 return (platformSpecific,f)
2134
2135 return (0, basepath)
2136
2137 # Clean up prior to the test, so that we can't spuriously conclude
2138 # that it passed on the basis of old run outputs.
2139 def pretest_cleanup(name):
2140 if getTestOpts().outputdir != None:
2141 odir = in_testdir(getTestOpts().outputdir)
2142 try:
2143 shutil.rmtree(odir)
2144 except:
2145 pass
2146 os.mkdir(odir)
2147
2148 rm_no_fail(qualify(name,'interp.stderr'))
2149 rm_no_fail(qualify(name,'interp.stdout'))
2150 rm_no_fail(qualify(name,'comp.stderr'))
2151 rm_no_fail(qualify(name,'comp.stdout'))
2152 rm_no_fail(qualify(name,'run.stderr'))
2153 rm_no_fail(qualify(name,'run.stdout'))
2154 rm_no_fail(qualify(name,'tix'))
2155 rm_no_fail(qualify(name,'exe.tix'))
2156 # simple_build zaps the following:
2157 # rm_nofail(qualify("o"))
2158 # rm_nofail(qualify(""))
2159 # not interested in the return code
2160
2161 # -----------------------------------------------------------------------------
2162 # Return a list of all the files ending in '.T' below directories roots.
2163
2164 def findTFiles(roots):
2165 # It would be better to use os.walk, but that
2166 # gives backslashes on Windows, which trip the
2167 # testsuite later :-(
2168 return [filename for root in roots for filename in findTFiles_(root)]
2169
2170 def findTFiles_(path):
2171 if os.path.isdir(path):
2172 paths = [path + '/' + x for x in os.listdir(path)]
2173 return findTFiles(paths)
2174 elif path[-2:] == '.T':
2175 return [path]
2176 else:
2177 return []
2178
2179 # -----------------------------------------------------------------------------
2180 # Output a test summary to the specified file object
2181
2182 def summary(t, file):
2183
2184 file.write('\n')
2185 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
2186 file.write('OVERALL SUMMARY for test run started at '
2187 + time.strftime("%c %Z", t.start_time) + '\n'
2188 + str(datetime.timedelta(seconds=
2189 round(time.time() - time.mktime(t.start_time)))).rjust(8)
2190 + ' spent to go through\n'
2191 + repr(t.total_tests).rjust(8)
2192 + ' total tests, which gave rise to\n'
2193 + repr(t.total_test_cases).rjust(8)
2194 + ' test cases, of which\n'
2195 + repr(t.n_tests_skipped).rjust(8)
2196 + ' were skipped\n'
2197 + '\n'
2198 + repr(t.n_missing_libs).rjust(8)
2199 + ' had missing libraries\n'
2200 + repr(t.n_expected_passes).rjust(8)
2201 + ' expected passes\n'
2202 + repr(t.n_expected_failures).rjust(8)
2203 + ' expected failures\n'
2204 + '\n'
2205 + repr(t.n_framework_failures).rjust(8)
2206 + ' caused framework failures\n'
2207 + repr(t.n_unexpected_passes).rjust(8)
2208 + ' unexpected passes\n'
2209 + repr(t.n_unexpected_failures).rjust(8)
2210 + ' unexpected failures\n'
2211 + repr(t.n_unexpected_stat_failures).rjust(8)
2212 + ' unexpected stat failures\n'
2213 + '\n')
2214
2215 if t.n_unexpected_passes > 0:
2216 file.write('Unexpected passes:\n')
2217 printPassingTestInfosSummary(file, t.unexpected_passes)
2218
2219 if t.n_unexpected_failures > 0:
2220 file.write('Unexpected failures:\n')
2221 printFailingTestInfosSummary(file, t.unexpected_failures)
2222
2223 if t.n_unexpected_stat_failures > 0:
2224 file.write('Unexpected stat failures:\n')
2225 printFailingTestInfosSummary(file, t.unexpected_stat_failures)
2226
2227 if config.check_files_written:
2228 checkForFilesWrittenProblems(file)
2229
2230 if stopping():
2231 file.write('WARNING: Testsuite run was terminated early\n')
2232
2233 def printUnexpectedTests(file, testInfoss):
2234 unexpected = []
2235 for testInfos in testInfoss:
2236 directories = testInfos.keys()
2237 for directory in directories:
2238 tests = list(testInfos[directory].keys())
2239 unexpected += tests
2240 if unexpected != []:
2241 file.write('Unexpected results from:\n')
2242 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2243 file.write('\n')
2244
2245 def printPassingTestInfosSummary(file, testInfos):
2246 directories = list(testInfos.keys())
2247 directories.sort()
2248 maxDirLen = max(len(x) for x in directories)
2249 for directory in directories:
2250 tests = list(testInfos[directory].keys())
2251 tests.sort()
2252 for test in tests:
2253 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2254 ' (' + ','.join(testInfos[directory][test]) + ')\n')
2255 file.write('\n')
2256
2257 def printFailingTestInfosSummary(file, testInfos):
2258 directories = list(testInfos.keys())
2259 directories.sort()
2260 maxDirLen = max(len(d) for d in directories)
2261 for directory in directories:
2262 tests = list(testInfos[directory].keys())
2263 tests.sort()
2264 for test in tests:
2265 reasons = testInfos[directory][test].keys()
2266 for reason in reasons:
2267 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2268 ' [' + reason + ']' + \
2269 ' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
2270 file.write('\n')
2271
2272 def getStdout(cmd):
2273 if have_subprocess:
2274 p = subprocess.Popen(strip_quotes(cmd),
2275 stdout=subprocess.PIPE,
2276 stderr=subprocess.PIPE)
2277 (stdout, stderr) = p.communicate()
2278 r = p.wait()
2279 if r != 0:
2280 raise Exception("Command failed: " + str(cmd))
2281 if stderr != '':
2282 raise Exception("stderr from command: " + str(cmd))
2283 return stdout
2284 else:
2285 raise Exception("Need subprocess to get stdout, but don't have it")
2286
2287 def strip_quotes(s):
2288 # Don't wrap commands to subprocess.call/Popen in quotes.
2289 return s.strip('\'"')