Testsuite: redirect stderr to /dev/null when running GS on bad.ps
[ghc.git] / testsuite / driver / testlib.py
1 #
2 # (c) Simon Marlow 2002
3 #
4
5 from __future__ import print_function
6
7 import shutil
8 import sys
9 import os
10 import errno
11 import string
12 import re
13 import traceback
14 import time
15 import datetime
16 import copy
17 import glob
18 from math import ceil, trunc
19 import collections
20
21 have_subprocess = False
22 try:
23 import subprocess
24 have_subprocess = True
25 except:
26 print("Warning: subprocess not found, will fall back to spawnv")
27
28 from testglobals import *
29 from testutil import *
30
31 if config.use_threads:
32 import threading
33 try:
34 import thread
35 except ImportError: # Python 3
36 import _thread as thread
37
38 global wantToStop
39 wantToStop = False
40 def stopNow():
41 global wantToStop
42 wantToStop = True
43 def stopping():
44 return wantToStop
45
46 # Options valid for the current test only (these get reset to
47 # testdir_testopts after each test).
48
49 global testopts_local
50 if config.use_threads:
51 testopts_local = threading.local()
52 else:
53 class TestOpts_Local:
54 pass
55 testopts_local = TestOpts_Local()
56
57 def getTestOpts():
58 return testopts_local.x
59
60 def setLocalTestOpts(opts):
61 global testopts_local
62 testopts_local.x=opts
63
64 def isStatsTest():
65 opts = getTestOpts()
66 return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0
67
68
69 # This can be called at the top of a file of tests, to set default test options
70 # for the following tests.
71 def setTestOpts( f ):
72 global thisdir_settings
73 thisdir_settings = [thisdir_settings, f]
74
75 # -----------------------------------------------------------------------------
76 # Canned setup functions for common cases. eg. for a test you might say
77 #
78 # test('test001', normal, compile, [''])
79 #
80 # to run it without any options, but change it to
81 #
82 # test('test001', expect_fail, compile, [''])
83 #
84 # to expect failure for this test.
85
86 def normal( name, opts ):
87 return;
88
89 def skip( name, opts ):
90 opts.skip = 1
91
92 def expect_fail( name, opts ):
93 opts.expect = 'fail';
94
95 def reqlib( lib ):
96 return lambda name, opts, l=lib: _reqlib (name, opts, l )
97
98 # Cache the results of looking to see if we have a library or not.
99 # This makes quite a difference, especially on Windows.
100 have_lib = {}
101
102 def _reqlib( name, opts, lib ):
103 if lib in have_lib:
104 got_it = have_lib[lib]
105 else:
106 if have_subprocess:
107 # By preference we use subprocess, as the alternative uses
108 # /dev/null which mingw doesn't have.
109 cmd = strip_quotes(config.ghc_pkg)
110 p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
111 stdout=subprocess.PIPE,
112 stderr=subprocess.PIPE)
113 # read from stdout and stderr to avoid blocking due to
114 # buffers filling
115 p.communicate()
116 r = p.wait()
117 else:
118 r = os.system(config.ghc_pkg + ' --no-user-package-db describe '
119 + lib + ' > /dev/null 2> /dev/null')
120 got_it = r == 0
121 have_lib[lib] = got_it
122
123 if not got_it:
124 opts.expect = 'missing-lib'
125
126 def req_profiling( name, opts ):
127 if not config.have_profiling:
128 opts.expect = 'fail'
129
130 def req_shared_libs( name, opts ):
131 if not config.have_shared_libs:
132 opts.expect = 'fail'
133
134 def req_interp( name, opts ):
135 if not config.have_interp:
136 opts.expect = 'fail'
137
138 def req_smp( name, opts ):
139 if not config.have_smp:
140 opts.expect = 'fail'
141
142 def ignore_output( name, opts ):
143 opts.ignore_output = 1
144
145 def no_stdin( name, opts ):
146 opts.no_stdin = 1
147
148 def combined_output( name, opts ):
149 opts.combined_output = True
150
151 # -----
152
153 def expect_fail_for( ways ):
154 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
155
156 def _expect_fail_for( name, opts, ways ):
157 opts.expect_fail_for = ways
158
159 def expect_broken( bug ):
160 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
161
162 def _expect_broken( name, opts, bug ):
163 record_broken(name, opts, bug)
164 opts.expect = 'fail';
165
166 def expect_broken_for( bug, ways ):
167 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
168
169 def _expect_broken_for( name, opts, bug, ways ):
170 record_broken(name, opts, bug)
171 opts.expect_fail_for = ways
172
173 def record_broken(name, opts, bug):
174 global brokens
175 me = (bug, opts.testdir, name)
176 if not me in brokens:
177 brokens.append(me)
178
179 # -----
180
181 def omit_ways( ways ):
182 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
183
184 def _omit_ways( name, opts, ways ):
185 opts.omit_ways = ways
186
187 # -----
188
189 def only_ways( ways ):
190 return lambda name, opts, w=ways: _only_ways( name, opts, w )
191
192 def _only_ways( name, opts, ways ):
193 opts.only_ways = ways
194
195 # -----
196
197 def extra_ways( ways ):
198 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
199
200 def _extra_ways( name, opts, ways ):
201 opts.extra_ways = ways
202
203 # -----
204
205 def omit_compiler_types( compiler_types ):
206 return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
207
208 def _omit_compiler_types( name, opts, compiler_types ):
209 if config.compiler_type in compiler_types:
210 opts.skip = 1
211
212 # -----
213
214 def only_compiler_types( compiler_types ):
215 return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
216
217 def _only_compiler_types( name, opts, compiler_types ):
218 if config.compiler_type not in compiler_types:
219 opts.skip = 1
220
221 # -----
222
223 def set_stdin( file ):
224 return lambda name, opts, f=file: _set_stdin(name, opts, f);
225
226 def _set_stdin( name, opts, f ):
227 opts.stdin = f
228
229 # -----
230
231 def exit_code( val ):
232 return lambda name, opts, v=val: _exit_code(name, opts, v);
233
234 def _exit_code( name, opts, v ):
235 opts.exit_code = v
236
237 def signal_exit_code( val ):
238 if opsys('solaris2'):
239 return exit_code( val );
240 else:
241 # When application running on Linux receives fatal error
242 # signal, then its exit code is encoded as 128 + signal
243 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
244 # I assume that Mac OS X behaves in the same way at least Mac
245 # OS X builder behavior suggests this.
246 return exit_code( val+128 );
247
248 # -----
249
250 def timeout_multiplier( val ):
251 return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
252
253 def _timeout_multiplier( name, opts, v ):
254 opts.timeout_multiplier = v
255
256 # -----
257
258 def extra_run_opts( val ):
259 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
260
261 def _extra_run_opts( name, opts, v ):
262 opts.extra_run_opts = v
263
264 # -----
265
266 def extra_hc_opts( val ):
267 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
268
269 def _extra_hc_opts( name, opts, v ):
270 opts.extra_hc_opts = v
271
272 # -----
273
274 def extra_clean( files ):
275 return lambda name, opts, v=files: _extra_clean(name, opts, v);
276
277 def _extra_clean( name, opts, v ):
278 opts.clean_files = v
279
280 # -----
281
282 def stats_num_field( field, expecteds ):
283 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
284
285 def _stats_num_field( name, opts, field, expecteds ):
286 if field in opts.stats_range_fields:
287 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
288
289 if type(expecteds) is list:
290 for (b, expected, dev) in expecteds:
291 if b:
292 opts.stats_range_fields[field] = (expected, dev)
293 return
294 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
295
296 else:
297 (expected, dev) = expecteds
298 opts.stats_range_fields[field] = (expected, dev)
299
300 def compiler_stats_num_field( field, expecteds ):
301 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
302
303 def _compiler_stats_num_field( name, opts, field, expecteds ):
304 if field in opts.compiler_stats_range_fields:
305 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
306
307 # Compiler performance numbers change when debugging is on, making the results
308 # useless and confusing. Therefore, skip if debugging is on.
309 if compiler_debugged():
310 skip(name, opts)
311
312 for (b, expected, dev) in expecteds:
313 if b:
314 opts.compiler_stats_range_fields[field] = (expected, dev)
315 return
316
317 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
318
319 # -----
320
321 def when(b, f):
322 # When list_brokens is on, we want to see all expect_broken calls,
323 # so we always do f
324 if b or config.list_broken:
325 return f
326 else:
327 return normal
328
329 def unless(b, f):
330 return when(not b, f)
331
332 def doing_ghci():
333 return 'ghci' in config.run_ways
334
335 def ghci_dynamic( ):
336 return config.ghc_dynamic
337
338 def fast():
339 return config.fast
340
341 def platform( plat ):
342 return config.platform == plat
343
344 def opsys( os ):
345 return config.os == os
346
347 def arch( arch ):
348 return config.arch == arch
349
350 def wordsize( ws ):
351 return config.wordsize == str(ws)
352
353 def msys( ):
354 return config.msys
355
356 def cygwin( ):
357 return config.cygwin
358
359 def have_vanilla( ):
360 return config.have_vanilla
361
362 def have_dynamic( ):
363 return config.have_dynamic
364
365 def have_profiling( ):
366 return config.have_profiling
367
368 def in_tree_compiler( ):
369 return config.in_tree_compiler
370
371 def compiler_type( compiler ):
372 return config.compiler_type == compiler
373
374 def compiler_lt( compiler, version ):
375 return config.compiler_type == compiler and \
376 version_lt(config.compiler_version, version)
377
378 def compiler_le( compiler, version ):
379 return config.compiler_type == compiler and \
380 version_le(config.compiler_version, version)
381
382 def compiler_gt( compiler, version ):
383 return config.compiler_type == compiler and \
384 version_gt(config.compiler_version, version)
385
386 def compiler_ge( compiler, version ):
387 return config.compiler_type == compiler and \
388 version_ge(config.compiler_version, version)
389
390 def unregisterised( ):
391 return config.unregisterised
392
393 def compiler_profiled( ):
394 return config.compiler_profiled
395
396 def compiler_debugged( ):
397 return config.compiler_debugged
398
399 def tag( t ):
400 return t in config.compiler_tags
401
402 # ---
403
404 def namebase( nb ):
405 return lambda opts, nb=nb: _namebase(opts, nb)
406
407 def _namebase( opts, nb ):
408 opts.with_namebase = nb
409
410 # ---
411
412 def high_memory_usage(name, opts):
413 opts.alone = True
414
415 # If a test is for a multi-CPU race, then running the test alone
416 # increases the chance that we'll actually see it.
417 def multi_cpu_race(name, opts):
418 opts.alone = True
419
420 # ---
421 def literate( name, opts ):
422 opts.literate = 1;
423
424 def c_src( name, opts ):
425 opts.c_src = 1;
426
427 def objc_src( name, opts ):
428 opts.objc_src = 1;
429
430 def objcpp_src( name, opts ):
431 opts.objcpp_src = 1;
432
433 def cmm_src( name, opts ):
434 opts.cmm_src = 1;
435
436 def outputdir( odir ):
437 return lambda name, opts, d=odir: _outputdir(name, opts, d)
438
439 def _outputdir( name, opts, odir ):
440 opts.outputdir = odir;
441
442 # ----
443
444 def pre_cmd( cmd ):
445 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
446
447 def _pre_cmd( name, opts, cmd ):
448 opts.pre_cmd = cmd
449
450 # ----
451
452 def clean_cmd( cmd ):
453 return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
454
455 def _clean_cmd( name, opts, cmd ):
456 opts.clean_cmd = cmd
457
458 # ----
459
460 def cmd_prefix( prefix ):
461 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
462
463 def _cmd_prefix( name, opts, prefix ):
464 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
465
466 # ----
467
468 def cmd_wrapper( fun ):
469 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
470
471 def _cmd_wrapper( name, opts, fun ):
472 opts.cmd_wrapper = fun
473
474 # ----
475
476 def compile_cmd_prefix( prefix ):
477 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
478
479 def _compile_cmd_prefix( name, opts, prefix ):
480 opts.compile_cmd_prefix = prefix
481
482 # ----
483
484 def check_stdout( f ):
485 return lambda name, opts, f=f: _check_stdout(name, opts, f)
486
487 def _check_stdout( name, opts, f ):
488 opts.check_stdout = f
489
490 # ----
491
492 def normalise_slashes( name, opts ):
493 _normalise_fun(name, opts, normalise_slashes_)
494
495 def normalise_exe( name, opts ):
496 _normalise_fun(name, opts, normalise_exe_)
497
498 def normalise_fun( *fs ):
499 return lambda name, opts: _normalise_fun(name, opts, fs)
500
501 def _normalise_fun( name, opts, *fs ):
502 opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
503
504 def normalise_errmsg_fun( *fs ):
505 return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
506
507 def _normalise_errmsg_fun( name, opts, *fs ):
508 opts.extra_errmsg_normaliser = join_normalisers(opts.extra_errmsg_normaliser, fs)
509
510 def normalise_version_( *pkgs ):
511 def normalise_version__( str ):
512 return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
513 '\\1-<VERSION>', str)
514 return normalise_version__
515
516 def normalise_version( *pkgs ):
517 def normalise_version__( name, opts ):
518 _normalise_fun(name, opts, normalise_version_(*pkgs))
519 _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
520 return normalise_version__
521
522 def join_normalisers(*a):
523 """
524 Compose functions, flattening sequences.
525
526 join_normalisers(f1,[f2,f3],f4)
527
528 is the same as
529
530 lambda x: f1(f2(f3(f4(x))))
531 """
532
533 def flatten(l):
534 """
535 Taken from http://stackoverflow.com/a/2158532/946226
536 """
537 for el in l:
538 if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
539 for sub in flatten(el):
540 yield sub
541 else:
542 yield el
543
544 a = flatten(a)
545
546 fn = lambda x:x # identity function
547 for f in a:
548 assert callable(f)
549 fn = lambda x,f=f,fn=fn: fn(f(x))
550 return fn
551
552 # ----
553 # Function for composing two opt-fns together
554
555 def executeSetups(fs, name, opts):
556 if type(fs) is list:
557 # If we have a list of setups, then execute each one
558 for f in fs:
559 executeSetups(f, name, opts)
560 else:
561 # fs is a single function, so just apply it
562 fs(name, opts)
563
564 # -----------------------------------------------------------------------------
565 # The current directory of tests
566
567 def newTestDir( dir ):
568 global thisdir_settings
569 # reset the options for this test directory
570 thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )
571
572 def _newTestDir( name, opts, dir ):
573 opts.testdir = dir
574 opts.compiler_always_flags = config.compiler_always_flags
575
576 # -----------------------------------------------------------------------------
577 # Actually doing tests
578
579 parallelTests = []
580 aloneTests = []
581 allTestNames = set([])
582
583 def runTest (opts, name, func, args):
584 ok = 0
585
586 if config.use_threads:
587 t.thread_pool.acquire()
588 try:
589 while config.threads<(t.running_threads+1):
590 t.thread_pool.wait()
591 t.running_threads = t.running_threads+1
592 ok=1
593 t.thread_pool.release()
594 thread.start_new_thread(test_common_thread, (name, opts, func, args))
595 except:
596 if not ok:
597 t.thread_pool.release()
598 else:
599 test_common_work (name, opts, func, args)
600
601 # name :: String
602 # setup :: TestOpts -> IO ()
603 def test (name, setup, func, args):
604 global aloneTests
605 global parallelTests
606 global allTestNames
607 global thisdir_settings
608 if name in allTestNames:
609 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
610 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
611 framework_fail(name, 'bad_name', 'This test has an invalid name')
612
613 # Make a deep copy of the default_testopts, as we need our own copy
614 # of any dictionaries etc inside it. Otherwise, if one test modifies
615 # them, all tests will see the modified version!
616 myTestOpts = copy.deepcopy(default_testopts)
617
618 executeSetups([thisdir_settings, setup], name, myTestOpts)
619
620 thisTest = lambda : runTest(myTestOpts, name, func, args)
621 if myTestOpts.alone:
622 aloneTests.append(thisTest)
623 else:
624 parallelTests.append(thisTest)
625 allTestNames.add(name)
626
627 if config.use_threads:
628 def test_common_thread(name, opts, func, args):
629 t.lock.acquire()
630 try:
631 test_common_work(name,opts,func,args)
632 finally:
633 t.lock.release()
634 t.thread_pool.acquire()
635 t.running_threads = t.running_threads - 1
636 t.thread_pool.notify()
637 t.thread_pool.release()
638
639 def get_package_cache_timestamp():
640 if config.package_conf_cache_file == '':
641 return 0.0
642 else:
643 try:
644 return os.stat(config.package_conf_cache_file).st_mtime
645 except:
646 return 0.0
647
648
649 def test_common_work (name, opts, func, args):
650 try:
651 t.total_tests = t.total_tests+1
652 setLocalTestOpts(opts)
653
654 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
655
656 # All the ways we might run this test
657 if func == compile or func == multimod_compile:
658 all_ways = config.compile_ways
659 elif func == compile_and_run or func == multimod_compile_and_run:
660 all_ways = config.run_ways
661 elif func == ghci_script:
662 if 'ghci' in config.run_ways:
663 all_ways = ['ghci']
664 else:
665 all_ways = []
666 else:
667 all_ways = ['normal']
668
669 # A test itself can request extra ways by setting opts.extra_ways
670 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
671
672 t.total_test_cases = t.total_test_cases + len(all_ways)
673
674 ok_way = lambda way: \
675 not getTestOpts().skip \
676 and (config.only == [] or name in config.only) \
677 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
678 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
679 and (not (config.skip_perf_tests and isStatsTest())) \
680 and way not in getTestOpts().omit_ways
681
682 # Which ways we are asked to skip
683 do_ways = list(filter (ok_way,all_ways))
684
685 # In fast mode, we skip all but one way
686 if config.fast and len(do_ways) > 0:
687 do_ways = [do_ways[0]]
688
689 if not config.clean_only:
690 # Run the required tests...
691 for way in do_ways:
692 if stopping():
693 break
694 do_test (name, way, func, args)
695
696 for way in all_ways:
697 if way not in do_ways:
698 skiptest (name,way)
699
700 if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
701 pretest_cleanup(name)
702 clean([name + suff for suff in [
703 '', '.exe', '.exe.manifest', '.genscript',
704 '.stderr.normalised', '.stdout.normalised',
705 '.run.stderr.normalised', '.run.stdout.normalised',
706 '.comp.stderr.normalised', '.comp.stdout.normalised',
707 '.interp.stderr.normalised', '.interp.stdout.normalised',
708 '.stats', '.comp.stats',
709 '.hi', '.o', '.prof', '.exe.prof', '.hc',
710 '_stub.h', '_stub.c', '_stub.o',
711 '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
712
713 if func == multi_compile or func == multi_compile_fail:
714 extra_mods = args[1]
715 clean([replace_suffix(fx[0],'o') for fx in extra_mods])
716 clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])
717
718
719 clean(getTestOpts().clean_files)
720
721 if getTestOpts().outputdir != None:
722 odir = in_testdir(getTestOpts().outputdir)
723 try:
724 shutil.rmtree(odir)
725 except:
726 pass
727
728 try:
729 shutil.rmtree(in_testdir('.hpc.' + name))
730 except:
731 pass
732
733 try:
734 cleanCmd = getTestOpts().clean_cmd
735 if cleanCmd != None:
736 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
737 if result != 0:
738 framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
739 except:
740 framework_fail(name, 'cleaning', 'clean-command exception')
741
742 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
743
744 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
745 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
746
747 try:
748 for f in files_written[name]:
749 if os.path.exists(f):
750 try:
751 if not f in files_written_not_removed[name]:
752 files_written_not_removed[name].append(f)
753 except:
754 files_written_not_removed[name] = [f]
755 except:
756 pass
757 except Exception as e:
758 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
759
760 def clean(strs):
761 for str in strs:
762 for name in glob.glob(in_testdir(str)):
763 clean_full_path(name)
764
765 def clean_full_path(name):
766 try:
767 # Remove files...
768 os.remove(name)
769 except OSError as e1:
770 try:
771 # ... and empty directories
772 os.rmdir(name)
773 except OSError as e2:
774 # We don't want to fail here, but we do want to know
775 # what went wrong, so print out the exceptions.
776 # ENOENT isn't a problem, though, as we clean files
777 # that don't necessarily exist.
778 if e1.errno != errno.ENOENT:
779 print(e1)
780 if e2.errno != errno.ENOENT:
781 print(e2)
782
783 def do_test(name, way, func, args):
784 full_name = name + '(' + way + ')'
785
786 try:
787 if_verbose(2, "=====> %s %d of %d %s " % \
788 (full_name, t.total_tests, len(allTestNames), \
789 [t.n_unexpected_passes, \
790 t.n_unexpected_failures, \
791 t.n_framework_failures]))
792
793 if config.use_threads:
794 t.lock.release()
795
796 try:
797 preCmd = getTestOpts().pre_cmd
798 if preCmd != None:
799 result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
800 if result != 0:
801 framework_fail(name, way, 'pre-command failed: ' + str(result))
802 except:
803 framework_fail(name, way, 'pre-command exception')
804
805 try:
806 result = func(*[name,way] + args)
807 finally:
808 if config.use_threads:
809 t.lock.acquire()
810
811 if getTestOpts().expect != 'pass' and \
812 getTestOpts().expect != 'fail' and \
813 getTestOpts().expect != 'missing-lib':
814 framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
815
816 try:
817 passFail = result['passFail']
818 except:
819 passFail = 'No passFail found'
820
821 if passFail == 'pass':
822 if getTestOpts().expect == 'pass' \
823 and way not in getTestOpts().expect_fail_for:
824 t.n_expected_passes = t.n_expected_passes + 1
825 if name in t.expected_passes:
826 t.expected_passes[name].append(way)
827 else:
828 t.expected_passes[name] = [way]
829 else:
830 if_verbose(1, '*** unexpected pass for %s' % full_name)
831 t.n_unexpected_passes = t.n_unexpected_passes + 1
832 addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
833 elif passFail == 'fail':
834 if getTestOpts().expect == 'pass' \
835 and way not in getTestOpts().expect_fail_for:
836 reason = result['reason']
837 tag = result.get('tag')
838 if tag == 'stat':
839 if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
840 t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
841 addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
842 else:
843 if_verbose(1, '*** unexpected failure for %s' % full_name)
844 t.n_unexpected_failures = t.n_unexpected_failures + 1
845 addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
846 else:
847 if getTestOpts().expect == 'missing-lib':
848 t.n_missing_libs = t.n_missing_libs + 1
849 if name in t.missing_libs:
850 t.missing_libs[name].append(way)
851 else:
852 t.missing_libs[name] = [way]
853 else:
854 t.n_expected_failures = t.n_expected_failures + 1
855 if name in t.expected_failures:
856 t.expected_failures[name].append(way)
857 else:
858 t.expected_failures[name] = [way]
859 else:
860 framework_fail(name, way, 'bad result ' + passFail)
861 except KeyboardInterrupt:
862 stopNow()
863 except:
864 framework_fail(name, way, 'do_test exception')
865 traceback.print_exc()
866
867 def addPassingTestInfo (testInfos, directory, name, way):
868 directory = re.sub('^\\.[/\\\\]', '', directory)
869
870 if not directory in testInfos:
871 testInfos[directory] = {}
872
873 if not name in testInfos[directory]:
874 testInfos[directory][name] = []
875
876 testInfos[directory][name].append(way)
877
878 def addFailingTestInfo (testInfos, directory, name, reason, way):
879 directory = re.sub('^\\.[/\\\\]', '', directory)
880
881 if not directory in testInfos:
882 testInfos[directory] = {}
883
884 if not name in testInfos[directory]:
885 testInfos[directory][name] = {}
886
887 if not reason in testInfos[directory][name]:
888 testInfos[directory][name][reason] = []
889
890 testInfos[directory][name][reason].append(way)
891
892 def skiptest (name, way):
893 # print 'Skipping test \"', name, '\"'
894 t.n_tests_skipped = t.n_tests_skipped + 1
895 if name in t.tests_skipped:
896 t.tests_skipped[name].append(way)
897 else:
898 t.tests_skipped[name] = [way]
899
900 def framework_fail( name, way, reason ):
901 full_name = name + '(' + way + ')'
902 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
903 t.n_framework_failures = t.n_framework_failures + 1
904 if name in t.framework_failures:
905 t.framework_failures[name].append(way)
906 else:
907 t.framework_failures[name] = [way]
908
909 def badResult(result):
910 try:
911 if result['passFail'] == 'pass':
912 return False
913 return True
914 except:
915 return True
916
917 def passed():
918 return {'passFail': 'pass'}
919
920 def failBecause(reason, tag=None):
921 return {'passFail': 'fail', 'reason': reason, 'tag': tag}
922
923 # -----------------------------------------------------------------------------
924 # Generic command tests
925
926 # A generic command test is expected to run and exit successfully.
927 #
928 # The expected exit code can be changed via exit_code() as normal, and
929 # the expected stdout/stderr are stored in <testname>.stdout and
930 # <testname>.stderr. The output of the command can be ignored
931 # altogether by using run_command_ignore_output instead of
932 # run_command.
933
934 def run_command( name, way, cmd ):
935 return simple_run( name, '', cmd, '' )
936
937 # -----------------------------------------------------------------------------
938 # GHCi tests
939
940 def ghci_script_without_flag(flag):
941 def apply(name, way, script):
942 overrides = [f for f in getTestOpts().compiler_always_flags if f != flag]
943 return ghci_script_override_default_flags(overrides)(name, way, script)
944
945 return apply
946
947 def ghci_script_override_default_flags(overrides):
948 def apply(name, way, script):
949 return ghci_script(name, way, script, overrides)
950
951 return apply
952
953 def ghci_script( name, way, script, override_flags = None ):
954 # filter out -fforce-recomp from compiler_always_flags, because we're
955 # actually testing the recompilation behaviour in the GHCi tests.
956 flags = ' '.join(get_compiler_flags(override_flags, noforce=True))
957
958 way_flags = ' '.join(config.way_flags(name)['ghci'])
959
960 # We pass HC and HC_OPTS as environment variables, so that the
961 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
962 cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
963 ).format(flags=flags, way_flags=way_flags)
964
965 getTestOpts().stdin = script
966 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
967
968 # -----------------------------------------------------------------------------
969 # Compile-only tests
970
971 def compile_override_default_flags(overrides):
972 def apply(name, way, extra_opts):
973 return do_compile(name, way, 0, '', [], extra_opts, overrides)
974
975 return apply
976
977 def compile_fail_override_default_flags(overrides):
978 def apply(name, way, extra_opts):
979 return do_compile(name, way, 1, '', [], extra_opts, overrides)
980
981 return apply
982
983 def compile_without_flag(flag):
984 def apply(name, way, extra_opts):
985 overrides = [f for f in getTestOpts().compiler_always_flags if f != flag]
986 return compile_override_default_flags(overrides)(name, way, extra_opts)
987
988 return apply
989
990 def compile_fail_without_flag(flag):
991 def apply(name, way, extra_opts):
992 overrides = [f for f in getTestOpts.compiler_always_flags if f != flag]
993 return compile_fail_override_default_flags(overrides)(name, way, extra_opts)
994
995 return apply
996
997 def compile( name, way, extra_hc_opts ):
998 return do_compile( name, way, 0, '', [], extra_hc_opts )
999
1000 def compile_fail( name, way, extra_hc_opts ):
1001 return do_compile( name, way, 1, '', [], extra_hc_opts )
1002
1003 def multimod_compile( name, way, top_mod, extra_hc_opts ):
1004 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
1005
1006 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
1007 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
1008
1009 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
1010 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
1011
1012 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
1013 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
1014
1015 def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts, override_flags = None ):
1016 # print 'Compile only, extra args = ', extra_hc_opts
1017 pretest_cleanup(name)
1018
1019 result = extras_build( way, extra_mods, extra_hc_opts )
1020 if badResult(result):
1021 return result
1022 extra_hc_opts = result['hc_opts']
1023
1024 force = 0
1025 if extra_mods:
1026 force = 1
1027 result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force, override_flags )
1028
1029 if badResult(result):
1030 return result
1031
1032 # the actual stderr should always match the expected, regardless
1033 # of whether we expected the compilation to fail or not (successful
1034 # compilations may generate warnings).
1035
1036 if getTestOpts().with_namebase == None:
1037 namebase = name
1038 else:
1039 namebase = getTestOpts().with_namebase
1040
1041 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1042 actual_stderr_file = qualify(name, 'comp.stderr')
1043
1044 if not compare_outputs('stderr',
1045 join_normalisers(getTestOpts().extra_errmsg_normaliser,
1046 normalise_errmsg,
1047 normalise_whitespace),
1048 expected_stderr_file, actual_stderr_file):
1049 return failBecause('stderr mismatch')
1050
1051 # no problems found, this test passed
1052 return passed()
1053
1054 def compile_cmp_asm( name, way, extra_hc_opts ):
1055 print('Compile only, extra args = ', extra_hc_opts)
1056 pretest_cleanup(name)
1057 result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)
1058
1059 if badResult(result):
1060 return result
1061
1062 # the actual stderr should always match the expected, regardless
1063 # of whether we expected the compilation to fail or not (successful
1064 # compilations may generate warnings).
1065
1066 if getTestOpts().with_namebase == None:
1067 namebase = name
1068 else:
1069 namebase = getTestOpts().with_namebase
1070
1071 (platform_specific, expected_asm_file) = platform_wordsize_qualify(namebase, 'asm')
1072 actual_asm_file = qualify(name, 's')
1073
1074 if not compare_outputs('asm', join_normalisers(normalise_errmsg, normalise_asm), \
1075 expected_asm_file, actual_asm_file):
1076 return failBecause('asm mismatch')
1077
1078 # no problems found, this test passed
1079 return passed()
1080
1081 # -----------------------------------------------------------------------------
1082 # Compile-and-run tests
1083
1084 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
1085 # print 'Compile and run, extra args = ', extra_hc_opts
1086 pretest_cleanup(name)
1087
1088 result = extras_build( way, extra_mods, extra_hc_opts )
1089 if badResult(result):
1090 return result
1091 extra_hc_opts = result['hc_opts']
1092
1093 if way == 'ghci': # interpreted...
1094 return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
1095 else: # compiled...
1096 force = 0
1097 if extra_mods:
1098 force = 1
1099
1100 result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
1101 if badResult(result):
1102 return result
1103
1104 cmd = './' + name;
1105
1106 # we don't check the compiler's stderr for a compile-and-run test
1107 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1108
1109 def compile_and_run( name, way, extra_hc_opts ):
1110 return compile_and_run__( name, way, '', [], extra_hc_opts)
1111
1112 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1113 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1114
1115 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1116 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1117
1118 def stats( name, way, stats_file ):
1119 opts = getTestOpts()
1120 return checkStats(name, way, stats_file, opts.stats_range_fields)
1121
1122 # -----------------------------------------------------------------------------
1123 # Check -t stats info
1124
1125 def checkStats(name, way, stats_file, range_fields):
1126 full_name = name + '(' + way + ')'
1127
1128 result = passed()
1129 if len(range_fields) > 0:
1130 f = open(in_testdir(stats_file))
1131 contents = f.read()
1132 f.close()
1133
1134 for (field, (expected, dev)) in range_fields.items():
1135 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1136 if m == None:
1137 print('Failed to find field: ', field)
1138 result = failBecause('no such stats field')
1139 val = int(m.group(1))
1140
1141 lowerBound = trunc( expected * ((100 - float(dev))/100))
1142 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1143
1144 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1145
1146 if val < lowerBound:
1147 print(field, 'value is too low:')
1148 print('(If this is because you have improved GHC, please')
1149 print('update the test so that GHC doesn\'t regress again)')
1150 result = failBecause('stat too good', tag='stat')
1151 if val > upperBound:
1152 print(field, 'value is too high:')
1153 result = failBecause('stat not good enough', tag='stat')
1154
1155 if val < lowerBound or val > upperBound or config.verbose >= 4:
1156 valStr = str(val)
1157 valLen = len(valStr)
1158 expectedStr = str(expected)
1159 expectedLen = len(expectedStr)
1160 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1161
1162 def display(descr, val, extra):
1163 print(descr, str(val).rjust(length), extra)
1164
1165 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1166 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1167 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1168 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1169 if val != expected:
1170 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1171
1172 return result
1173
1174 # -----------------------------------------------------------------------------
1175 # Build a single-module program
1176
1177 def extras_build( way, extra_mods, extra_hc_opts ):
1178 for modopts in extra_mods:
1179 mod, opts = modopts
1180 result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
1181 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1182 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1183 if badResult(result):
1184 return result
1185
1186 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1187
1188
1189 def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce, override_flags = None ):
1190 opts = getTestOpts()
1191 errname = add_suffix(name, 'comp.stderr')
1192 rm_no_fail( qualify(errname, '') )
1193
1194 if top_mod != '':
1195 srcname = top_mod
1196 rm_no_fail( qualify(name, '') )
1197 base, suf = os.path.splitext(top_mod)
1198 rm_no_fail( qualify(base, '') )
1199 rm_no_fail( qualify(base, 'exe') )
1200 elif addsuf:
1201 srcname = add_hs_lhs_suffix(name)
1202 rm_no_fail( qualify(name, '') )
1203 else:
1204 srcname = name
1205 rm_no_fail( qualify(name, 'o') )
1206
1207 rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
1208
1209 to_do = ''
1210 if top_mod != '':
1211 to_do = '--make '
1212 if link:
1213 to_do = to_do + '-o ' + name
1214 elif link:
1215 to_do = '-o ' + name
1216 elif opts.compile_to_hc:
1217 to_do = '-C'
1218 else:
1219 to_do = '-c' # just compile
1220
1221 stats_file = name + '.comp.stats'
1222 if len(opts.compiler_stats_range_fields) > 0:
1223 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1224
1225 # Required by GHC 7.3+, harmless for earlier versions:
1226 if (getTestOpts().c_src or
1227 getTestOpts().objc_src or
1228 getTestOpts().objcpp_src or
1229 getTestOpts().cmm_src):
1230 extra_hc_opts += ' -no-hs-main '
1231
1232 if getTestOpts().compile_cmd_prefix == '':
1233 cmd_prefix = ''
1234 else:
1235 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1236
1237 flags = ' '.join(get_compiler_flags(override_flags, noforce) +
1238 config.way_flags(name)[way])
1239
1240 cmd = ('cd {opts.testdir} && {cmd_prefix} '
1241 '{{compiler}} {to_do} {srcname} {flags} {extra_hc_opts} '
1242 '> {errname} 2>&1'
1243 ).format(**locals())
1244
1245 result = runCmdFor(name, cmd)
1246
1247 if result != 0 and not should_fail:
1248 actual_stderr = qualify(name, 'comp.stderr')
1249 if_verbose(1,'Compile failed (status ' + repr(result) + ') errors were:')
1250 if_verbose_dump(1,actual_stderr)
1251
1252 # ToDo: if the sub-shell was killed by ^C, then exit
1253
1254 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1255
1256 if badResult(statsResult):
1257 return statsResult
1258
1259 if should_fail:
1260 if result == 0:
1261 return failBecause('exit code 0')
1262 else:
1263 if result != 0:
1264 return failBecause('exit code non-0')
1265
1266 return passed()
1267
1268 # -----------------------------------------------------------------------------
1269 # Run a program and check its output
1270 #
1271 # If testname.stdin exists, route input from that, else
1272 # from /dev/null. Route output to testname.run.stdout and
1273 # testname.run.stderr. Returns the exit code of the run.
1274
1275 def simple_run( name, way, prog, args ):
1276 opts = getTestOpts()
1277
1278 # figure out what to use for stdin
1279 if opts.stdin != '':
1280 use_stdin = opts.stdin
1281 else:
1282 stdin_file = add_suffix(name, 'stdin')
1283 if os.path.exists(in_testdir(stdin_file)):
1284 use_stdin = stdin_file
1285 else:
1286 use_stdin = '/dev/null'
1287
1288 run_stdout = add_suffix(name,'run.stdout')
1289 run_stderr = add_suffix(name,'run.stderr')
1290
1291 rm_no_fail(qualify(name,'run.stdout'))
1292 rm_no_fail(qualify(name,'run.stderr'))
1293 rm_no_fail(qualify(name, 'hp'))
1294 rm_no_fail(qualify(name,'ps'))
1295 rm_no_fail(qualify(name, 'prof'))
1296
1297 my_rts_flags = rts_flags(way)
1298
1299 stats_file = name + '.stats'
1300 if len(opts.stats_range_fields) > 0:
1301 args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1302
1303 if opts.no_stdin:
1304 stdin_comes_from = ''
1305 else:
1306 stdin_comes_from = ' <' + use_stdin
1307
1308 if opts.combined_output:
1309 redirection = ' > {0} 2>&1'.format(run_stdout)
1310 redirection_append = ' >> {0} 2>&1'.format(run_stdout)
1311 else:
1312 redirection = ' > {0} 2> {1}'.format(run_stdout, run_stderr)
1313 redirection_append = ' >> {0} 2>> {1}'.format(run_stdout, run_stderr)
1314
1315 cmd = prog + ' ' + args + ' ' \
1316 + my_rts_flags + ' ' \
1317 + stdin_comes_from \
1318 + redirection
1319
1320 if opts.cmd_wrapper != None:
1321 cmd = opts.cmd_wrapper(cmd) + redirection_append
1322
1323 cmd = 'cd ' + opts.testdir + ' && ' + cmd
1324
1325 # run the command
1326 result = runCmdFor(name, cmd, timeout_multiplier=opts.timeout_multiplier)
1327
1328 exit_code = result >> 8
1329 signal = result & 0xff
1330
1331 # check the exit code
1332 if exit_code != opts.exit_code:
1333 print('Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')')
1334 dump_stdout(name)
1335 dump_stderr(name)
1336 return failBecause('bad exit code')
1337
1338 check_hp = my_rts_flags.find("-h") != -1
1339 check_prof = my_rts_flags.find("-p") != -1
1340
1341 if not opts.ignore_output:
1342 bad_stderr = not opts.combined_output and not check_stderr_ok(name)
1343 bad_stdout = not check_stdout_ok(name)
1344 if bad_stderr:
1345 return failBecause('bad stderr')
1346 if bad_stdout:
1347 return failBecause('bad stdout')
1348 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1349 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1350 return failBecause('bad heap profile')
1351 if check_prof and not check_prof_ok(name):
1352 return failBecause('bad profile')
1353
1354 return checkStats(name, way, stats_file, opts.stats_range_fields)
1355
1356 def rts_flags(way):
1357 if (way == ''):
1358 return ''
1359 else:
1360 args = config.way_rts_flags[way]
1361
1362 if args == []:
1363 return ''
1364 else:
1365 return '+RTS ' + ' '.join(args) + ' -RTS'
1366
1367 # -----------------------------------------------------------------------------
1368 # Run a program in the interpreter and check its output
1369
1370 def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1371 outname = add_suffix(name, 'interp.stdout')
1372 errname = add_suffix(name, 'interp.stderr')
1373 rm_no_fail(outname)
1374 rm_no_fail(errname)
1375 rm_no_fail(name)
1376
1377 if (top_mod == ''):
1378 srcname = add_hs_lhs_suffix(name)
1379 else:
1380 srcname = top_mod
1381
1382 scriptname = add_suffix(name, 'genscript')
1383 qscriptname = in_testdir(scriptname)
1384 rm_no_fail(qscriptname)
1385
1386 delimiter = '===== program output begins here\n'
1387
1388 script = open(qscriptname, 'w')
1389 if not compile_only:
1390 # set the prog name and command-line args to match the compiled
1391 # environment.
1392 script.write(':set prog ' + name + '\n')
1393 script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1394 # Add marker lines to the stdout and stderr output files, so we
1395 # can separate GHCi's output from the program's.
1396 script.write(':! echo ' + delimiter)
1397 script.write(':! echo 1>&2 ' + delimiter)
1398 # Set stdout to be line-buffered to match the compiled environment.
1399 script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1400 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1401 # in the event of an exception as for the compiled program.
1402 script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1403 script.close()
1404
1405 # figure out what to use for stdin
1406 if getTestOpts().stdin != '':
1407 stdin_file = in_testdir(getTestOpts().stdin)
1408 else:
1409 stdin_file = qualify(name, 'stdin')
1410
1411 if os.path.exists(stdin_file):
1412 stdin = open(stdin_file, 'r')
1413 os.system('cat ' + stdin_file + ' >>' + qscriptname)
1414
1415 script.close()
1416
1417 flags = ' '.join(get_compiler_flags(override_flags=None, noforce=False) +
1418 config.way_flags(name)[way])
1419
1420 if getTestOpts().combined_output:
1421 redirection = ' > {0} 2>&1'.format(outname)
1422 redirection_append = ' >> {0} 2>&1'.format(outname)
1423 else:
1424 redirection = ' > {0} 2> {1}'.format(outname, errname)
1425 redirection_append = ' >> {0} 2>> {1}'.format(outname, errname)
1426
1427 cmd = ('{{compiler}} {srcname} {flags} {extra_hc_opts} '
1428 '< {scriptname} {redirection}'
1429 ).format(**locals())
1430
1431 if getTestOpts().cmd_wrapper != None:
1432 cmd = getTestOpts().cmd_wrapper(cmd) + redirection_append;
1433
1434 cmd = 'cd ' + getTestOpts().testdir + " && " + cmd
1435
1436 result = runCmdFor(name, cmd, timeout_multiplier=getTestOpts().timeout_multiplier)
1437
1438 exit_code = result >> 8
1439 signal = result & 0xff
1440
1441 # split the stdout into compilation/program output
1442 split_file(in_testdir(outname), delimiter,
1443 qualify(name, 'comp.stdout'),
1444 qualify(name, 'run.stdout'))
1445 split_file(in_testdir(errname), delimiter,
1446 qualify(name, 'comp.stderr'),
1447 qualify(name, 'run.stderr'))
1448
1449 # check the exit code
1450 if exit_code != getTestOpts().exit_code:
1451 print('Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1452 dump_stdout(name)
1453 dump_stderr(name)
1454 return failBecause('bad exit code')
1455
1456 # ToDo: if the sub-shell was killed by ^C, then exit
1457
1458 if getTestOpts().ignore_output or (check_stderr_ok(name) and
1459 check_stdout_ok(name)):
1460 return passed()
1461 else:
1462 return failBecause('bad stdout or stderr')
1463
1464
1465 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1466 infile = open(in_fn)
1467 out1 = open(out1_fn, 'w')
1468 out2 = open(out2_fn, 'w')
1469
1470 line = infile.readline()
1471 line = re.sub('\r', '', line) # ignore Windows EOL
1472 while (re.sub('^\s*','',line) != delimiter and line != ''):
1473 out1.write(line)
1474 line = infile.readline()
1475 line = re.sub('\r', '', line)
1476 out1.close()
1477
1478 line = infile.readline()
1479 while (line != ''):
1480 out2.write(line)
1481 line = infile.readline()
1482 out2.close()
1483
1484 # -----------------------------------------------------------------------------
1485 # Utils
1486 def get_compiler_flags(override_flags, noforce):
1487 opts = getTestOpts()
1488
1489 if override_flags is not None:
1490 flags = copy.copy(override_flags)
1491 else:
1492 flags = copy.copy(opts.compiler_always_flags)
1493
1494 if noforce:
1495 flags = [f for f in flags if f != '-fforce-recomp']
1496
1497 flags.append(opts.extra_hc_opts)
1498
1499 if opts.outputdir != None:
1500 flags.extend(["-outputdir", opts.outputdir])
1501
1502 return flags
1503
1504 def check_stdout_ok( name ):
1505 if getTestOpts().with_namebase == None:
1506 namebase = name
1507 else:
1508 namebase = getTestOpts().with_namebase
1509
1510 actual_stdout_file = qualify(name, 'run.stdout')
1511 (platform_specific, expected_stdout_file) = platform_wordsize_qualify(namebase, 'stdout')
1512
1513 def norm(str):
1514 if platform_specific:
1515 return str
1516 else:
1517 return normalise_output(str)
1518
1519 extra_norm = join_normalisers(norm, getTestOpts().extra_normaliser)
1520
1521 check_stdout = getTestOpts().check_stdout
1522 if check_stdout:
1523 return check_stdout(actual_stdout_file, extra_norm)
1524
1525 return compare_outputs('stdout', \
1526 extra_norm, \
1527 expected_stdout_file, actual_stdout_file)
1528
1529 def dump_stdout( name ):
1530 print('Stdout:')
1531 print(read_no_crs(qualify(name, 'run.stdout')))
1532
1533 def check_stderr_ok( name ):
1534 if getTestOpts().with_namebase == None:
1535 namebase = name
1536 else:
1537 namebase = getTestOpts().with_namebase
1538
1539 actual_stderr_file = qualify(name, 'run.stderr')
1540 (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
1541
1542 def norm(str):
1543 if platform_specific:
1544 return str
1545 else:
1546 return normalise_errmsg(str)
1547
1548 return compare_outputs('stderr', \
1549 join_normalisers(norm, getTestOpts().extra_errmsg_normaliser), \
1550 expected_stderr_file, actual_stderr_file)
1551
1552 def dump_stderr( name ):
1553 print("Stderr:")
1554 print(read_no_crs(qualify(name, 'run.stderr')))
1555
1556 def read_no_crs(file):
1557 str = ''
1558 try:
1559 h = open(file)
1560 str = h.read()
1561 h.close
1562 except:
1563 # On Windows, if the program fails very early, it seems the
1564 # files stdout/stderr are redirected to may not get created
1565 pass
1566 return re.sub('\r', '', str)
1567
1568 def write_file(file, str):
1569 h = open(file, 'w')
1570 h.write(str)
1571 h.close
1572
1573 def check_hp_ok(name):
1574
1575 # do not qualify for hp2ps because we should be in the right directory
1576 hp2psCmd = "cd " + getTestOpts().testdir + " && {hp2ps} " + name
1577
1578 hp2psResult = runCmdExitCode(hp2psCmd)
1579
1580 actual_ps_file = qualify(name, 'ps')
1581
1582 if(hp2psResult == 0):
1583 if (os.path.exists(actual_ps_file)):
1584 if gs_working:
1585 gsResult = runCmdExitCode(genGSCmd(actual_ps_file))
1586 if (gsResult == 0):
1587 return (True)
1588 else:
1589 print("hp2ps output for " + name + "is not valid PostScript")
1590 else: return (True) # assume postscript is valid without ghostscript
1591 else:
1592 print("hp2ps did not generate PostScript for " + name)
1593 return (False)
1594 else:
1595 print("hp2ps error when processing heap profile for " + name)
1596 return(False)
1597
1598 def check_prof_ok(name):
1599
1600 prof_file = qualify(name,'prof')
1601
1602 if not os.path.exists(prof_file):
1603 print(prof_file + " does not exist")
1604 return(False)
1605
1606 if os.path.getsize(qualify(name,'prof')) == 0:
1607 print(prof_file + " is empty")
1608 return(False)
1609
1610 if getTestOpts().with_namebase == None:
1611 namebase = name
1612 else:
1613 namebase = getTestOpts().with_namebase
1614
1615 (platform_specific, expected_prof_file) = \
1616 platform_wordsize_qualify(namebase, 'prof.sample')
1617
1618 # sample prof file is not required
1619 if not os.path.exists(expected_prof_file):
1620 return True
1621 else:
1622 return compare_outputs('prof', \
1623 join_normalisers(normalise_whitespace,normalise_prof), \
1624 expected_prof_file, prof_file)
1625
1626 # Compare expected output to actual output, and optionally accept the
1627 # new output. Returns true if output matched or was accepted, false
1628 # otherwise.
1629 def compare_outputs( kind, normaliser, expected_file, actual_file ):
1630 if os.path.exists(expected_file):
1631 expected_raw = read_no_crs(expected_file)
1632 # print "norm:", normaliser(expected_raw)
1633 expected_str = normaliser(expected_raw)
1634 expected_file_for_diff = expected_file
1635 else:
1636 expected_str = ''
1637 expected_file_for_diff = '/dev/null'
1638
1639 actual_raw = read_no_crs(actual_file)
1640 actual_str = normaliser(actual_raw)
1641
1642 if expected_str == actual_str:
1643 return 1
1644 else:
1645 if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
1646
1647 if expected_file_for_diff == '/dev/null':
1648 expected_normalised_file = '/dev/null'
1649 else:
1650 expected_normalised_file = expected_file + ".normalised"
1651 write_file(expected_normalised_file, expected_str)
1652
1653 actual_normalised_file = actual_file + ".normalised"
1654 write_file(actual_normalised_file, actual_str)
1655
1656 # Ignore whitespace when diffing. We should only get to this
1657 # point if there are non-whitespace differences
1658 #
1659 # Note we are diffing the *actual* output, not the normalised
1660 # output. The normalised output may have whitespace squashed
1661 # (including newlines) so the diff would be hard to read.
1662 # This does mean that the diff might contain changes that
1663 # would be normalised away.
1664 if (config.verbose >= 1):
1665 r = os.system( 'diff -uw ' + expected_file_for_diff + \
1666 ' ' + actual_file )
1667
1668 # If for some reason there were no non-whitespace differences,
1669 # then do a full diff
1670 if r == 0:
1671 r = os.system( 'diff -u ' + expected_file_for_diff + \
1672 ' ' + actual_file )
1673
1674 if config.accept:
1675 if_verbose(1, 'Accepting new output.')
1676 write_file(expected_file, actual_raw)
1677 return 1
1678 else:
1679 return 0
1680
1681
1682 def normalise_whitespace( str ):
1683 # Merge contiguous whitespace characters into a single space.
1684 str = re.sub('[ \t\n]+', ' ', str)
1685 return str
1686
1687 def normalise_errmsg( str ):
1688 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1689 # the colon is there because it appears in error messages; this
1690 # hacky solution is used in place of more sophisticated filename
1691 # mangling
1692 str = re.sub('([^\\s])\\.exe', '\\1', str)
1693 # normalise slashes, minimise Windows/Unix filename differences
1694 str = re.sub('\\\\', '/', str)
1695 # The inplace ghc's are called ghc-stage[123] to avoid filename
1696 # collisions, so we need to normalise that to just "ghc"
1697 str = re.sub('ghc-stage[123]', 'ghc', str)
1698 # Error messages simetimes contain integer implementation package
1699 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1700 return str
1701
1702 # normalise a .prof file, so that we can reasonably compare it against
1703 # a sample. This doesn't compare any of the actual profiling data,
1704 # only the shape of the profile and the number of entries.
1705 def normalise_prof (str):
1706 # strip everything up to the line beginning "COST CENTRE"
1707 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1708
1709 # strip results for CAFs, these tend to change unpredictably
1710 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1711
1712 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1713 # sometimes under MAIN.
1714 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1715
1716 # We have somthing like this:
1717
1718 # MAIN MAIN 101 0 0.0 0.0 100.0 100.0
1719 # k Main 204 1 0.0 0.0 0.0 0.0
1720 # foo Main 205 1 0.0 0.0 0.0 0.0
1721 # foo.bar Main 207 1 0.0 0.0 0.0 0.0
1722
1723 # then we remove all the specific profiling data, leaving only the
1724 # cost centre name, module, and entries, to end up with this:
1725
1726 # MAIN MAIN 0
1727 # k Main 1
1728 # foo Main 1
1729 # foo.bar Main 1
1730
1731 str = re.sub('\n([ \t]*[^ \t]+)([ \t]+[^ \t]+)([ \t]+\\d+)([ \t]+\\d+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)[ \t]+([\\d\\.]+)','\n\\1 \\2 \\4',str)
1732 return str
1733
1734 def normalise_slashes_( str ):
1735 str = re.sub('\\\\', '/', str)
1736 return str
1737
1738 def normalise_exe_( str ):
1739 str = re.sub('\.exe', '', str)
1740 return str
1741
1742 def normalise_output( str ):
1743 # Remove a .exe extension (for Windows)
1744 # This can occur in error messages generated by the program.
1745 str = re.sub('([^\\s])\\.exe', '\\1', str)
1746 return str
1747
1748 def normalise_asm( str ):
1749 lines = str.split('\n')
1750 # Only keep instructions and labels not starting with a dot.
1751 metadata = re.compile('^[ \t]*\\..*$')
1752 out = []
1753 for line in lines:
1754 # Drop metadata directives (e.g. ".type")
1755 if not metadata.match(line):
1756 line = re.sub('@plt', '', line)
1757 instr = line.lstrip().split()
1758 # Drop empty lines.
1759 if not instr:
1760 continue
1761 # Drop operands, except for call instructions.
1762 elif instr[0] == 'call':
1763 out.append(instr[0] + ' ' + instr[1])
1764 else:
1765 out.append(instr[0])
1766 out = '\n'.join(out)
1767 return out
1768
1769 def if_verbose( n, s ):
1770 if config.verbose >= n:
1771 print(s)
1772
1773 def if_verbose_dump( n, f ):
1774 if config.verbose >= n:
1775 try:
1776 print(open(f).read())
1777 except:
1778 print('')
1779
1780 def rawSystem(cmd_and_args):
1781 # We prefer subprocess.call to os.spawnv as the latter
1782 # seems to send its arguments through a shell or something
1783 # with the Windows (non-cygwin) python. An argument "a b c"
1784 # turns into three arguments ["a", "b", "c"].
1785
1786 # However, subprocess is new in python 2.4, so fall back to
1787 # using spawnv if we don't have it
1788 cmd = cmd_and_args[0]
1789 if have_subprocess:
1790 return subprocess.call([strip_quotes(cmd)] + cmd_and_args[1:])
1791 else:
1792 return os.spawnv(os.P_WAIT, cmd, cmd_and_args)
1793
1794 # When running under native msys Python, any invocations of non-msys binaries,
1795 # including timeout.exe, will have their arguments munged according to some
1796 # heuristics, which leads to malformed command lines (#9626). The easiest way
1797 # to avoid problems is to invoke through /usr/bin/cmd which sidesteps argument
1798 # munging because it is a native msys application.
1799 def passThroughCmd(cmd_and_args):
1800 args = []
1801 # cmd needs a Windows-style path for its first argument.
1802 args.append(cmd_and_args[0].replace('/', '\\'))
1803 # Other arguments need to be quoted to deal with spaces.
1804 args.extend(['"%s"' % arg for arg in cmd_and_args[1:]])
1805 return ["cmd", "/c", " ".join(args)]
1806
1807 # Note that this doesn't handle the timeout itself; it is just used for
1808 # commands that have timeout handling built-in.
1809 def rawSystemWithTimeout(cmd_and_args):
1810 if config.os == 'mingw32' and sys.executable.startswith('/usr'):
1811 # This is only needed when running under msys python.
1812 cmd_and_args = passThroughCmd(cmd_and_args)
1813 r = rawSystem(cmd_and_args)
1814 if r == 98:
1815 # The python timeout program uses 98 to signal that ^C was pressed
1816 stopNow()
1817 return r
1818
1819 # cmd is a complex command in Bourne-shell syntax
1820 # e.g (cd . && 'c:/users/simonpj/darcs/HEAD/compiler/stage1/ghc-inplace' ...etc)
1821 # Hence it must ultimately be run by a Bourne shell
1822 #
1823 # Mostly it invokes the command wrapped in 'timeout' thus
1824 # timeout 300 'cd . && ...blah blah'
1825 # so it's timeout's job to invoke the Bourne shell
1826 #
1827 # But watch out for the case when there is no timeout program!
1828 # Then, when using the native Python, os.system will invoke the cmd shell
1829
1830 def runCmd( cmd ):
1831 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1832 cmd = cmd.format(**config.__dict__)
1833
1834 if_verbose( 3, cmd )
1835 r = 0
1836 if config.os == 'mingw32':
1837 # On MinGW, we will always have timeout
1838 assert config.timeout_prog!=''
1839
1840 if config.timeout_prog != '':
1841 r = rawSystemWithTimeout([config.timeout_prog, str(config.timeout), cmd])
1842 else:
1843 r = os.system(cmd)
1844 return r << 8
1845
1846 def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
1847 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1848 cmd = cmd.format(**config.__dict__)
1849
1850 if_verbose( 3, cmd )
1851 r = 0
1852 if config.os == 'mingw32':
1853 # On MinGW, we will always have timeout
1854 assert config.timeout_prog!=''
1855 timeout = int(ceil(config.timeout * timeout_multiplier))
1856
1857 if config.timeout_prog != '':
1858 if config.check_files_written:
1859 fn = name + ".strace"
1860 r = rawSystemWithTimeout(
1861 ["strace", "-o", fn, "-fF",
1862 "-e", "creat,open,chdir,clone,vfork",
1863 config.timeout_prog, str(timeout), cmd])
1864 addTestFilesWritten(name, fn)
1865 rm_no_fail(fn)
1866 else:
1867 r = rawSystemWithTimeout([config.timeout_prog, str(timeout), cmd])
1868 else:
1869 r = os.system(cmd)
1870 return r << 8
1871
1872 def runCmdExitCode( cmd ):
1873 return (runCmd(cmd) >> 8);
1874
1875
1876 # -----------------------------------------------------------------------------
1877 # checking for files being written to by multiple tests
1878
1879 re_strace_call_end = '(\) += ([0-9]+|-1 E.*)| <unfinished ...>)$'
1880 re_strace_unavailable = re.compile('^\) += \? <unavailable>$')
1881 re_strace_pid = re.compile('^([0-9]+) +(.*)')
1882 re_strace_clone = re.compile('^(clone\(|<... clone resumed> ).*\) = ([0-9]+)$')
1883 re_strace_clone_unfinished = re.compile('^clone\( <unfinished \.\.\.>$')
1884 re_strace_vfork = re.compile('^(vfork\(\)|<\.\.\. vfork resumed> \)) += ([0-9]+)$')
1885 re_strace_vfork_unfinished = re.compile('^vfork\( <unfinished \.\.\.>$')
1886 re_strace_chdir = re.compile('^chdir\("([^"]*)"(\) += 0| <unfinished ...>)$')
1887 re_strace_chdir_resumed = re.compile('^<\.\.\. chdir resumed> \) += 0$')
1888 re_strace_open = re.compile('^open\("([^"]*)", ([A-Z_|]*)(, [0-9]+)?' + re_strace_call_end)
1889 re_strace_open_resumed = re.compile('^<... open resumed> ' + re_strace_call_end)
1890 re_strace_ignore_sigchild = re.compile('^--- SIGCHLD \(Child exited\) @ 0 \(0\) ---$')
1891 re_strace_ignore_sigvtalarm = re.compile('^--- SIGVTALRM \(Virtual timer expired\) @ 0 \(0\) ---$')
1892 re_strace_ignore_sigint = re.compile('^--- SIGINT \(Interrupt\) @ 0 \(0\) ---$')
1893 re_strace_ignore_sigfpe = re.compile('^--- SIGFPE \(Floating point exception\) @ 0 \(0\) ---$')
1894 re_strace_ignore_sigsegv = re.compile('^--- SIGSEGV \(Segmentation fault\) @ 0 \(0\) ---$')
1895 re_strace_ignore_sigpipe = re.compile('^--- SIGPIPE \(Broken pipe\) @ 0 \(0\) ---$')
1896
1897 # Files that are read or written but shouldn't be:
1898 # * ghci_history shouldn't be read or written by tests
1899 # * things under package.conf.d shouldn't be written by tests
1900 bad_file_usages = {}
1901
1902 # Mapping from tests to the list of files that they write
1903 files_written = {}
1904
1905 # Mapping from tests to the list of files that they write but don't clean
1906 files_written_not_removed = {}
1907
1908 def add_bad_file_usage(name, file):
1909 try:
1910 if not file in bad_file_usages[name]:
1911 bad_file_usages[name].append(file)
1912 except:
1913 bad_file_usages[name] = [file]
1914
1915 def mkPath(curdir, path):
1916 # Given the current full directory is 'curdir', what is the full
1917 # path to 'path'?
1918 return os.path.realpath(os.path.join(curdir, path))
1919
1920 def addTestFilesWritten(name, fn):
1921 if config.use_threads:
1922 with t.lockFilesWritten:
1923 addTestFilesWrittenHelper(name, fn)
1924 else:
1925 addTestFilesWrittenHelper(name, fn)
1926
1927 def addTestFilesWrittenHelper(name, fn):
1928 started = False
1929 working_directories = {}
1930
1931 with open(fn, 'r') as f:
1932 for line in f:
1933 m_pid = re_strace_pid.match(line)
1934 if m_pid:
1935 pid = m_pid.group(1)
1936 content = m_pid.group(2)
1937 elif re_strace_unavailable.match(line):
1938 next
1939 else:
1940 framework_fail(name, 'strace', "Can't find pid in strace line: " + line)
1941
1942 m_open = re_strace_open.match(content)
1943 m_chdir = re_strace_chdir.match(content)
1944 m_clone = re_strace_clone.match(content)
1945 m_vfork = re_strace_vfork.match(content)
1946
1947 if not started:
1948 working_directories[pid] = os.getcwd()
1949 started = True
1950
1951 if m_open:
1952 file = m_open.group(1)
1953 file = mkPath(working_directories[pid], file)
1954 if file.endswith("ghci_history"):
1955 add_bad_file_usage(name, file)
1956 elif not file in ['/dev/tty', '/dev/null'] and not file.startswith("/tmp/ghc"):
1957 flags = m_open.group(2).split('|')
1958 if 'O_WRONLY' in flags or 'O_RDWR' in flags:
1959 if re.match('package\.conf\.d', file):
1960 add_bad_file_usage(name, file)
1961 else:
1962 try:
1963 if not file in files_written[name]:
1964 files_written[name].append(file)
1965 except:
1966 files_written[name] = [file]
1967 elif 'O_RDONLY' in flags:
1968 pass
1969 else:
1970 framework_fail(name, 'strace', "Can't understand flags in open strace line: " + line)
1971 elif m_chdir:
1972 # We optimistically assume that unfinished chdir's are going to succeed
1973 dir = m_chdir.group(1)
1974 working_directories[pid] = mkPath(working_directories[pid], dir)
1975 elif m_clone:
1976 working_directories[m_clone.group(2)] = working_directories[pid]
1977 elif m_vfork:
1978 working_directories[m_vfork.group(2)] = working_directories[pid]
1979 elif re_strace_open_resumed.match(content):
1980 pass
1981 elif re_strace_chdir_resumed.match(content):
1982 pass
1983 elif re_strace_vfork_unfinished.match(content):
1984 pass
1985 elif re_strace_clone_unfinished.match(content):
1986 pass
1987 elif re_strace_ignore_sigchild.match(content):
1988 pass
1989 elif re_strace_ignore_sigvtalarm.match(content):
1990 pass
1991 elif re_strace_ignore_sigint.match(content):
1992 pass
1993 elif re_strace_ignore_sigfpe.match(content):
1994 pass
1995 elif re_strace_ignore_sigsegv.match(content):
1996 pass
1997 elif re_strace_ignore_sigpipe.match(content):
1998 pass
1999 else:
2000 framework_fail(name, 'strace', "Can't understand strace line: " + line)
2001
2002 def checkForFilesWrittenProblems(file):
2003 foundProblem = False
2004
2005 files_written_inverted = {}
2006 for t in files_written.keys():
2007 for f in files_written[t]:
2008 try:
2009 files_written_inverted[f].append(t)
2010 except:
2011 files_written_inverted[f] = [t]
2012
2013 for f in files_written_inverted.keys():
2014 if len(files_written_inverted[f]) > 1:
2015 if not foundProblem:
2016 foundProblem = True
2017 file.write("\n")
2018 file.write("\nSome files are written by multiple tests:\n")
2019 file.write(" " + f + " (" + str(files_written_inverted[f]) + ")\n")
2020 if foundProblem:
2021 file.write("\n")
2022
2023 # -----
2024
2025 if len(files_written_not_removed) > 0:
2026 file.write("\n")
2027 file.write("\nSome files written but not removed:\n")
2028 tests = list(files_written_not_removed.keys())
2029 tests.sort()
2030 for t in tests:
2031 for f in files_written_not_removed[t]:
2032 file.write(" " + t + ": " + f + "\n")
2033 file.write("\n")
2034
2035 # -----
2036
2037 if len(bad_file_usages) > 0:
2038 file.write("\n")
2039 file.write("\nSome bad file usages:\n")
2040 tests = list(bad_file_usages.keys())
2041 tests.sort()
2042 for t in tests:
2043 for f in bad_file_usages[t]:
2044 file.write(" " + t + ": " + f + "\n")
2045 file.write("\n")
2046
2047 # -----------------------------------------------------------------------------
2048 # checking if ghostscript is available for checking the output of hp2ps
2049
2050 def genGSCmd(psfile):
2051 return (config.gs + ' -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE ' + psfile);
2052
2053 def gsNotWorking():
2054 global gs_working
2055 print("GhostScript not available for hp2ps tests")
2056
2057 global gs_working
2058 gs_working = 0
2059 if config.have_profiling:
2060 if config.gs != '':
2061 resultGood = runCmdExitCode(genGSCmd(config.confdir + '/good.ps'));
2062 if resultGood == 0:
2063 resultBad = runCmdExitCode(genGSCmd(config.confdir + '/bad.ps') +
2064 ' >/dev/null 2>&1')
2065 if resultBad != 0:
2066 print("GhostScript available for hp2ps tests")
2067 gs_working = 1;
2068 else:
2069 gsNotWorking();
2070 else:
2071 gsNotWorking();
2072 else:
2073 gsNotWorking();
2074
2075 def rm_no_fail( file ):
2076 try:
2077 os.remove( file )
2078 finally:
2079 return
2080
2081 def add_suffix( name, suffix ):
2082 if suffix == '':
2083 return name
2084 else:
2085 return name + '.' + suffix
2086
2087 def add_hs_lhs_suffix(name):
2088 if getTestOpts().c_src:
2089 return add_suffix(name, 'c')
2090 elif getTestOpts().cmm_src:
2091 return add_suffix(name, 'cmm')
2092 elif getTestOpts().objc_src:
2093 return add_suffix(name, 'm')
2094 elif getTestOpts().objcpp_src:
2095 return add_suffix(name, 'mm')
2096 elif getTestOpts().literate:
2097 return add_suffix(name, 'lhs')
2098 else:
2099 return add_suffix(name, 'hs')
2100
2101 def replace_suffix( name, suffix ):
2102 base, suf = os.path.splitext(name)
2103 return base + '.' + suffix
2104
2105 def in_testdir( name ):
2106 return (getTestOpts().testdir + '/' + name)
2107
2108 def qualify( name, suff ):
2109 return in_testdir(add_suffix(name, suff))
2110
2111
2112 # Finding the sample output. The filename is of the form
2113 #
2114 # <test>.stdout[-<compiler>][-<version>][-ws-<wordsize>][-<platform>]
2115 #
2116 # and we pick the most specific version available. The <version> is
2117 # the major version of the compiler (e.g. 6.8.2 would be "6.8"). For
2118 # more fine-grained control use if_compiler_lt().
2119 #
2120 def platform_wordsize_qualify( name, suff ):
2121
2122 basepath = qualify(name, suff)
2123
2124 paths = [(platformSpecific, basepath + comp + vers + ws + plat)
2125 for (platformSpecific, plat) in [(1, '-' + config.platform),
2126 (1, '-' + config.os),
2127 (0, '')]
2128 for ws in ['-ws-' + config.wordsize, '']
2129 for comp in ['-' + config.compiler_type, '']
2130 for vers in ['-' + config.compiler_maj_version, '']]
2131
2132 dir = glob.glob(basepath + '*')
2133 dir = [normalise_slashes_(d) for d in dir]
2134
2135 for (platformSpecific, f) in paths:
2136 if f in dir:
2137 return (platformSpecific,f)
2138
2139 return (0, basepath)
2140
2141 # Clean up prior to the test, so that we can't spuriously conclude
2142 # that it passed on the basis of old run outputs.
2143 def pretest_cleanup(name):
2144 if getTestOpts().outputdir != None:
2145 odir = in_testdir(getTestOpts().outputdir)
2146 try:
2147 shutil.rmtree(odir)
2148 except:
2149 pass
2150 os.mkdir(odir)
2151
2152 rm_no_fail(qualify(name,'interp.stderr'))
2153 rm_no_fail(qualify(name,'interp.stdout'))
2154 rm_no_fail(qualify(name,'comp.stderr'))
2155 rm_no_fail(qualify(name,'comp.stdout'))
2156 rm_no_fail(qualify(name,'run.stderr'))
2157 rm_no_fail(qualify(name,'run.stdout'))
2158 rm_no_fail(qualify(name,'tix'))
2159 rm_no_fail(qualify(name,'exe.tix'))
2160 # simple_build zaps the following:
2161 # rm_nofail(qualify("o"))
2162 # rm_nofail(qualify(""))
2163 # not interested in the return code
2164
2165 # -----------------------------------------------------------------------------
2166 # Return a list of all the files ending in '.T' below directories roots.
2167
2168 def findTFiles(roots):
2169 # It would be better to use os.walk, but that
2170 # gives backslashes on Windows, which trip the
2171 # testsuite later :-(
2172 return [filename for root in roots for filename in findTFiles_(root)]
2173
2174 def findTFiles_(path):
2175 if os.path.isdir(path):
2176 paths = [path + '/' + x for x in os.listdir(path)]
2177 return findTFiles(paths)
2178 elif path[-2:] == '.T':
2179 return [path]
2180 else:
2181 return []
2182
2183 # -----------------------------------------------------------------------------
2184 # Output a test summary to the specified file object
2185
2186 def summary(t, file):
2187
2188 file.write('\n')
2189 printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
2190 file.write('OVERALL SUMMARY for test run started at '
2191 + time.strftime("%c %Z", t.start_time) + '\n'
2192 + str(datetime.timedelta(seconds=
2193 round(time.time() - time.mktime(t.start_time)))).rjust(8)
2194 + ' spent to go through\n'
2195 + repr(t.total_tests).rjust(8)
2196 + ' total tests, which gave rise to\n'
2197 + repr(t.total_test_cases).rjust(8)
2198 + ' test cases, of which\n'
2199 + repr(t.n_tests_skipped).rjust(8)
2200 + ' were skipped\n'
2201 + '\n'
2202 + repr(t.n_missing_libs).rjust(8)
2203 + ' had missing libraries\n'
2204 + repr(t.n_expected_passes).rjust(8)
2205 + ' expected passes\n'
2206 + repr(t.n_expected_failures).rjust(8)
2207 + ' expected failures\n'
2208 + '\n'
2209 + repr(t.n_framework_failures).rjust(8)
2210 + ' caused framework failures\n'
2211 + repr(t.n_unexpected_passes).rjust(8)
2212 + ' unexpected passes\n'
2213 + repr(t.n_unexpected_failures).rjust(8)
2214 + ' unexpected failures\n'
2215 + repr(t.n_unexpected_stat_failures).rjust(8)
2216 + ' unexpected stat failures\n'
2217 + '\n')
2218
2219 if t.n_unexpected_passes > 0:
2220 file.write('Unexpected passes:\n')
2221 printPassingTestInfosSummary(file, t.unexpected_passes)
2222
2223 if t.n_unexpected_failures > 0:
2224 file.write('Unexpected failures:\n')
2225 printFailingTestInfosSummary(file, t.unexpected_failures)
2226
2227 if t.n_unexpected_stat_failures > 0:
2228 file.write('Unexpected stat failures:\n')
2229 printFailingTestInfosSummary(file, t.unexpected_stat_failures)
2230
2231 if config.check_files_written:
2232 checkForFilesWrittenProblems(file)
2233
2234 if stopping():
2235 file.write('WARNING: Testsuite run was terminated early\n')
2236
2237 def printUnexpectedTests(file, testInfoss):
2238 unexpected = []
2239 for testInfos in testInfoss:
2240 directories = testInfos.keys()
2241 for directory in directories:
2242 tests = list(testInfos[directory].keys())
2243 unexpected += tests
2244 if unexpected != []:
2245 file.write('Unexpected results from:\n')
2246 file.write('TEST="' + ' '.join(unexpected) + '"\n')
2247 file.write('\n')
2248
2249 def printPassingTestInfosSummary(file, testInfos):
2250 directories = list(testInfos.keys())
2251 directories.sort()
2252 maxDirLen = max(len(x) for x in directories)
2253 for directory in directories:
2254 tests = list(testInfos[directory].keys())
2255 tests.sort()
2256 for test in tests:
2257 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2258 ' (' + ','.join(testInfos[directory][test]) + ')\n')
2259 file.write('\n')
2260
2261 def printFailingTestInfosSummary(file, testInfos):
2262 directories = list(testInfos.keys())
2263 directories.sort()
2264 maxDirLen = max(len(d) for d in directories)
2265 for directory in directories:
2266 tests = list(testInfos[directory].keys())
2267 tests.sort()
2268 for test in tests:
2269 reasons = testInfos[directory][test].keys()
2270 for reason in reasons:
2271 file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
2272 ' [' + reason + ']' + \
2273 ' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
2274 file.write('\n')
2275
2276 def getStdout(cmd_and_args):
2277 if have_subprocess:
2278 p = subprocess.Popen([strip_quotes(cmd_and_args[0])] + cmd_and_args[1:],
2279 stdout=subprocess.PIPE,
2280 stderr=subprocess.PIPE)
2281 (stdout, stderr) = p.communicate()
2282 r = p.wait()
2283 if r != 0:
2284 raise Exception("Command failed: " + str(cmd_and_args))
2285 if stderr != '':
2286 raise Exception("stderr from command: " + str(cmd_and_args))
2287 return stdout
2288 else:
2289 raise Exception("Need subprocess to get stdout, but don't have it")