Typos in comments
[ghc.git] / testsuite / driver / testlib.py
1 # coding=utf8
2 #
3 # (c) Simon Marlow 2002
4 #
5
6 from __future__ import print_function
7
8 import io
9 import shutil
10 import os
11 import errno
12 import string
13 import re
14 import traceback
15 import time
16 import datetime
17 import copy
18 import glob
19 from math import ceil, trunc
20 import collections
21 import subprocess
22
23 from testglobals import *
24 from testutil import *
25 from extra_files import extra_src_files
26
27 try:
28 basestring
29 except: # Python 3
30 basestring = (str,bytes)
31
32 if config.use_threads:
33 import threading
34 try:
35 import thread
36 except ImportError: # Python 3
37 import _thread as thread
38
39 global wantToStop
40 wantToStop = False
41 def stopNow():
42 global wantToStop
43 wantToStop = True
44 def stopping():
45 return wantToStop
46
47 # Options valid for the current test only (these get reset to
48 # testdir_testopts after each test).
49
50 global testopts_local
51 if config.use_threads:
52 testopts_local = threading.local()
53 else:
54 class TestOpts_Local:
55 pass
56 testopts_local = TestOpts_Local()
57
58 def getTestOpts():
59 return testopts_local.x
60
61 def setLocalTestOpts(opts):
62 global testopts_local
63 testopts_local.x=opts
64
65 def isStatsTest():
66 opts = getTestOpts()
67 return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
68
69
70 # This can be called at the top of a file of tests, to set default test options
71 # for the following tests.
72 def setTestOpts( f ):
73 global thisdir_settings
74 thisdir_settings = [thisdir_settings, f]
75
76 # -----------------------------------------------------------------------------
77 # Canned setup functions for common cases. eg. for a test you might say
78 #
79 # test('test001', normal, compile, [''])
80 #
81 # to run it without any options, but change it to
82 #
83 # test('test001', expect_fail, compile, [''])
84 #
85 # to expect failure for this test.
86
87 def normal( name, opts ):
88 return;
89
90 def skip( name, opts ):
91 opts.skip = 1
92
93 def expect_fail( name, opts ):
94 # The compiler, testdriver, OS or platform is missing a certain
95 # feature, and we don't plan to or can't fix it now or in the
96 # future.
97 opts.expect = 'fail';
98
99 def reqlib( lib ):
100 return lambda name, opts, l=lib: _reqlib (name, opts, l )
101
102 def stage1(name, opts):
103 # See Note [Why is there no stage1 setup function?]
104 framework_fail(name, 'stage1 setup function does not exist',
105 'add your test to testsuite/tests/stage1 instead')
106
107 # Note [Why is there no stage1 setup function?]
108 #
109 # Presumably a stage1 setup function would signal that the stage1
110 # compiler should be used to compile a test.
111 #
112 # Trouble is, the path to the compiler + the `ghc --info` settings for
113 # that compiler are currently passed in from the `make` part of the
114 # testsuite driver.
115 #
116 # Switching compilers in the Python part would be entirely too late, as
117 # all ghc_with_* settings would be wrong. See config/ghc for possible
118 # consequences (for example, config.run_ways would still be
119 # based on the default compiler, quite likely causing ./validate --slow
120 # to fail).
121 #
122 # It would be possible to let the Python part of the testsuite driver
123 # make the call to `ghc --info`, but doing so would require quite some
124 # work. Care has to be taken to not affect the run_command tests for
125 # example, as they also use the `ghc --info` settings:
126 # quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
127 #
128 # If you want a test to run using the stage1 compiler, add it to the
129 # testsuite/tests/stage1 directory. Validate runs the tests in that
130 # directory with `make stage=1`.
131
132 # Cache the results of looking to see if we have a library or not.
133 # This makes quite a difference, especially on Windows.
134 have_lib = {}
135
136 def _reqlib( name, opts, lib ):
137 if lib in have_lib:
138 got_it = have_lib[lib]
139 else:
140 cmd = strip_quotes(config.ghc_pkg)
141 p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
142 stdout=subprocess.PIPE,
143 stderr=subprocess.PIPE)
144 # read from stdout and stderr to avoid blocking due to
145 # buffers filling
146 p.communicate()
147 r = p.wait()
148 got_it = r == 0
149 have_lib[lib] = got_it
150
151 if not got_it:
152 opts.expect = 'missing-lib'
153
154 def req_haddock( name, opts ):
155 if not config.haddock:
156 opts.expect = 'missing-lib'
157
158 def req_profiling( name, opts ):
159 '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
160 if not config.have_profiling:
161 opts.expect = 'fail'
162
163 def req_shared_libs( name, opts ):
164 if not config.have_shared_libs:
165 opts.expect = 'fail'
166
167 def req_interp( name, opts ):
168 if not config.have_interp:
169 opts.expect = 'fail'
170
171 def req_smp( name, opts ):
172 if not config.have_smp:
173 opts.expect = 'fail'
174
175 def ignore_stdout(name, opts):
176 opts.ignore_stdout = True
177
178 def ignore_stderr(name, opts):
179 opts.ignore_stderr = True
180
181 def combined_output( name, opts ):
182 opts.combined_output = True
183
184 # -----
185
186 def expect_fail_for( ways ):
187 return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
188
189 def _expect_fail_for( name, opts, ways ):
190 opts.expect_fail_for = ways
191
192 def expect_broken( bug ):
193 # This test is a expected not to work due to the indicated trac bug
194 # number.
195 return lambda name, opts, b=bug: _expect_broken (name, opts, b )
196
197 def _expect_broken( name, opts, bug ):
198 record_broken(name, opts, bug)
199 opts.expect = 'fail';
200
201 def expect_broken_for( bug, ways ):
202 return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
203
204 def _expect_broken_for( name, opts, bug, ways ):
205 record_broken(name, opts, bug)
206 opts.expect_fail_for = ways
207
208 def record_broken(name, opts, bug):
209 global brokens
210 me = (bug, opts.testdir, name)
211 if not me in brokens:
212 brokens.append(me)
213
214 def _expect_pass(way):
215 # Helper function. Not intended for use in .T files.
216 opts = getTestOpts()
217 return opts.expect == 'pass' and way not in opts.expect_fail_for
218
219 # -----
220
221 def omit_ways( ways ):
222 return lambda name, opts, w=ways: _omit_ways( name, opts, w )
223
224 def _omit_ways( name, opts, ways ):
225 opts.omit_ways = ways
226
227 # -----
228
229 def only_ways( ways ):
230 return lambda name, opts, w=ways: _only_ways( name, opts, w )
231
232 def _only_ways( name, opts, ways ):
233 opts.only_ways = ways
234
235 # -----
236
237 def extra_ways( ways ):
238 return lambda name, opts, w=ways: _extra_ways( name, opts, w )
239
240 def _extra_ways( name, opts, ways ):
241 opts.extra_ways = ways
242
243 # -----
244
245 def set_stdin( file ):
246 return lambda name, opts, f=file: _set_stdin(name, opts, f);
247
248 def _set_stdin( name, opts, f ):
249 opts.stdin = f
250
251 # -----
252
253 def exit_code( val ):
254 return lambda name, opts, v=val: _exit_code(name, opts, v);
255
256 def _exit_code( name, opts, v ):
257 opts.exit_code = v
258
259 def signal_exit_code( val ):
260 if opsys('solaris2'):
261 return exit_code( val );
262 else:
263 # When application running on Linux receives fatal error
264 # signal, then its exit code is encoded as 128 + signal
265 # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
266 # I assume that Mac OS X behaves in the same way at least Mac
267 # OS X builder behavior suggests this.
268 return exit_code( val+128 );
269
270 # -----
271
272 def compile_timeout_multiplier( val ):
273 return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
274
275 def _compile_timeout_multiplier( name, opts, v ):
276 opts.compile_timeout_multiplier = v
277
278 def run_timeout_multiplier( val ):
279 return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)
280
281 def _run_timeout_multiplier( name, opts, v ):
282 opts.run_timeout_multiplier = v
283
284 # -----
285
286 def extra_run_opts( val ):
287 return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
288
289 def _extra_run_opts( name, opts, v ):
290 opts.extra_run_opts = v
291
292 # -----
293
294 def extra_hc_opts( val ):
295 return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
296
297 def _extra_hc_opts( name, opts, v ):
298 opts.extra_hc_opts = v
299
300 # -----
301
302 def extra_clean( files ):
303 # TODO. Remove all calls to extra_clean.
304 return lambda _name, _opts: None
305
306 def extra_files(files):
307 return lambda name, opts: _extra_files(name, opts, files)
308
309 def _extra_files(name, opts, files):
310 opts.extra_files.extend(files)
311
312 # -----
313
314 def stats_num_field( field, expecteds ):
315 return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);
316
317 def _stats_num_field( name, opts, field, expecteds ):
318 if field in opts.stats_range_fields:
319 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
320
321 if type(expecteds) is list:
322 for (b, expected, dev) in expecteds:
323 if b:
324 opts.stats_range_fields[field] = (expected, dev)
325 return
326 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
327
328 else:
329 (expected, dev) = expecteds
330 opts.stats_range_fields[field] = (expected, dev)
331
332 def compiler_stats_num_field( field, expecteds ):
333 return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
334
335 def _compiler_stats_num_field( name, opts, field, expecteds ):
336 if field in opts.compiler_stats_range_fields:
337 framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')
338
339 # Compiler performance numbers change when debugging is on, making the results
340 # useless and confusing. Therefore, skip if debugging is on.
341 if compiler_debugged():
342 skip(name, opts)
343
344 for (b, expected, dev) in expecteds:
345 if b:
346 opts.compiler_stats_range_fields[field] = (expected, dev)
347 return
348
349 framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
350
351 # -----
352
353 def when(b, f):
354 # When list_brokens is on, we want to see all expect_broken calls,
355 # so we always do f
356 if b or config.list_broken:
357 return f
358 else:
359 return normal
360
361 def unless(b, f):
362 return when(not b, f)
363
364 def doing_ghci():
365 return 'ghci' in config.run_ways
366
367 def ghc_dynamic():
368 return config.ghc_dynamic
369
370 def fast():
371 return config.speed == 2
372
373 def platform( plat ):
374 return config.platform == plat
375
376 def opsys( os ):
377 return config.os == os
378
379 def arch( arch ):
380 return config.arch == arch
381
382 def wordsize( ws ):
383 return config.wordsize == str(ws)
384
385 def msys( ):
386 return config.msys
387
388 def cygwin( ):
389 return config.cygwin
390
391 def have_vanilla( ):
392 return config.have_vanilla
393
394 def have_dynamic( ):
395 return config.have_dynamic
396
397 def have_profiling( ):
398 return config.have_profiling
399
400 def in_tree_compiler( ):
401 return config.in_tree_compiler
402
403 def unregisterised( ):
404 return config.unregisterised
405
406 def compiler_profiled( ):
407 return config.compiler_profiled
408
409 def compiler_debugged( ):
410 return config.compiler_debugged
411
412 # ---
413
414 def high_memory_usage(name, opts):
415 opts.alone = True
416
417 # If a test is for a multi-CPU race, then running the test alone
418 # increases the chance that we'll actually see it.
419 def multi_cpu_race(name, opts):
420 opts.alone = True
421
422 # ---
423 def literate( name, opts ):
424 opts.literate = 1;
425
426 def c_src( name, opts ):
427 opts.c_src = 1;
428
429 def objc_src( name, opts ):
430 opts.objc_src = 1;
431
432 def objcpp_src( name, opts ):
433 opts.objcpp_src = 1;
434
435 def cmm_src( name, opts ):
436 opts.cmm_src = 1;
437
438 def outputdir( odir ):
439 return lambda name, opts, d=odir: _outputdir(name, opts, d)
440
441 def _outputdir( name, opts, odir ):
442 opts.outputdir = odir;
443
444 # ----
445
446 def pre_cmd( cmd ):
447 return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
448
449 def _pre_cmd( name, opts, cmd ):
450 opts.pre_cmd = cmd
451
452 # ----
453
454 def clean_cmd( cmd ):
455 # TODO. Remove all calls to clean_cmd.
456 return lambda _name, _opts: None
457
458 # ----
459
460 def cmd_prefix( prefix ):
461 return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
462
463 def _cmd_prefix( name, opts, prefix ):
464 opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;
465
466 # ----
467
468 def cmd_wrapper( fun ):
469 return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
470
471 def _cmd_wrapper( name, opts, fun ):
472 opts.cmd_wrapper = fun
473
474 # ----
475
476 def compile_cmd_prefix( prefix ):
477 return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
478
479 def _compile_cmd_prefix( name, opts, prefix ):
480 opts.compile_cmd_prefix = prefix
481
482 # ----
483
484 def check_stdout( f ):
485 return lambda name, opts, f=f: _check_stdout(name, opts, f)
486
487 def _check_stdout( name, opts, f ):
488 opts.check_stdout = f
489
490 def no_check_hp(name, opts):
491 opts.check_hp = False
492
493 # ----
494
495 def normalise_slashes( name, opts ):
496 _normalise_fun(name, opts, normalise_slashes_)
497
498 def normalise_exe( name, opts ):
499 _normalise_fun(name, opts, normalise_exe_)
500
501 def normalise_fun( *fs ):
502 return lambda name, opts: _normalise_fun(name, opts, fs)
503
504 def _normalise_fun( name, opts, *fs ):
505 opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
506
507 def normalise_errmsg_fun( *fs ):
508 return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
509
510 def _normalise_errmsg_fun( name, opts, *fs ):
511 opts.extra_errmsg_normaliser = join_normalisers(opts.extra_errmsg_normaliser, fs)
512
513 def normalise_version_( *pkgs ):
514 def normalise_version__( str ):
515 return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
516 '\\1-<VERSION>', str)
517 return normalise_version__
518
519 def normalise_version( *pkgs ):
520 def normalise_version__( name, opts ):
521 _normalise_fun(name, opts, normalise_version_(*pkgs))
522 _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
523 return normalise_version__
524
525 def normalise_drive_letter(name, opts):
526 # Windows only. Change D:\\ to C:\\.
527 _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))
528
529 def keep_prof_callstacks(name, opts):
530 """Keep profiling callstacks.
531
532 Use together with `only_ways(prof_ways)`.
533 """
534 opts.keep_prof_callstacks = True
535
536 def join_normalisers(*a):
537 """
538 Compose functions, flattening sequences.
539
540 join_normalisers(f1,[f2,f3],f4)
541
542 is the same as
543
544 lambda x: f1(f2(f3(f4(x))))
545 """
546
547 def flatten(l):
548 """
549 Taken from http://stackoverflow.com/a/2158532/946226
550 """
551 for el in l:
552 if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
553 for sub in flatten(el):
554 yield sub
555 else:
556 yield el
557
558 a = flatten(a)
559
560 fn = lambda x:x # identity function
561 for f in a:
562 assert callable(f)
563 fn = lambda x,f=f,fn=fn: fn(f(x))
564 return fn
565
566 # ----
567 # Function for composing two opt-fns together
568
569 def executeSetups(fs, name, opts):
570 if type(fs) is list:
571 # If we have a list of setups, then execute each one
572 for f in fs:
573 executeSetups(f, name, opts)
574 else:
575 # fs is a single function, so just apply it
576 fs(name, opts)
577
578 # -----------------------------------------------------------------------------
579 # The current directory of tests
580
581 def newTestDir(tempdir, dir):
582
583 global thisdir_settings
584 # reset the options for this test directory
585 def settings(name, opts, tempdir=tempdir, dir=dir):
586 return _newTestDir(name, opts, tempdir, dir)
587 thisdir_settings = settings
588
589 # Should be equal to entry in toplevel .gitignore.
590 testdir_suffix = '.run'
591
592 def _newTestDir(name, opts, tempdir, dir):
593 opts.srcdir = os.path.join(os.getcwd(), dir)
594 opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
595 opts.compiler_always_flags = config.compiler_always_flags
596
597 # -----------------------------------------------------------------------------
598 # Actually doing tests
599
600 parallelTests = []
601 aloneTests = []
602 allTestNames = set([])
603
604 def runTest (opts, name, func, args):
605 ok = 0
606
607 if config.use_threads:
608 t.thread_pool.acquire()
609 try:
610 while config.threads<(t.running_threads+1):
611 t.thread_pool.wait()
612 t.running_threads = t.running_threads+1
613 ok=1
614 t.thread_pool.release()
615 thread.start_new_thread(test_common_thread, (name, opts, func, args))
616 except:
617 if not ok:
618 t.thread_pool.release()
619 else:
620 test_common_work (name, opts, func, args)
621
622 # name :: String
623 # setup :: TestOpts -> IO ()
624 def test (name, setup, func, args):
625 global aloneTests
626 global parallelTests
627 global allTestNames
628 global thisdir_settings
629 if name in allTestNames:
630 framework_fail(name, 'duplicate', 'There are multiple tests with this name')
631 if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
632 framework_fail(name, 'bad_name', 'This test has an invalid name')
633
634 if config.run_only_some_tests:
635 if name not in config.only:
636 return
637 else:
638 # Note [Mutating config.only]
639 # config.only is initiallly the set of tests requested by
640 # the user (via 'make TEST='). We then remove all tests that
641 # we've already seen (in .T files), so that we can later
642 # report on any tests we couldn't find and error out.
643 config.only.remove(name)
644
645 # Make a deep copy of the default_testopts, as we need our own copy
646 # of any dictionaries etc inside it. Otherwise, if one test modifies
647 # them, all tests will see the modified version!
648 myTestOpts = copy.deepcopy(default_testopts)
649
650 executeSetups([thisdir_settings, setup], name, myTestOpts)
651
652 thisTest = lambda : runTest(myTestOpts, name, func, args)
653 if myTestOpts.alone:
654 aloneTests.append(thisTest)
655 else:
656 parallelTests.append(thisTest)
657 allTestNames.add(name)
658
659 if config.use_threads:
660 def test_common_thread(name, opts, func, args):
661 t.lock.acquire()
662 try:
663 test_common_work(name,opts,func,args)
664 finally:
665 t.lock.release()
666 t.thread_pool.acquire()
667 t.running_threads = t.running_threads - 1
668 t.thread_pool.notify()
669 t.thread_pool.release()
670
671 def get_package_cache_timestamp():
672 if config.package_conf_cache_file == '':
673 return 0.0
674 else:
675 try:
676 return os.stat(config.package_conf_cache_file).st_mtime
677 except:
678 return 0.0
679
680 do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
681
682 def test_common_work (name, opts, func, args):
683 try:
684 t.total_tests += 1
685 setLocalTestOpts(opts)
686
687 package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
688
689 # All the ways we might run this test
690 if func == compile or func == multimod_compile:
691 all_ways = config.compile_ways
692 elif func == compile_and_run or func == multimod_compile_and_run:
693 all_ways = config.run_ways
694 elif func == ghci_script:
695 if 'ghci' in config.run_ways:
696 all_ways = ['ghci']
697 else:
698 all_ways = []
699 else:
700 all_ways = ['normal']
701
702 # A test itself can request extra ways by setting opts.extra_ways
703 all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
704
705 t.total_test_cases += len(all_ways)
706
707 ok_way = lambda way: \
708 not getTestOpts().skip \
709 and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
710 and (config.cmdline_ways == [] or way in config.cmdline_ways) \
711 and (not (config.skip_perf_tests and isStatsTest())) \
712 and way not in getTestOpts().omit_ways
713
714 # Which ways we are asked to skip
715 do_ways = list(filter (ok_way,all_ways))
716
717 # Only run all ways in slow mode.
718 # See Note [validate and testsuite speed] in toplevel Makefile.
719 if config.accept:
720 # Only ever run one way
721 do_ways = do_ways[:1]
722 elif config.speed > 0:
723 # However, if we EXPLICITLY asked for a way (with extra_ways)
724 # please test it!
725 explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
726 other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
727 do_ways = other_ways[:1] + explicit_ways
728
729 # Find all files in the source directory that this test
730 # depends on. Do this only once for all ways.
731 # Generously add all filenames that start with the name of
732 # the test to this set, as a convenience to test authors.
733 # They will have to use the `extra_files` setup function to
734 # specify all other files that their test depends on (but
735 # this seems to be necessary for only about 10% of all
736 # tests).
737 files = set(f for f in os.listdir(opts.srcdir)
738 if f.startswith(name) and not f == name and
739 not f.endswith(testdir_suffix) and
740 not os.path.splitext(f)[1] in do_not_copy)
741 for filename in (opts.extra_files + extra_src_files.get(name, [])):
742 if filename.startswith('/'):
743 framework_fail(name, 'whole-test',
744 'no absolute paths in extra_files please: ' + filename)
745
746 elif '*' in filename:
747 # Don't use wildcards in extra_files too much, as
748 # globbing is slow.
749 files.update((os.path.relpath(f, opts.srcdir)
750 for f in glob.iglob(in_srcdir(filename))))
751
752 elif filename:
753 files.add(filename)
754
755 else:
756 framework_fail(name, 'whole-test', 'extra_file is empty string')
757
758 # Run the required tests...
759 for way in do_ways:
760 if stopping():
761 break
762 try:
763 do_test(name, way, func, args, files)
764 except KeyboardInterrupt:
765 stopNow()
766 except Exception as e:
767 framework_fail(name, way, str(e))
768 traceback.print_exc()
769
770 t.n_tests_skipped += len(set(all_ways) - set(do_ways))
771
772 if config.cleanup and do_ways:
773 cleanup()
774
775 package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
776
777 if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
778 framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
779
780 except Exception as e:
781 framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
782
783 def do_test(name, way, func, args, files):
784 opts = getTestOpts()
785
786 full_name = name + '(' + way + ')'
787
788 if_verbose(2, "=====> {0} {1} of {2} {3}".format(
789 full_name, t.total_tests, len(allTestNames),
790 [len(t.unexpected_passes),
791 len(t.unexpected_failures),
792 len(t.framework_failures)]))
793
794 # Clean up prior to the test, so that we can't spuriously conclude
795 # that it passed on the basis of old run outputs.
796 cleanup()
797 os.makedirs(opts.testdir)
798
799 # Link all source files for this test into a new directory in
800 # /tmp, and run the test in that directory. This makes it
801 # possible to run tests in parallel, without modification, that
802 # would otherwise (accidentally) write to the same output file.
803 # It also makes it easier to keep the testsuite clean.
804
805 for extra_file in files:
806 src = in_srcdir(extra_file)
807 dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
808 if os.path.isfile(src):
809 link_or_copy_file(src, dst)
810 elif os.path.isdir(src):
811 os.mkdir(dst)
812 lndir(src, dst)
813 else:
814 if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
815 # When using a ghc built without haddock support, .t
816 # files are rightfully missing. Don't
817 # framework_fail. Test will be skipped later.
818 pass
819 else:
820 framework_fail(name, way,
821 'extra_file does not exist: ' + extra_file)
822
823 if func.__name__ == 'run_command' or opts.pre_cmd:
824 # When running 'MAKE' make sure 'TOP' still points to the
825 # root of the testsuite.
826 src_makefile = in_srcdir('Makefile')
827 dst_makefile = in_testdir('Makefile')
828 if os.path.exists(src_makefile):
829 with io.open(src_makefile, 'r', encoding='utf8') as src:
830 makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
831 with io.open(dst_makefile, 'w', encoding='utf8') as dst:
832 dst.write(makefile)
833
834 if config.use_threads:
835 t.lock.release()
836
837 if opts.pre_cmd:
838 exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, opts.pre_cmd))
839 if exit_code != 0:
840 framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
841
842 try:
843 result = func(*[name,way] + args)
844 finally:
845 if config.use_threads:
846 t.lock.acquire()
847
848 if opts.expect not in ['pass', 'fail', 'missing-lib']:
849 framework_fail(name, way, 'bad expected ' + opts.expect)
850
851 try:
852 passFail = result['passFail']
853 except:
854 passFail = 'No passFail found'
855
856 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
857
858 if passFail == 'pass':
859 if _expect_pass(way):
860 t.n_expected_passes += 1
861 else:
862 if_verbose(1, '*** unexpected pass for %s' % full_name)
863 t.unexpected_passes.append((directory, name, 'unexpected', way))
864 elif passFail == 'fail':
865 if _expect_pass(way):
866 reason = result['reason']
867 tag = result.get('tag')
868 if tag == 'stat':
869 if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
870 t.unexpected_stat_failures.append((directory, name, reason, way))
871 else:
872 if_verbose(1, '*** unexpected failure for %s' % full_name)
873 t.unexpected_failures.append((directory, name, reason, way))
874 else:
875 if opts.expect == 'missing-lib':
876 t.missing_libs.append((directory, name, 'missing-lib', way))
877 else:
878 t.n_expected_failures += 1
879 else:
880 framework_fail(name, way, 'bad result ' + passFail)
881
882 def framework_fail(name, way, reason):
883 opts = getTestOpts()
884 directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
885 full_name = name + '(' + way + ')'
886 if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
887 t.framework_failures.append((directory, name, way, reason))
888
889 def badResult(result):
890 try:
891 if result['passFail'] == 'pass':
892 return False
893 return True
894 except:
895 return True
896
897 def passed():
898 return {'passFail': 'pass'}
899
900 def failBecause(reason, tag=None):
901 return {'passFail': 'fail', 'reason': reason, 'tag': tag}
902
903 # -----------------------------------------------------------------------------
904 # Generic command tests
905
906 # A generic command test is expected to run and exit successfully.
907 #
908 # The expected exit code can be changed via exit_code() as normal, and
909 # the expected stdout/stderr are stored in <testname>.stdout and
910 # <testname>.stderr. The output of the command can be ignored
911 # altogether by using the setup function ignore_stdout instead of
912 # run_command.
913
914 def run_command( name, way, cmd ):
915 return simple_run( name, '', cmd, '' )
916
917 # -----------------------------------------------------------------------------
918 # GHCi tests
919
920 def ghci_script( name, way, script):
921 flags = ' '.join(get_compiler_flags())
922 way_flags = ' '.join(config.way_flags[way])
923
924 # We pass HC and HC_OPTS as environment variables, so that the
925 # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
926 cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
927 ).format(flags=flags, way_flags=way_flags)
928
929 getTestOpts().stdin = script
930 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
931
932 # -----------------------------------------------------------------------------
933 # Compile-only tests
934
935 def compile( name, way, extra_hc_opts ):
936 return do_compile( name, way, 0, '', [], extra_hc_opts )
937
938 def compile_fail( name, way, extra_hc_opts ):
939 return do_compile( name, way, 1, '', [], extra_hc_opts )
940
941 def backpack_typecheck( name, way, extra_hc_opts ):
942 return do_compile( name, way, 0, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
943
944 def backpack_typecheck_fail( name, way, extra_hc_opts ):
945 return do_compile( name, way, 1, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
946
947 def backpack_compile( name, way, extra_hc_opts ):
948 return do_compile( name, way, 0, '', [], extra_hc_opts, backpack=1 )
949
950 def backpack_compile_fail( name, way, extra_hc_opts ):
951 return do_compile( name, way, 1, '', [], extra_hc_opts, backpack=1 )
952
953 def backpack_run( name, way, extra_hc_opts ):
954 return compile_and_run__( name, way, '', [], extra_hc_opts, backpack=1 )
955
956 def multimod_compile( name, way, top_mod, extra_hc_opts ):
957 return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
958
959 def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
960 return do_compile( name, way, 1, top_mod, [], extra_hc_opts )
961
962 def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
963 return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)
964
965 def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
966 return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
967
968 def do_compile(name, way, should_fail, top_mod, extra_mods, extra_hc_opts, **kwargs):
969 # print 'Compile only, extra args = ', extra_hc_opts
970
971 result = extras_build( way, extra_mods, extra_hc_opts )
972 if badResult(result):
973 return result
974 extra_hc_opts = result['hc_opts']
975
976 result = simple_build(name, way, extra_hc_opts, should_fail, top_mod, 0, 1, **kwargs)
977
978 if badResult(result):
979 return result
980
981 # the actual stderr should always match the expected, regardless
982 # of whether we expected the compilation to fail or not (successful
983 # compilations may generate warnings).
984
985 expected_stderr_file = find_expected_file(name, 'stderr')
986 actual_stderr_file = add_suffix(name, 'comp.stderr')
987
988 if not compare_outputs(way, 'stderr',
989 join_normalisers(getTestOpts().extra_errmsg_normaliser,
990 normalise_errmsg),
991 expected_stderr_file, actual_stderr_file,
992 whitespace_normaliser=normalise_whitespace):
993 return failBecause('stderr mismatch')
994
995 # no problems found, this test passed
996 return passed()
997
998 def compile_cmp_asm( name, way, extra_hc_opts ):
999 print('Compile only, extra args = ', extra_hc_opts)
1000 result = simple_build(name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0)
1001
1002 if badResult(result):
1003 return result
1004
1005 # the actual stderr should always match the expected, regardless
1006 # of whether we expected the compilation to fail or not (successful
1007 # compilations may generate warnings).
1008
1009 expected_asm_file = find_expected_file(name, 'asm')
1010 actual_asm_file = add_suffix(name, 's')
1011
1012 if not compare_outputs(way, 'asm',
1013 join_normalisers(normalise_errmsg, normalise_asm),
1014 expected_asm_file, actual_asm_file):
1015 return failBecause('asm mismatch')
1016
1017 # no problems found, this test passed
1018 return passed()
1019
1020 # -----------------------------------------------------------------------------
1021 # Compile-and-run tests
1022
1023 def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts, backpack=0 ):
1024 # print 'Compile and run, extra args = ', extra_hc_opts
1025
1026 result = extras_build( way, extra_mods, extra_hc_opts )
1027 if badResult(result):
1028 return result
1029 extra_hc_opts = result['hc_opts']
1030
1031 if way.startswith('ghci'): # interpreted...
1032 return interpreter_run(name, way, extra_hc_opts, top_mod)
1033 else: # compiled...
1034 result = simple_build(name, way, extra_hc_opts, 0, top_mod, 1, 1, backpack = backpack)
1035 if badResult(result):
1036 return result
1037
1038 cmd = './' + name;
1039
1040 # we don't check the compiler's stderr for a compile-and-run test
1041 return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
1042
1043 def compile_and_run( name, way, extra_hc_opts ):
1044 return compile_and_run__( name, way, '', [], extra_hc_opts)
1045
1046 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
1047 return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
1048
1049 def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
1050 return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
1051
1052 def stats( name, way, stats_file ):
1053 opts = getTestOpts()
1054 return checkStats(name, way, stats_file, opts.stats_range_fields)
1055
1056 # -----------------------------------------------------------------------------
1057 # Check -t stats info
1058
1059 def checkStats(name, way, stats_file, range_fields):
1060 full_name = name + '(' + way + ')'
1061
1062 result = passed()
1063 if range_fields:
1064 try:
1065 f = open(in_testdir(stats_file))
1066 except IOError as e:
1067 return failBecause(str(e))
1068 contents = f.read()
1069 f.close()
1070
1071 for (field, (expected, dev)) in range_fields.items():
1072 m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
1073 if m == None:
1074 print('Failed to find field: ', field)
1075 result = failBecause('no such stats field')
1076 val = int(m.group(1))
1077
1078 lowerBound = trunc( expected * ((100 - float(dev))/100))
1079 upperBound = trunc(0.5 + ceil(expected * ((100 + float(dev))/100)))
1080
1081 deviation = round(((float(val) * 100)/ expected) - 100, 1)
1082
1083 if val < lowerBound:
1084 print(field, 'value is too low:')
1085 print('(If this is because you have improved GHC, please')
1086 print('update the test so that GHC doesn\'t regress again)')
1087 result = failBecause('stat too good', tag='stat')
1088 if val > upperBound:
1089 print(field, 'value is too high:')
1090 result = failBecause('stat not good enough', tag='stat')
1091
1092 if val < lowerBound or val > upperBound or config.verbose >= 4:
1093 length = max(len(str(x)) for x in [expected, lowerBound, upperBound, val])
1094
1095 def display(descr, val, extra):
1096 print(descr, str(val).rjust(length), extra)
1097
1098 display(' Expected ' + full_name + ' ' + field + ':', expected, '+/-' + str(dev) + '%')
1099 display(' Lower bound ' + full_name + ' ' + field + ':', lowerBound, '')
1100 display(' Upper bound ' + full_name + ' ' + field + ':', upperBound, '')
1101 display(' Actual ' + full_name + ' ' + field + ':', val, '')
1102 if val != expected:
1103 display(' Deviation ' + full_name + ' ' + field + ':', deviation, '%')
1104
1105 return result
1106
1107 # -----------------------------------------------------------------------------
1108 # Build a single-module program
1109
1110 def extras_build( way, extra_mods, extra_hc_opts ):
1111 for mod, opts in extra_mods:
1112 result = simple_build(mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0)
1113 if not (mod.endswith('.hs') or mod.endswith('.lhs')):
1114 extra_hc_opts += ' ' + replace_suffix(mod, 'o')
1115 if badResult(result):
1116 return result
1117
1118 return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}
1119
1120 def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, backpack = False):
1121 opts = getTestOpts()
1122
1123 # Redirect stdout and stderr to the same file
1124 stdout = in_testdir(name, 'comp.stderr')
1125 stderr = subprocess.STDOUT
1126
1127 if top_mod != '':
1128 srcname = top_mod
1129 elif addsuf:
1130 if backpack:
1131 srcname = add_suffix(name, 'bkp')
1132 else:
1133 srcname = add_hs_lhs_suffix(name)
1134 else:
1135 srcname = name
1136
1137 if top_mod != '':
1138 to_do = '--make '
1139 if link:
1140 to_do = to_do + '-o ' + name
1141 elif backpack:
1142 if link:
1143 to_do = '-o ' + name + ' '
1144 else:
1145 to_do = ''
1146 to_do = to_do + '--backpack '
1147 elif link:
1148 to_do = '-o ' + name
1149 else:
1150 to_do = '-c' # just compile
1151
1152 stats_file = name + '.comp.stats'
1153 if opts.compiler_stats_range_fields:
1154 extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1155 if backpack:
1156 extra_hc_opts += ' -outputdir ' + name + '.out'
1157
1158 # Required by GHC 7.3+, harmless for earlier versions:
1159 if (getTestOpts().c_src or
1160 getTestOpts().objc_src or
1161 getTestOpts().objcpp_src or
1162 getTestOpts().cmm_src):
1163 extra_hc_opts += ' -no-hs-main '
1164
1165 if getTestOpts().compile_cmd_prefix == '':
1166 cmd_prefix = ''
1167 else:
1168 cmd_prefix = getTestOpts().compile_cmd_prefix + ' '
1169
1170 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1171
1172 cmd = ('cd "{opts.testdir}" && {cmd_prefix} '
1173 '{{compiler}} {to_do} {srcname} {flags} {extra_hc_opts}'
1174 ).format(**locals())
1175
1176 exit_code = runCmd(cmd, None, stdout, stderr, opts.compile_timeout_multiplier)
1177
1178 if exit_code != 0 and not should_fail:
1179 if config.verbose >= 1 and _expect_pass(way):
1180 print('Compile failed (exit code {0}) errors were:'.format(exit_code))
1181 actual_stderr_path = in_testdir(name, 'comp.stderr')
1182 if_verbose_dump(1, actual_stderr_path)
1183
1184 # ToDo: if the sub-shell was killed by ^C, then exit
1185
1186 statsResult = checkStats(name, way, stats_file, opts.compiler_stats_range_fields)
1187
1188 if badResult(statsResult):
1189 return statsResult
1190
1191 if should_fail:
1192 if exit_code == 0:
1193 return failBecause('exit code 0')
1194 else:
1195 if exit_code != 0:
1196 return failBecause('exit code non-0')
1197
1198 return passed()
1199
1200 # -----------------------------------------------------------------------------
1201 # Run a program and check its output
1202 #
1203 # If testname.stdin exists, route input from that, else
1204 # from /dev/null. Route output to testname.run.stdout and
1205 # testname.run.stderr. Returns the exit code of the run.
1206
1207 def simple_run(name, way, prog, extra_run_opts):
1208 opts = getTestOpts()
1209
1210 # figure out what to use for stdin
1211 if opts.stdin:
1212 stdin = in_testdir(opts.stdin)
1213 elif os.path.exists(in_testdir(name, 'stdin')):
1214 stdin = in_testdir(name, 'stdin')
1215 else:
1216 stdin = None
1217
1218 stdout = in_testdir(name, 'run.stdout')
1219 if opts.combined_output:
1220 stderr = subprocess.STDOUT
1221 else:
1222 stderr = in_testdir(name, 'run.stderr')
1223
1224 my_rts_flags = rts_flags(way)
1225
1226 stats_file = name + '.stats'
1227 if opts.stats_range_fields:
1228 stats_args = ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1229 else:
1230 stats_args = ''
1231
1232 # Put extra_run_opts last: extra_run_opts('+RTS foo') should work.
1233 cmd = prog + stats_args + ' ' + my_rts_flags + ' ' + extra_run_opts
1234
1235 if opts.cmd_wrapper != None:
1236 cmd = opts.cmd_wrapper(cmd)
1237
1238 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1239
1240 # run the command
1241 exit_code = runCmd(cmd, stdin, stdout, stderr, opts.run_timeout_multiplier)
1242
1243 # check the exit code
1244 if exit_code != opts.exit_code:
1245 if config.verbose >= 1 and _expect_pass(way):
1246 print('Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')')
1247 dump_stdout(name)
1248 dump_stderr(name)
1249 return failBecause('bad exit code')
1250
1251 if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
1252 return failBecause('bad stderr')
1253 if not (opts.ignore_stdout or stdout_ok(name, way)):
1254 return failBecause('bad stdout')
1255
1256 check_hp = '-h' in my_rts_flags and opts.check_hp
1257 check_prof = '-p' in my_rts_flags
1258
1259 # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1260 if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1261 return failBecause('bad heap profile')
1262 if check_prof and not check_prof_ok(name, way):
1263 return failBecause('bad profile')
1264
1265 return checkStats(name, way, stats_file, opts.stats_range_fields)
1266
1267 def rts_flags(way):
1268 args = config.way_rts_flags.get(way, [])
1269 return '+RTS {0} -RTS'.format(' '.join(args)) if args else ''
1270
1271 # -----------------------------------------------------------------------------
1272 # Run a program in the interpreter and check its output
1273
1274 def interpreter_run(name, way, extra_hc_opts, top_mod):
1275 opts = getTestOpts()
1276
1277 stdout = in_testdir(name, 'interp.stdout')
1278 stderr = in_testdir(name, 'interp.stderr')
1279 script = in_testdir(name, 'genscript')
1280
1281 if opts.combined_output:
1282 framework_fail(name, 'unsupported',
1283 'WAY=ghci and combined_output together is not supported')
1284
1285 if (top_mod == ''):
1286 srcname = add_hs_lhs_suffix(name)
1287 else:
1288 srcname = top_mod
1289
1290 delimiter = '===== program output begins here\n'
1291
1292 with io.open(script, 'w', encoding='utf8') as f:
1293 # set the prog name and command-line args to match the compiled
1294 # environment.
1295 f.write(u':set prog ' + name + u'\n')
1296 f.write(u':set args ' + opts.extra_run_opts + u'\n')
1297 # Add marker lines to the stdout and stderr output files, so we
1298 # can separate GHCi's output from the program's.
1299 f.write(u':! echo ' + delimiter)
1300 f.write(u':! echo 1>&2 ' + delimiter)
1301 # Set stdout to be line-buffered to match the compiled environment.
1302 f.write(u'System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1303 # wrapping in GHC.TopHandler.runIO ensures we get the same output
1304 # in the event of an exception as for the compiled program.
1305 f.write(u'GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
1306
1307 stdin = in_testdir(opts.stdin if opts.stdin else add_suffix(name, 'stdin'))
1308 if os.path.exists(stdin):
1309 os.system('cat "{0}" >> "{1}"'.format(stdin, script))
1310
1311 flags = ' '.join(get_compiler_flags() + config.way_flags[way])
1312
1313 cmd = ('{{compiler}} {srcname} {flags} {extra_hc_opts}'
1314 ).format(**locals())
1315
1316 if getTestOpts().cmd_wrapper != None:
1317 cmd = opts.cmd_wrapper(cmd);
1318
1319 cmd = 'cd "{opts.testdir}" && {cmd}'.format(**locals())
1320
1321 exit_code = runCmd(cmd, script, stdout, stderr, opts.run_timeout_multiplier)
1322
1323 # split the stdout into compilation/program output
1324 split_file(stdout, delimiter,
1325 in_testdir(name, 'comp.stdout'),
1326 in_testdir(name, 'run.stdout'))
1327 split_file(stderr, delimiter,
1328 in_testdir(name, 'comp.stderr'),
1329 in_testdir(name, 'run.stderr'))
1330
1331 # check the exit code
1332 if exit_code != getTestOpts().exit_code:
1333 print('Wrong exit code (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
1334 dump_stdout(name)
1335 dump_stderr(name)
1336 return failBecause('bad exit code')
1337
1338 # ToDo: if the sub-shell was killed by ^C, then exit
1339
1340 if not (opts.ignore_stderr or stderr_ok(name, way)):
1341 return failBecause('bad stderr')
1342 elif not (opts.ignore_stdout or stdout_ok(name, way)):
1343 return failBecause('bad stdout')
1344 else:
1345 return passed()
1346
1347 def split_file(in_fn, delimiter, out1_fn, out2_fn):
1348 # See Note [Universal newlines].
1349 infile = io.open(in_fn, 'r', encoding='utf8', errors='replace', newline=None)
1350 out1 = io.open(out1_fn, 'w', encoding='utf8', newline='')
1351 out2 = io.open(out2_fn, 'w', encoding='utf8', newline='')
1352
1353 line = infile.readline()
1354 while (re.sub('^\s*','',line) != delimiter and line != ''):
1355 out1.write(line)
1356 line = infile.readline()
1357 out1.close()
1358
1359 line = infile.readline()
1360 while (line != ''):
1361 out2.write(line)
1362 line = infile.readline()
1363 out2.close()
1364
1365 # -----------------------------------------------------------------------------
1366 # Utils
1367 def get_compiler_flags():
1368 opts = getTestOpts()
1369
1370 flags = copy.copy(opts.compiler_always_flags)
1371
1372 flags.append(opts.extra_hc_opts)
1373
1374 if opts.outputdir != None:
1375 flags.extend(["-outputdir", opts.outputdir])
1376
1377 return flags
1378
1379 def stdout_ok(name, way):
1380 actual_stdout_file = add_suffix(name, 'run.stdout')
1381 expected_stdout_file = find_expected_file(name, 'stdout')
1382
1383 extra_norm = join_normalisers(normalise_output, getTestOpts().extra_normaliser)
1384
1385 check_stdout = getTestOpts().check_stdout
1386 if check_stdout:
1387 actual_stdout_path = in_testdir(actual_stdout_file)
1388 return check_stdout(actual_stdout_path, extra_norm)
1389
1390 return compare_outputs(way, 'stdout', extra_norm,
1391 expected_stdout_file, actual_stdout_file)
1392
1393 def dump_stdout( name ):
1394 print('Stdout:')
1395 print(open(in_testdir(name, 'run.stdout')).read())
1396
1397 def stderr_ok(name, way):
1398 actual_stderr_file = add_suffix(name, 'run.stderr')
1399 expected_stderr_file = find_expected_file(name, 'stderr')
1400
1401 return compare_outputs(way, 'stderr',
1402 join_normalisers(normalise_errmsg, getTestOpts().extra_errmsg_normaliser), \
1403 expected_stderr_file, actual_stderr_file,
1404 whitespace_normaliser=normalise_whitespace)
1405
1406 def dump_stderr( name ):
1407 print("Stderr:")
1408 print(open(in_testdir(name, 'run.stderr')).read())
1409
1410 def read_no_crs(file):
1411 str = ''
1412 try:
1413 # See Note [Universal newlines].
1414 h = io.open(file, 'r', encoding='utf8', errors='replace', newline=None)
1415 str = h.read()
1416 h.close
1417 except:
1418 # On Windows, if the program fails very early, it seems the
1419 # files stdout/stderr are redirected to may not get created
1420 pass
1421 return str
1422
1423 def write_file(file, str):
1424 # See Note [Universal newlines].
1425 h = io.open(file, 'w', encoding='utf8', newline='')
1426 h.write(str)
1427 h.close
1428
1429 # Note [Universal newlines]
1430 #
1431 # We don't want to write any Windows style line endings ever, because
1432 # it would mean that `make accept` would touch every line of the file
1433 # when switching between Linux and Windows.
1434 #
1435 # Furthermore, when reading a file, it is convenient to translate all
1436 # Windows style endings to '\n', as it simplifies searching or massaging
1437 # the content.
1438 #
1439 # Solution: use `io.open` instead of `open`
1440 # * when reading: use newline=None to translate '\r\n' to '\n'
1441 # * when writing: use newline='' to not translate '\n' to '\r\n'
1442 #
1443 # See https://docs.python.org/2/library/io.html#io.open.
1444 #
1445 # This should work with both python2 and python3, and with both mingw*
1446 # as msys2 style Python.
1447 #
1448 # Do note that io.open returns unicode strings. So we have to specify
1449 # the expected encoding. But there is at least one file which is not
1450 # valid utf8 (decodingerror002.stdout). Solution: use errors='replace'.
1451 # Another solution would be to open files in binary mode always, and
1452 # operate on bytes.
1453
1454 def check_hp_ok(name):
1455 opts = getTestOpts()
1456
1457 # do not qualify for hp2ps because we should be in the right directory
1458 hp2psCmd = 'cd "{opts.testdir}" && {{hp2ps}} {name}'.format(**locals())
1459
1460 hp2psResult = runCmd(hp2psCmd)
1461
1462 actual_ps_path = in_testdir(name, 'ps')
1463
1464 if hp2psResult == 0:
1465 if os.path.exists(actual_ps_path):
1466 if gs_working:
1467 gsResult = runCmd(genGSCmd(actual_ps_path))
1468 if (gsResult == 0):
1469 return (True)
1470 else:
1471 print("hp2ps output for " + name + "is not valid PostScript")
1472 else: return (True) # assume postscript is valid without ghostscript
1473 else:
1474 print("hp2ps did not generate PostScript for " + name)
1475 return (False)
1476 else:
1477 print("hp2ps error when processing heap profile for " + name)
1478 return(False)
1479
1480 def check_prof_ok(name, way):
1481 expected_prof_file = find_expected_file(name, 'prof.sample')
1482 expected_prof_path = in_testdir(expected_prof_file)
1483
1484 # Check actual prof file only if we have an expected prof file to
1485 # compare it with.
1486 if not os.path.exists(expected_prof_path):
1487 return True
1488
1489 actual_prof_file = add_suffix(name, 'prof')
1490 actual_prof_path = in_testdir(actual_prof_file)
1491
1492 if not os.path.exists(actual_prof_path):
1493 print(actual_prof_path + " does not exist")
1494 return(False)
1495
1496 if os.path.getsize(actual_prof_path) == 0:
1497 print(actual_prof_path + " is empty")
1498 return(False)
1499
1500 return compare_outputs(way, 'prof', normalise_prof,
1501 expected_prof_file, actual_prof_file,
1502 whitespace_normaliser=normalise_whitespace)
1503
1504 # Compare expected output to actual output, and optionally accept the
1505 # new output. Returns true if output matched or was accepted, false
1506 # otherwise. See Note [Output comparison] for the meaning of the
1507 # normaliser and whitespace_normaliser parameters.
1508 def compare_outputs(way, kind, normaliser, expected_file, actual_file,
1509 whitespace_normaliser=lambda x:x):
1510
1511 expected_path = in_srcdir(expected_file)
1512 actual_path = in_testdir(actual_file)
1513
1514 if os.path.exists(expected_path):
1515 expected_str = normaliser(read_no_crs(expected_path))
1516 # Create the .normalised file in the testdir, not in the srcdir.
1517 expected_normalised_file = add_suffix(expected_file, 'normalised')
1518 expected_normalised_path = in_testdir(expected_normalised_file)
1519 else:
1520 expected_str = ''
1521 expected_normalised_path = '/dev/null'
1522
1523 actual_raw = read_no_crs(actual_path)
1524 actual_str = normaliser(actual_raw)
1525
1526 # See Note [Output comparison].
1527 if whitespace_normaliser(expected_str) == whitespace_normaliser(actual_str):
1528 return 1
1529 else:
1530 if config.verbose >= 1 and _expect_pass(way):
1531 print('Actual ' + kind + ' output differs from expected:')
1532
1533 if expected_normalised_path != '/dev/null':
1534 write_file(expected_normalised_path, expected_str)
1535
1536 actual_normalised_path = add_suffix(actual_path, 'normalised')
1537 write_file(actual_normalised_path, actual_str)
1538
1539 if config.verbose >= 1 and _expect_pass(way):
1540 # See Note [Output comparison].
1541 r = os.system('diff -uw "{0}" "{1}"'.format(expected_normalised_path,
1542 actual_normalised_path))
1543
1544 # If for some reason there were no non-whitespace differences,
1545 # then do a full diff
1546 if r == 0:
1547 r = os.system('diff -u "{0}" "{1}"'.format(expected_normalised_path,
1548 actual_normalised_path))
1549
1550 if config.accept and (getTestOpts().expect == 'fail' or
1551 way in getTestOpts().expect_fail_for):
1552 if_verbose(1, 'Test is expected to fail. Not accepting new output.')
1553 return 0
1554 elif config.accept and actual_raw:
1555 if_verbose(1, 'Accepting new output.')
1556 write_file(expected_path, actual_raw)
1557 return 1
1558 elif config.accept:
1559 if_verbose(1, 'No output. Deleting "{0}".'.format(expected_path))
1560 os.remove(expected_path)
1561 return 1
1562 else:
1563 return 0
1564
1565 # Note [Output comparison]
1566 #
1567 # We do two types of output comparison:
1568 #
1569 # 1. To decide whether a test has failed. We apply a `normaliser` and an
1570 # optional `whitespace_normaliser` to the expected and the actual
1571 # output, before comparing the two.
1572 #
1573 # 2. To show as a diff to the user when the test indeed failed. We apply
1574 # the same `normaliser` function to the outputs, to make the diff as
1575 # small as possible (only showing the actual problem). But we don't
1576 # apply the `whitespace_normaliser` here, because it might completely
1577 # squash all whitespace, making the diff unreadable. Instead we rely
1578 # on the `diff` program to ignore whitespace changes as much as
1579 # possible (#10152).
1580
1581 def normalise_whitespace( str ):
1582 # Merge contiguous whitespace characters into a single space.
1583 return u' '.join(w for w in str.split())
1584
1585 callSite_re = re.compile(r', called at (.+):[\d]+:[\d]+ in [\w\-\.]+:')
1586
1587 def normalise_callstacks(s):
1588 opts = getTestOpts()
1589 def repl(matches):
1590 location = matches.group(1)
1591 location = normalise_slashes_(location)
1592 return ', called at {0}:<line>:<column> in <package-id>:'.format(location)
1593 # Ignore line number differences in call stacks (#10834).
1594 s = re.sub(callSite_re, repl, s)
1595 # Ignore the change in how we identify implicit call-stacks
1596 s = s.replace('from ImplicitParams', 'from HasCallStack')
1597 if not opts.keep_prof_callstacks:
1598 # Don't output prof callstacks. Test output should be
1599 # independent from the WAY we run the test.
1600 s = re.sub(r'CallStack \(from -prof\):(\n .*)*\n?', '', s)
1601 return s
1602
1603 tyCon_re = re.compile(r'TyCon\s*\d+L?\#\#\s*\d+L?\#\#\s*', flags=re.MULTILINE)
1604
1605 def normalise_type_reps(str):
1606 """ Normalise out fingerprints from Typeable TyCon representations """
1607 return re.sub(tyCon_re, 'TyCon FINGERPRINT FINGERPRINT ', str)
1608
1609 def normalise_errmsg( str ):
1610 """Normalise error-messages emitted via stderr"""
1611 # IBM AIX's `ld` is a bit chatty
1612 if opsys('aix'):
1613 str = str.replace('ld: 0706-027 The -x flag is ignored.\n', '')
1614 # remove " error:" and lower-case " Warning:" to make patch for
1615 # trac issue #10021 smaller
1616 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1617 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1618 str = normalise_callstacks(str)
1619 str = normalise_type_reps(str)
1620
1621 # If somefile ends in ".exe" or ".exe:", zap ".exe" (for Windows)
1622 # the colon is there because it appears in error messages; this
1623 # hacky solution is used in place of more sophisticated filename
1624 # mangling
1625 str = re.sub('([^\\s])\\.exe', '\\1', str)
1626 # normalise slashes, minimise Windows/Unix filename differences
1627 str = re.sub('\\\\', '/', str)
1628 # The inplace ghc's are called ghc-stage[123] to avoid filename
1629 # collisions, so we need to normalise that to just "ghc"
1630 str = re.sub('ghc-stage[123]', 'ghc', str)
1631 # Error messages simetimes contain integer implementation package
1632 str = re.sub('integer-(gmp|simple)-[0-9.]+', 'integer-<IMPL>-<VERSION>', str)
1633 # Also filter out bullet characters. This is because bullets are used to
1634 # separate error sections, and tests shouldn't be sensitive to how the
1635 # the division happens.
1636 bullet = u'•'.encode('utf8') if isinstance(str, bytes) else u'•'
1637 str = str.replace(bullet, '')
1638 return str
1639
1640 # normalise a .prof file, so that we can reasonably compare it against
1641 # a sample. This doesn't compare any of the actual profiling data,
1642 # only the shape of the profile and the number of entries.
1643 def normalise_prof (str):
1644 # strip everything up to the line beginning "COST CENTRE"
1645 str = re.sub('^(.*\n)*COST CENTRE[^\n]*\n','',str)
1646
1647 # strip results for CAFs, these tend to change unpredictably
1648 str = re.sub('[ \t]*(CAF|IDLE).*\n','',str)
1649
1650 # XXX Ignore Main.main. Sometimes this appears under CAF, and
1651 # sometimes under MAIN.
1652 str = re.sub('[ \t]*main[ \t]+Main.*\n','',str)
1653
1654 # We have something like this:
1655 #
1656 # MAIN MAIN <built-in> 53 0 0.0 0.2 0.0 100.0
1657 # CAF Main <entire-module> 105 0 0.0 0.3 0.0 62.5
1658 # readPrec Main Main_1.hs:7:13-16 109 1 0.0 0.6 0.0 0.6
1659 # readPrec Main Main_1.hs:4:13-16 107 1 0.0 0.6 0.0 0.6
1660 # main Main Main_1.hs:(10,1)-(20,20) 106 1 0.0 20.2 0.0 61.0
1661 # == Main Main_1.hs:7:25-26 114 1 0.0 0.0 0.0 0.0
1662 # == Main Main_1.hs:4:25-26 113 1 0.0 0.0 0.0 0.0
1663 # showsPrec Main Main_1.hs:7:19-22 112 2 0.0 1.2 0.0 1.2
1664 # showsPrec Main Main_1.hs:4:19-22 111 2 0.0 0.9 0.0 0.9
1665 # readPrec Main Main_1.hs:7:13-16 110 0 0.0 18.8 0.0 18.8
1666 # readPrec Main Main_1.hs:4:13-16 108 0 0.0 19.9 0.0 19.9
1667 #
1668 # then we remove all the specific profiling data, leaving only the cost
1669 # centre name, module, src, and entries, to end up with this: (modulo
1670 # whitespace between columns)
1671 #
1672 # MAIN MAIN <built-in> 0
1673 # readPrec Main Main_1.hs:7:13-16 1
1674 # readPrec Main Main_1.hs:4:13-16 1
1675 # == Main Main_1.hs:7:25-26 1
1676 # == Main Main_1.hs:4:25-26 1
1677 # showsPrec Main Main_1.hs:7:19-22 2
1678 # showsPrec Main Main_1.hs:4:19-22 2
1679 # readPrec Main Main_1.hs:7:13-16 0
1680 # readPrec Main Main_1.hs:4:13-16 0
1681
1682 # Split 9 whitespace-separated groups, take columns 1 (cost-centre), 2
1683 # (module), 3 (src), and 5 (entries). SCC names can't have whitespace, so
1684 # this works fine.
1685 str = re.sub(r'\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*',
1686 '\\1 \\2 \\3 \\5\n', str)
1687 return str
1688
1689 def normalise_slashes_( str ):
1690 str = re.sub('\\\\', '/', str)
1691 return str
1692
1693 def normalise_exe_( str ):
1694 str = re.sub('\.exe', '', str)
1695 return str
1696
1697 def normalise_output( str ):
1698 # remove " error:" and lower-case " Warning:" to make patch for
1699 # trac issue #10021 smaller
1700 str = modify_lines(str, lambda l: re.sub(' error:', '', l))
1701 str = modify_lines(str, lambda l: re.sub(' Warning:', ' warning:', l))
1702 # Remove a .exe extension (for Windows)
1703 # This can occur in error messages generated by the program.
1704 str = re.sub('([^\\s])\\.exe', '\\1', str)
1705 str = normalise_callstacks(str)
1706 str = normalise_type_reps(str)
1707 return str
1708
1709 def normalise_asm( str ):
1710 lines = str.split('\n')
1711 # Only keep instructions and labels not starting with a dot.
1712 metadata = re.compile('^[ \t]*\\..*$')
1713 out = []
1714 for line in lines:
1715 # Drop metadata directives (e.g. ".type")
1716 if not metadata.match(line):
1717 line = re.sub('@plt', '', line)
1718 instr = line.lstrip().split()
1719 # Drop empty lines.
1720 if not instr:
1721 continue
1722 # Drop operands, except for call instructions.
1723 elif instr[0] == 'call':
1724 out.append(instr[0] + ' ' + instr[1])
1725 else:
1726 out.append(instr[0])
1727 out = u'\n'.join(out)
1728 return out
1729
1730 def if_verbose( n, s ):
1731 if config.verbose >= n:
1732 print(s)
1733
1734 def if_verbose_dump( n, f ):
1735 if config.verbose >= n:
1736 try:
1737 print(open(f).read())
1738 except:
1739 print('')
1740
1741 def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0):
1742 timeout_prog = strip_quotes(config.timeout_prog)
1743 timeout = str(int(ceil(config.timeout * timeout_multiplier)))
1744
1745 # Format cmd using config. Example: cmd='{hpc} report A.tix'
1746 cmd = cmd.format(**config.__dict__)
1747 if_verbose(3, cmd + ('< ' + os.path.basename(stdin) if stdin else ''))
1748
1749 if stdin:
1750 stdin = open(stdin, 'r')
1751 if stdout:
1752 stdout = open(stdout, 'w')
1753 if stderr and stderr is not subprocess.STDOUT:
1754 stderr = open(stderr, 'w')
1755
1756 # cmd is a complex command in Bourne-shell syntax
1757 # e.g (cd . && 'C:/users/simonpj/HEAD/inplace/bin/ghc-stage2' ...etc)
1758 # Hence it must ultimately be run by a Bourne shell. It's timeout's job
1759 # to invoke the Bourne shell
1760 r = subprocess.call([timeout_prog, timeout, cmd],
1761 stdin=stdin, stdout=stdout, stderr=stderr)
1762
1763 if stdin:
1764 stdin.close()
1765 if stdout:
1766 stdout.close()
1767 if stderr and stderr is not subprocess.STDOUT:
1768 stderr.close()
1769
1770 if r == 98:
1771 # The python timeout program uses 98 to signal that ^C was pressed
1772 stopNow()
1773 if r == 99 and getTestOpts().exit_code != 99:
1774 # Only print a message when timeout killed the process unexpectedly.
1775 if_verbose(1, 'Timeout happened...killed process "{0}"...\n'.format(cmd))
1776 return r
1777
1778 # -----------------------------------------------------------------------------
1779 # checking if ghostscript is available for checking the output of hp2ps
1780
1781 def genGSCmd(psfile):
1782 return '{{gs}} -dNODISPLAY -dBATCH -dQUIET -dNOPAUSE "{0}"'.format(psfile)
1783
1784 def gsNotWorking():
1785 global gs_working
1786 print("GhostScript not available for hp2ps tests")
1787
1788 global gs_working
1789 gs_working = 0
1790 if config.have_profiling:
1791 if config.gs != '':
1792 resultGood = runCmd(genGSCmd(config.confdir + '/good.ps'));
1793 if resultGood == 0:
1794 resultBad = runCmd(genGSCmd(config.confdir + '/bad.ps') +
1795 ' >/dev/null 2>&1')
1796 if resultBad != 0:
1797 print("GhostScript available for hp2ps tests")
1798 gs_working = 1;
1799 else:
1800 gsNotWorking();
1801 else:
1802 gsNotWorking();
1803 else:
1804 gsNotWorking();
1805
1806 def add_suffix( name, suffix ):
1807 if suffix == '':
1808 return name
1809 else:
1810 return name + '.' + suffix
1811
1812 def add_hs_lhs_suffix(name):
1813 if getTestOpts().c_src:
1814 return add_suffix(name, 'c')
1815 elif getTestOpts().cmm_src:
1816 return add_suffix(name, 'cmm')
1817 elif getTestOpts().objc_src:
1818 return add_suffix(name, 'm')
1819 elif getTestOpts().objcpp_src:
1820 return add_suffix(name, 'mm')
1821 elif getTestOpts().literate:
1822 return add_suffix(name, 'lhs')
1823 else:
1824 return add_suffix(name, 'hs')
1825
1826 def replace_suffix( name, suffix ):
1827 base, suf = os.path.splitext(name)
1828 return base + '.' + suffix
1829
1830 def in_testdir(name, suffix=''):
1831 return os.path.join(getTestOpts().testdir, add_suffix(name, suffix))
1832
1833 def in_srcdir(name, suffix=''):
1834 return os.path.join(getTestOpts().srcdir, add_suffix(name, suffix))
1835
1836 # Finding the sample output. The filename is of the form
1837 #
1838 # <test>.stdout[-ws-<wordsize>][-<platform>]
1839 #
1840 def find_expected_file(name, suff):
1841 basename = add_suffix(name, suff)
1842
1843 files = [basename + ws + plat
1844 for plat in ['-' + config.platform, '-' + config.os, '']
1845 for ws in ['-ws-' + config.wordsize, '']]
1846
1847 for f in files:
1848 if os.path.exists(in_srcdir(f)):
1849 return f
1850
1851 return basename
1852
1853 if config.msys:
1854 import stat
1855 def cleanup():
1856 def on_error(function, path, excinfo):
1857 # At least one test (T11489) removes the write bit from a file it
1858 # produces. Windows refuses to delete read-only files with a
1859 # permission error. Try setting the write bit and try again.
1860 if excinfo[1].errno == 13:
1861 os.chmod(path, stat.S_IWRITE)
1862 os.unlink(path)
1863
1864 testdir = getTestOpts().testdir
1865 shutil.rmtree(testdir, ignore_errors=False, onerror=on_error)
1866 else:
1867 def cleanup():
1868 testdir = getTestOpts().testdir
1869 if os.path.exists(testdir):
1870 shutil.rmtree(testdir, ignore_errors=False)
1871
1872
1873 # -----------------------------------------------------------------------------
1874 # Return a list of all the files ending in '.T' below directories roots.
1875
1876 def findTFiles(roots):
1877 for root in roots:
1878 for path, dirs, files in os.walk(root, topdown=True):
1879 # Never pick up .T files in uncleaned .run directories.
1880 dirs[:] = [dir for dir in sorted(dirs)
1881 if not dir.endswith(testdir_suffix)]
1882 for filename in files:
1883 if filename.endswith('.T'):
1884 yield os.path.join(path, filename)
1885
1886 # -----------------------------------------------------------------------------
1887 # Output a test summary to the specified file object
1888
1889 def summary(t, file, short=False):
1890
1891 file.write('\n')
1892 printUnexpectedTests(file,
1893 [t.unexpected_passes, t.unexpected_failures,
1894 t.unexpected_stat_failures, t.framework_failures])
1895
1896 if short:
1897 # Only print the list of unexpected tests above.
1898 return
1899
1900 file.write('SUMMARY for test run started at '
1901 + time.strftime("%c %Z", t.start_time) + '\n'
1902 + str(datetime.timedelta(seconds=
1903 round(time.time() - time.mktime(t.start_time)))).rjust(8)
1904 + ' spent to go through\n'
1905 + repr(t.total_tests).rjust(8)
1906 + ' total tests, which gave rise to\n'
1907 + repr(t.total_test_cases).rjust(8)
1908 + ' test cases, of which\n'
1909 + repr(t.n_tests_skipped).rjust(8)
1910 + ' were skipped\n'
1911 + '\n'
1912 + repr(len(t.missing_libs)).rjust(8)
1913 + ' had missing libraries\n'
1914 + repr(t.n_expected_passes).rjust(8)
1915 + ' expected passes\n'
1916 + repr(t.n_expected_failures).rjust(8)
1917 + ' expected failures\n'
1918 + '\n'
1919 + repr(len(t.framework_failures)).rjust(8)
1920 + ' caused framework failures\n'
1921 + repr(len(t.unexpected_passes)).rjust(8)
1922 + ' unexpected passes\n'
1923 + repr(len(t.unexpected_failures)).rjust(8)
1924 + ' unexpected failures\n'
1925 + repr(len(t.unexpected_stat_failures)).rjust(8)
1926 + ' unexpected stat failures\n'
1927 + '\n')
1928
1929 if t.unexpected_passes:
1930 file.write('Unexpected passes:\n')
1931 printTestInfosSummary(file, t.unexpected_passes)
1932
1933 if t.unexpected_failures:
1934 file.write('Unexpected failures:\n')
1935 printTestInfosSummary(file, t.unexpected_failures)
1936
1937 if t.unexpected_stat_failures:
1938 file.write('Unexpected stat failures:\n')
1939 printTestInfosSummary(file, t.unexpected_stat_failures)
1940
1941 if t.framework_failures:
1942 file.write('Framework failures:\n')
1943 printTestInfosSummary(file, t.framework_failures)
1944
1945 if stopping():
1946 file.write('WARNING: Testsuite run was terminated early\n')
1947
1948 def printUnexpectedTests(file, testInfoss):
1949 unexpected = set(name for testInfos in testInfoss
1950 for (_, name, _, _) in testInfos
1951 if not name.endswith('.T'))
1952 if unexpected:
1953 file.write('Unexpected results from:\n')
1954 file.write('TEST="' + ' '.join(unexpected) + '"\n')
1955 file.write('\n')
1956
1957 def printTestInfosSummary(file, testInfos):
1958 maxDirLen = max(len(directory) for (directory, _, _, _) in testInfos)
1959 for (directory, name, reason, way) in testInfos:
1960 directory = directory.ljust(maxDirLen)
1961 file.write(' {directory} {name} [{reason}] ({way})\n'.format(**locals()))
1962 file.write('\n')
1963
1964 def modify_lines(s, f):
1965 s = u'\n'.join([f(l) for l in s.splitlines()])
1966 if s and s[-1] != '\n':
1967 # Prevent '\ No newline at end of file' warnings when diffing.
1968 s += '\n'
1969 return s