Exit with exit code 1 when tests unexpectedly pass
authorMatthew Pickering <matthewtpickering@gmail.com>
Tue, 19 Feb 2019 20:23:48 +0000 (20:23 +0000)
committerMarge Bot <ben+marge-bot@smart-cactus.org>
Sun, 24 Feb 2019 02:50:02 +0000 (21:50 -0500)
This was causing gitlab to not report from builds as failing. It also
highlighted a problem with the LLVM tests where some of the external
interpreter tests are failing.

testsuite/driver/runtests.py
testsuite/driver/testlib.py
testsuite/tests/th/all.T

index 51f682e..b439abd 100644 (file)
@@ -439,6 +439,7 @@ else:
 
 if len(t.unexpected_failures) > 0 or \
    len(t.unexpected_stat_failures) > 0 or \
+   len(t.unexpected_passes) > 0 or \
    len(t.framework_failures) > 0:
     exitcode = 1
 else:
index fb7888c..11c828c 100644 (file)
@@ -2118,6 +2118,7 @@ def summary(t, file, short=False, color=False):
     if color:
         if len(t.unexpected_failures) > 0 or \
             len(t.unexpected_stat_failures) > 0 or \
+            len(t.unexpected_passes) > 0 or \
             len(t.framework_failures) > 0:
             colorize = str_fail
         else:
index 2aaa48c..7d6340b 100644 (file)
@@ -12,8 +12,14 @@ setTestOpts(req_interp)
 if config.have_ext_interp :
        setTestOpts(extra_ways(['ext-interp']))
        setTestOpts(only_ways(['normal','ghci','ext-interp']))
-       if config.ghc_built_by_llvm:
-           setTestOpts(expect_broken_for(16087, ['ext-interp']))
+
+broken_tests = ["ClosedFam1TH","T10620","T10828","T11721_TH","T11797","T12045TH2","T12478_1","T12646","T13642","T14060","T15502","T15738","T15792","T15845","T16180","T1835","T3920","T4135","T4188","T5037","T5362","T7477","T7910","T8761","T8884","T8953","T9262","T9692","T9738","TH_Lift","TH_RichKinds","TH_RichKinds2","TH_Roles3","TH_TyInstWhere2","TH_implicitParams","TH_recursiveDo","TH_reifyDecl1","TH_reifyExplicitForAllFams","TH_reifyInstances","TH_reifyMkName","TH_repE2","TH_repGuard","TH_repPrim","TH_repPrim2","TH_repUnboxedTuples","TH_spliceE6"]
+# ext-interp, integer-gmp and llvm is broken see #16087
+def broken_ext_interp(name, opts):
+       if name in broken_tests and config.ghc_built_by_llvm:
+               expect_broken_for(16087, ['ext-interp'])(name, opts)
+
+setTestOpts(broken_ext_interp)
 
 test('TH_mkName', normal, compile, ['-v0'])
 test('TH_overloadedlabels', normal, compile, ['-v0'])