testsuite: Identify framework failures in testsuite summary
authorBen Gamari <bgamari.foss@gmail.com>
Fri, 25 Mar 2016 23:43:24 +0000 (00:43 +0100)
committerBen Gamari <ben@smart-cactus.org>
Fri, 25 Mar 2016 23:43:25 +0000 (00:43 +0100)
Currently the testsuite driver tells you how many tests failed due to a
framework failure but you need to manually grep through the testsuite
output to identify which ones.

Test Plan: Validate with, e.g., a timing out testcase

Reviewers: austin, thomie

Reviewed By: austin, thomie

Differential Revision: https://phabricator.haskell.org/D2026

GHC Trac Issues: #11165

testsuite/driver/testlib.py

index 0e2ba49..bba3516 100644 (file)
@@ -2265,6 +2265,10 @@ def summary(t, file, short=False):
         file.write('Unexpected stat failures:\n')
         printFailingTestInfosSummary(file, t.unexpected_stat_failures)
 
+    if t.n_framework_failures > 0:
+        file.write('Test framework failures:\n')
+        printFrameworkFailureSummary(file, t.framework_failures)
+
     if config.check_files_written:
         checkForFilesWrittenProblems(file)
 
@@ -2310,6 +2314,16 @@ def printFailingTestInfosSummary(file, testInfos):
                           ' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
     file.write('\n')
 
+def printFrameworkFailureSummary(file, testInfos):
+    names = list(testInfos.keys())
+    names.sort()
+    maxNameLen = max(len(n) for n in names)
+    for name in names:
+        ways = testInfos[name]
+        file.write('   ' + name.ljust(maxNameLen + 2) + \
+                   ' (' + ','.join(ways) + ')\n')
+    file.write('\n')
+
 def modify_lines(s, f):
     s = '\n'.join([f(l) for l in s.splitlines()])
     if s and s[-1] != '\n':