testsuite: Classify missing expected perf numbers as merely warnings
authorBen Gamari <ben@smart-cactus.org>
Sun, 2 Apr 2017 21:20:34 +0000 (17:20 -0400)
committerBen Gamari <ben@smart-cactus.org>
Mon, 3 Apr 2017 01:13:10 +0000 (21:13 -0400)
Previously these were considered to be framework failures, meaning that validate
would fail. For better or worse, Windows lacks a good number of metrics and I
don't see this changing any time soon. Let's consider these to be non-fatal.

testsuite/driver/testglobals.py
testsuite/driver/testlib.py

index 9f37e1a..fc050e6 100644 (file)
@@ -138,6 +138,7 @@ class TestRun:
 
        self.missing_libs = []
        self.framework_failures = []
+       self.framework_warnings = []
 
        self.unexpected_passes = []
        self.unexpected_failures = []
index a5da1e9..457e380 100644 (file)
@@ -324,7 +324,7 @@ def _stats_num_field( name, opts, field, expecteds ):
             if b:
                 opts.stats_range_fields[field] = (expected, dev)
                 return
-        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
+        framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
 
     else:
         (expected, dev) = expecteds
@@ -347,7 +347,7 @@ def _compiler_stats_num_field( name, opts, field, expecteds ):
             opts.compiler_stats_range_fields[field] = (expected, dev)
             return
 
-    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
+    framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
 
 # -----
 
@@ -893,6 +893,13 @@ def framework_fail(name, way, reason):
     if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
     t.framework_failures.append((directory, name, way, reason))
 
+def framework_warn(name, way, reason):
+    opts = getTestOpts()
+    directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
+    full_name = name + '(' + way + ')'
+    if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
+    t.framework_warnings.append((directory, name, way, reason))
+
 def badResult(result):
     try:
         if result['passFail'] == 'pass':
@@ -1990,6 +1997,8 @@ def summary(t, file, short=False):
                + '\n'
                + repr(len(t.framework_failures)).rjust(8)
                + ' caused framework failures\n'
+               + repr(len(t.framework_warnings)).rjust(8)
+               + ' caused framework warnings\n'
                + repr(len(t.unexpected_passes)).rjust(8)
                + ' unexpected passes\n'
                + repr(len(t.unexpected_failures)).rjust(8)
@@ -2014,6 +2023,10 @@ def summary(t, file, short=False):
         file.write('Framework failures:\n')
         printTestInfosSummary(file, t.framework_failures)
 
+    if t.framework_warnings:
+        file.write('Framework warnings:\n')
+        printTestInfosSummary(file, t.framework_warnings)
+
     if stopping():
         file.write('WARNING: Testsuite run was terminated early\n')