summaryrefslogtreecommitdiffstats
path: root/js/src/tests/lib/results.py
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/tests/lib/results.py')
-rw-r--r--js/src/tests/lib/results.py278
1 files changed, 278 insertions, 0 deletions
diff --git a/js/src/tests/lib/results.py b/js/src/tests/lib/results.py
new file mode 100644
index 000000000..451fe5b65
--- /dev/null
+++ b/js/src/tests/lib/results.py
@@ -0,0 +1,278 @@
+from __future__ import print_function
+
+import re
+from progressbar import NullProgressBar, ProgressBar
+import pipes
+
+# subprocess.list2cmdline does not properly escape for sh-like shells
+def escape_cmdline(args):
+ return ' '.join([pipes.quote(a) for a in args])
+
+class TestOutput:
+ """Output from a test run."""
+ def __init__(self, test, cmd, out, err, rc, dt, timed_out):
+ self.test = test # Test
+ self.cmd = cmd # str: command line of test
+ self.out = out # str: stdout
+ self.err = err # str: stderr
+ self.rc = rc # int: return code
+ self.dt = dt # float: run time
+ self.timed_out = timed_out # bool: did the test time out
+
+ def describe_failure(self):
+ if self.timed_out:
+ return "Timeout"
+ lines = self.err.splitlines()
+ for line in lines:
+ # Skip the asm.js compilation success message.
+ if "Successfully compiled asm.js code" not in line:
+ return line
+ return "Unknown"
+
+class NullTestOutput:
+ """Variant of TestOutput that indicates a test was not run."""
+ def __init__(self, test):
+ self.test = test
+ self.cmd = ''
+ self.out = ''
+ self.err = ''
+ self.rc = 0
+ self.dt = 0.0
+ self.timed_out = False
+
+class TestResult:
+ PASS = 'PASS'
+ FAIL = 'FAIL'
+ CRASH = 'CRASH'
+
+ """Classified result from a test run."""
+ def __init__(self, test, result, results):
+ self.test = test
+ self.result = result
+ self.results = results
+
+ @classmethod
+ def from_output(cls, output):
+ test = output.test
+ result = None # str: overall result, see class-level variables
+ results = [] # (str,str) list: subtest results (pass/fail, message)
+
+ out, rc = output.out, output.rc
+
+ failures = 0
+ passes = 0
+
+ expected_rcs = []
+ if test.path.endswith('-n.js'):
+ expected_rcs.append(3)
+
+ for line in out.split('\n'):
+ if line.startswith(' FAILED!'):
+ failures += 1
+ msg = line[len(' FAILED! '):]
+ results.append((cls.FAIL, msg))
+ elif line.startswith(' PASSED!'):
+ passes += 1
+ msg = line[len(' PASSED! '):]
+ results.append((cls.PASS, msg))
+ else:
+ m = re.match('--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE'
+ ' ((?:-|\\d)+) ---', line)
+ if m:
+ expected_rcs.append(int(m.group(1)))
+
+ if rc and not rc in expected_rcs:
+ if rc == 3:
+ result = cls.FAIL
+ else:
+ result = cls.CRASH
+ else:
+ if (rc or passes > 0) and failures == 0:
+ result = cls.PASS
+ else:
+ result = cls.FAIL
+
+ return cls(test, result, results)
+
+class ResultsSink:
+ def __init__(self, options, testcount):
+ self.options = options
+ self.fp = options.output_fp
+
+ self.groups = {}
+ self.output_dict = {}
+ self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0}
+ self.n = 0
+
+ if options.hide_progress:
+ self.pb = NullProgressBar()
+ else:
+ fmt = [
+ {'value': 'PASS', 'color': 'green'},
+ {'value': 'FAIL', 'color': 'red'},
+ {'value': 'TIMEOUT', 'color': 'blue'},
+ {'value': 'SKIP', 'color': 'brightgray'},
+ ]
+ self.pb = ProgressBar(testcount, fmt)
+
+ def push(self, output):
+ if output.timed_out:
+ self.counts['TIMEOUT'] += 1
+ if isinstance(output, NullTestOutput):
+ if self.options.format == 'automation':
+ self.print_automation_result(
+ 'TEST-KNOWN-FAIL', output.test, time=output.dt,
+ skip=True)
+ self.counts['SKIP'] += 1
+ self.n += 1
+ else:
+ result = TestResult.from_output(output)
+ tup = (result.result, result.test.expect, result.test.random)
+ dev_label = self.LABELS[tup][1]
+
+ if self.options.check_output:
+ if output.test.path in self.output_dict.keys():
+ if self.output_dict[output.test.path] != output:
+ self.counts['FAIL'] += 1
+ self.print_automation_result(
+ "TEST-UNEXPECTED-FAIL", result.test, time=output.dt,
+ message="Same test with different flag producing different output")
+ else:
+ self.output_dict[output.test.path] = output
+
+ if output.timed_out:
+ dev_label = 'TIMEOUTS'
+ self.groups.setdefault(dev_label, []).append(result)
+
+ if dev_label == 'REGRESSIONS':
+ show_output = self.options.show_output \
+ or not self.options.no_show_failed
+ elif dev_label == 'TIMEOUTS':
+ show_output = self.options.show_output
+ else:
+ show_output = self.options.show_output \
+ and not self.options.failed_only
+
+ if dev_label in ('REGRESSIONS', 'TIMEOUTS'):
+ show_cmd = self.options.show_cmd
+ else:
+ show_cmd = self.options.show_cmd \
+ and not self.options.failed_only
+
+ if show_output or show_cmd:
+ self.pb.beginline()
+
+ if show_output:
+ print('## {}: rc = {:d}, run time = {}'.format(
+ output.test.path, output.rc, output.dt), file=self.fp)
+
+ if show_cmd:
+ print(escape_cmdline(output.cmd), file=self.fp)
+
+ if show_output:
+ self.fp.write(output.out)
+ self.fp.write(output.err)
+
+ self.n += 1
+
+ if result.result == TestResult.PASS and not result.test.random:
+ self.counts['PASS'] += 1
+ elif result.test.expect and not result.test.random:
+ self.counts['FAIL'] += 1
+ else:
+ self.counts['SKIP'] += 1
+
+ if self.options.format == 'automation':
+ if result.result != TestResult.PASS and len(result.results) > 1:
+ for sub_ok, msg in result.results:
+ tup = (sub_ok, result.test.expect, result.test.random)
+ label = self.LABELS[tup][0]
+ if label == 'TEST-UNEXPECTED-PASS':
+ label = 'TEST-PASS (EXPECTED RANDOM)'
+ self.print_automation_result(
+ label, result.test, time=output.dt,
+ message=msg)
+ tup = (result.result, result.test.expect, result.test.random)
+ self.print_automation_result(
+ self.LABELS[tup][0], result.test, time=output.dt)
+ return
+
+ if dev_label:
+ def singular(label):
+ return "FIXED" if label == "FIXES" else label[:-1]
+ self.pb.message("{} - {}".format(singular(dev_label),
+ output.test.path))
+
+ self.pb.update(self.n, self.counts)
+
+ def finish(self, completed):
+ self.pb.finish(completed)
+ if not self.options.format == 'automation':
+ self.list(completed)
+
+ # Conceptually, this maps (test result x test expection) to text labels.
+ # key is (result, expect, random)
+ # value is (automation label, dev test category)
+ LABELS = {
+ (TestResult.CRASH, False, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
+ (TestResult.CRASH, False, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
+ (TestResult.CRASH, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
+ (TestResult.CRASH, True, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
+
+ (TestResult.FAIL, False, False): ('TEST-KNOWN-FAIL', ''),
+ (TestResult.FAIL, False, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''),
+ (TestResult.FAIL, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
+ (TestResult.FAIL, True, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''),
+
+ (TestResult.PASS, False, False): ('TEST-UNEXPECTED-PASS', 'FIXES'),
+ (TestResult.PASS, False, True): ('TEST-PASS (EXPECTED RANDOM)', ''),
+ (TestResult.PASS, True, False): ('TEST-PASS', ''),
+ (TestResult.PASS, True, True): ('TEST-PASS (EXPECTED RANDOM)', ''),
+ }
+
+ def list(self, completed):
+ for label, results in sorted(self.groups.items()):
+ if label == '':
+ continue
+
+ print(label)
+ for result in results:
+ print(' {}'.format(' '.join(result.test.jitflags +
+ [result.test.path])))
+
+ if self.options.failure_file:
+ failure_file = open(self.options.failure_file, 'w')
+ if not self.all_passed():
+ if 'REGRESSIONS' in self.groups:
+ for result in self.groups['REGRESSIONS']:
+ print(result.test.path, file=failure_file)
+ if 'TIMEOUTS' in self.groups:
+ for result in self.groups['TIMEOUTS']:
+ print(result.test.path, file=failure_file)
+ failure_file.close()
+
+ suffix = '' if completed else ' (partial run -- interrupted by user)'
+ if self.all_passed():
+ print('PASS' + suffix)
+ else:
+ print('FAIL' + suffix)
+
+ def all_passed(self):
+ return 'REGRESSIONS' not in self.groups and 'TIMEOUTS' not in self.groups
+
+ def print_automation_result(self, label, test, message=None, skip=False,
+ time=None):
+ result = label
+ result += " | " + test.path
+ args = []
+ if self.options.shell_args:
+ args.append(self.options.shell_args)
+ args += test.jitflags
+ result += ' | (args: "{}")'.format(' '.join(args))
+ if message:
+ result += " | " + message
+ if skip:
+ result += ' | (SKIP)'
+ if time > self.options.timeout:
+ result += ' | (TIMEOUT)'
+ print(result)
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-11 07:46:19 +0000