From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- js/src/tests/lib/results.py | 471 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 js/src/tests/lib/results.py (limited to 'js/src/tests/lib/results.py') diff --git a/js/src/tests/lib/results.py b/js/src/tests/lib/results.py new file mode 100644 index 0000000000..9c623b863f --- /dev/null +++ b/js/src/tests/lib/results.py @@ -0,0 +1,471 @@ +import json +import pipes +import re + +from .progressbar import NullProgressBar, ProgressBar +from .structuredlog import TestLogger + +# subprocess.list2cmdline does not properly escape for sh-like shells + + +def escape_cmdline(args): + return " ".join([pipes.quote(a) for a in args]) + + +class TestOutput: + """Output from a test run.""" + + def __init__(self, test, cmd, out, err, rc, dt, timed_out, extra=None): + self.test = test # Test + self.cmd = cmd # str: command line of test + self.out = out # str: stdout + self.err = err # str: stderr + self.rc = rc # int: return code + self.dt = dt # float: run time + self.timed_out = timed_out # bool: did the test time out + self.extra = extra # includes the pid on some platforms + + def describe_failure(self): + if self.timed_out: + return "Timeout" + lines = self.err.splitlines() + for line in lines: + # Skip the asm.js compilation success message. + if "Successfully compiled asm.js code" not in line: + return line + return "Unknown" + + +class NullTestOutput: + """Variant of TestOutput that indicates a test was not run.""" + + def __init__(self, test): + self.test = test + self.cmd = "" + self.out = "" + self.err = "" + self.rc = 0 + self.dt = 0.0 + self.timed_out = False + + +class TestResult: + PASS = "PASS" + FAIL = "FAIL" + CRASH = "CRASH" + + """Classified result from a test run.""" + + def __init__(self, test, result, results, wpt_results=None): + self.test = test + self.result = result + self.results = results + self.wpt_results = wpt_results # Only used for wpt tests. + + @classmethod + def from_wpt_output(cls, output): + """Parse the output from a web-platform test that uses testharness.js. + (The output is written to stdout in js/src/tests/testharnessreport.js.) + """ + from wptrunner.executors.base import testharness_result_converter + + rc = output.rc + stdout = output.out.split("\n") + if rc != 0: + if rc == 3: + harness_status = "ERROR" + harness_message = "Exit code reported exception" + else: + harness_status = "CRASH" + harness_message = "Exit code reported crash" + tests = [] + else: + for idx, line in enumerate(stdout): + if line.startswith("WPT OUTPUT: "): + msg = line[len("WPT OUTPUT: ") :] + data = [output.test.wpt.url] + json.loads(msg) + harness_status_obj, tests = testharness_result_converter( + output.test.wpt, data + ) + harness_status = harness_status_obj.status + harness_message = "Reported by harness: %s" % ( + harness_status_obj.message, + ) + del stdout[idx] + break + else: + harness_status = "ERROR" + harness_message = "No harness output found" + tests = [] + stdout.append("Harness status: %s (%s)" % (harness_status, harness_message)) + + result = cls.PASS + results = [] + subtests = [] + expected_harness_status = output.test.wpt.expected() + if harness_status != expected_harness_status: + if harness_status == "CRASH": + result = cls.CRASH + else: + result = cls.FAIL + else: + for test in tests: + test_output = 'Subtest "%s": ' % (test.name,) + expected = output.test.wpt.expected(test.name) + if test.status == expected: + test_result = (cls.PASS, "") + test_output += "as expected: %s" % (test.status,) + else: + test_result = (cls.FAIL, test.message) + result = cls.FAIL + test_output += "expected %s, found %s" % (expected, test.status) + if test.message: + test_output += ' (with message: "%s")' % (test.message,) + subtests.append( + { + "test": output.test.wpt.id, + "subtest": test.name, + "status": test.status, + "expected": expected, + } + ) + results.append(test_result) + stdout.append(test_output) + + output.out = "\n".join(stdout) + "\n" + + wpt_results = { + "name": output.test.wpt.id, + "status": harness_status, + "expected": expected_harness_status, + "subtests": subtests, + } + + return cls(output.test, result, results, wpt_results) + + @classmethod + def from_output(cls, output): + test = output.test + result = None # str: overall result, see class-level variables + results = [] # (str,str) list: subtest results (pass/fail, message) + + if test.wpt: + return cls.from_wpt_output(output) + + out, err, rc = output.out, output.err, output.rc + + failures = 0 + passes = 0 + + expected_rcs = [] + if test.path.endswith("-n.js"): + expected_rcs.append(3) + + for line in out.split("\n"): + if line.startswith(" FAILED!"): + failures += 1 + msg = line[len(" FAILED! ") :] + results.append((cls.FAIL, msg)) + elif line.startswith(" PASSED!"): + passes += 1 + msg = line[len(" PASSED! ") :] + results.append((cls.PASS, msg)) + else: + m = re.match( + "--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE" + " ((?:-|\\d)+) ---", + line, + ) + if m: + expected_rcs.append(int(m.group(1))) + + if test.error is not None: + expected_rcs.append(3) + if test.error not in err: + failures += 1 + results.append( + (cls.FAIL, "Expected uncaught error: {}".format(test.error)) + ) + + if rc and rc not in expected_rcs: + if rc == 3: + result = cls.FAIL + else: + result = cls.CRASH + else: + if (rc or passes > 0) and failures == 0: + result = cls.PASS + else: + result = cls.FAIL + + return cls(test, result, results) + + +class TestDuration: + def __init__(self, test, duration): + self.test = test + self.duration = duration + + +class ResultsSink: + def __init__(self, testsuite, options, testcount): + self.options = options + self.fp = options.output_fp + if self.options.format == "automation": + self.slog = TestLogger(testsuite) + self.slog.suite_start() + + self.wptreport = None + if self.options.wptreport: + try: + from .wptreport import WptreportHandler + + self.wptreport = WptreportHandler(self.options.wptreport) + self.wptreport.suite_start() + except ImportError: + pass + + self.groups = {} + self.output_dict = {} + self.counts = {"PASS": 0, "FAIL": 0, "TIMEOUT": 0, "SKIP": 0} + self.slow_tests = [] + self.n = 0 + + if options.hide_progress: + self.pb = NullProgressBar() + else: + fmt = [ + {"value": "PASS", "color": "green"}, + {"value": "FAIL", "color": "red"}, + {"value": "TIMEOUT", "color": "blue"}, + {"value": "SKIP", "color": "brightgray"}, + ] + self.pb = ProgressBar(testcount, fmt) + + def push(self, output): + if self.options.show_slow and output.dt >= self.options.slow_test_threshold: + self.slow_tests.append(TestDuration(output.test, output.dt)) + if output.timed_out: + self.counts["TIMEOUT"] += 1 + if isinstance(output, NullTestOutput): + if self.options.format == "automation": + self.print_automation_result( + "TEST-KNOWN-FAIL", output.test, time=output.dt, skip=True + ) + self.counts["SKIP"] += 1 + self.n += 1 + else: + result = TestResult.from_output(output) + + if self.wptreport is not None and result.wpt_results: + self.wptreport.test(result.wpt_results, output.dt) + + tup = (result.result, result.test.expect, result.test.random) + dev_label = self.LABELS[tup][1] + + if self.options.check_output: + if output.test.path in self.output_dict.keys(): + if self.output_dict[output.test.path] != output: + self.counts["FAIL"] += 1 + self.print_automation_result( + "TEST-UNEXPECTED-FAIL", + result.test, + time=output.dt, + message="Same test with different flag producing different output", + ) + else: + self.output_dict[output.test.path] = output + + if output.timed_out: + dev_label = "TIMEOUTS" + self.groups.setdefault(dev_label, []).append(result) + + if dev_label == "REGRESSIONS": + show_output = ( + self.options.show_output or not self.options.no_show_failed + ) + elif dev_label == "TIMEOUTS": + show_output = self.options.show_output + else: + show_output = self.options.show_output and not self.options.failed_only + + if dev_label in ("REGRESSIONS", "TIMEOUTS"): + show_cmd = self.options.show_cmd + else: + show_cmd = self.options.show_cmd and not self.options.failed_only + + if show_output or show_cmd: + self.pb.beginline() + + if show_output: + print( + "## {}: rc = {:d}, run time = {}".format( + output.test.path, output.rc, output.dt + ), + file=self.fp, + ) + + if show_cmd: + print(escape_cmdline(output.cmd), file=self.fp) + + if show_output: + + def write_with_fallback(fp, data): + try: + fp.write(data) + except UnicodeEncodeError as e: + # In case the data contains something not directly + # encodable, use \uXXXX. + fp.write( + "WARNING: Falling back from exception: {}\n".format(e) + ) + fp.write("WARNING: The following output is escaped, ") + fp.write("and may be different than original one.\n") + fp.write( + data.encode("ascii", "namereplace").decode("ascii") + ) + + write_with_fallback(self.fp, output.out) + write_with_fallback(self.fp, output.err) + + self.n += 1 + + if result.result == TestResult.PASS and not result.test.random: + self.counts["PASS"] += 1 + elif result.test.expect and not result.test.random: + self.counts["FAIL"] += 1 + else: + self.counts["SKIP"] += 1 + + if self.options.format == "automation": + if result.result != TestResult.PASS and len(result.results) > 1: + for sub_ok, msg in result.results: + tup = (sub_ok, result.test.expect, result.test.random) + label = self.LABELS[tup][0] + if label == "TEST-UNEXPECTED-PASS": + label = "TEST-PASS (EXPECTED RANDOM)" + self.print_automation_result( + label, result.test, time=output.dt, message=msg + ) + tup = (result.result, result.test.expect, result.test.random) + self.print_automation_result( + self.LABELS[tup][0], + result.test, + time=output.dt, + extra=getattr(output, "extra", None), + ) + return + + if dev_label: + + def singular(label): + return "FIXED" if label == "FIXES" else label[:-1] + + self.pb.message("{} - {}".format(singular(dev_label), output.test.path)) + + self.pb.update(self.n, self.counts) + + def finish(self, completed): + self.pb.finish(completed) + if self.options.format == "automation": + self.slog.suite_end() + else: + self.list(completed) + + if self.wptreport is not None: + self.wptreport.suite_end() + + # Conceptually, this maps (test result x test expectation) to text labels. + # key is (result, expect, random) + # value is (automation label, dev test category) + LABELS = { + (TestResult.CRASH, False, False): ("TEST-UNEXPECTED-FAIL", "REGRESSIONS"), + (TestResult.CRASH, False, True): ("TEST-UNEXPECTED-FAIL", "REGRESSIONS"), + (TestResult.CRASH, True, False): ("TEST-UNEXPECTED-FAIL", "REGRESSIONS"), + (TestResult.CRASH, True, True): ("TEST-UNEXPECTED-FAIL", "REGRESSIONS"), + (TestResult.FAIL, False, False): ("TEST-KNOWN-FAIL", ""), + (TestResult.FAIL, False, True): ("TEST-KNOWN-FAIL (EXPECTED RANDOM)", ""), + (TestResult.FAIL, True, False): ("TEST-UNEXPECTED-FAIL", "REGRESSIONS"), + (TestResult.FAIL, True, True): ("TEST-KNOWN-FAIL (EXPECTED RANDOM)", ""), + (TestResult.PASS, False, False): ("TEST-UNEXPECTED-PASS", "FIXES"), + (TestResult.PASS, False, True): ("TEST-PASS (EXPECTED RANDOM)", ""), + (TestResult.PASS, True, False): ("TEST-PASS", ""), + (TestResult.PASS, True, True): ("TEST-PASS (EXPECTED RANDOM)", ""), + } + + def list(self, completed): + for label, results in sorted(self.groups.items()): + if label == "": + continue + + print(label) + for result in results: + print( + " {}".format( + " ".join( + result.test.jitflags + + result.test.options + + [result.test.path] + ) + ) + ) + + if self.options.failure_file: + failure_file = open(self.options.failure_file, "w") + if not self.all_passed(): + if "REGRESSIONS" in self.groups: + for result in self.groups["REGRESSIONS"]: + print(result.test.path, file=failure_file) + if "TIMEOUTS" in self.groups: + for result in self.groups["TIMEOUTS"]: + print(result.test.path, file=failure_file) + failure_file.close() + + suffix = "" if completed else " (partial run -- interrupted by user)" + if self.all_passed(): + print("PASS" + suffix) + else: + print("FAIL" + suffix) + + if self.options.show_slow: + min_duration = self.options.slow_test_threshold + print("Slow tests (duration > {}s)".format(min_duration)) + slow_tests = sorted(self.slow_tests, key=lambda x: x.duration, reverse=True) + any = False + for test in slow_tests: + print("{:>5} {}".format(round(test.duration, 2), test.test)) + any = True + if not any: + print("None") + + def all_passed(self): + return "REGRESSIONS" not in self.groups and "TIMEOUTS" not in self.groups + + def print_automation_result( + self, label, test, message=None, skip=False, time=None, extra=None + ): + result = label + result += " | " + test.path + args = [] + if self.options.shell_args: + args.append(self.options.shell_args) + args += test.jitflags + result += ' | (args: "{}")'.format(" ".join(args)) + if message: + result += " | " + message + if skip: + result += " | (SKIP)" + if time > self.options.timeout: + result += " | (TIMEOUT)" + result += " [{:.1f} s]".format(time) + print(result) + + details = {"extra": extra.copy() if extra else {}} + if self.options.shell_args: + details["extra"]["shell_args"] = self.options.shell_args + details["extra"]["jitflags"] = test.jitflags + if message: + details["message"] = message + status = "FAIL" if "TEST-UNEXPECTED" in label else "PASS" + + self.slog.test(test.path, status, time or 0, **details) -- cgit v1.2.3