# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import argparse import json import os import re import sys from mozlog import commandline run_infos = { "linux-opt": { "os": "linux", "processor": "x86_64", "version": "Ubuntu 18.04", "os_version": "18.04", "bits": 64, "has_sandbox": True, "automation": True, "linux_distro": "Ubuntu", "apple_silicon": False, "appname": "firefox", "artifact": False, "asan": False, "bin_suffix": "", "buildapp": "browser", "buildtype_guess": "pgo", "cc_type": "clang", "ccov": False, "crashreporter": True, "datareporting": True, "debug": False, "devedition": False, "early_beta_or_earlier": True, "healthreport": True, "nightly_build": True, "normandy": True, "official": True, "pgo": True, "platform_guess": "linux64", "release_or_beta": False, "require_signing": False, "stylo": True, "sync": True, "telemetry": False, "tests_enabled": True, "toolkit": "gtk", "tsan": False, "ubsan": False, "updater": True, "python_version": 3, "product": "firefox", "verify": False, "wasm": True, "e10s": True, "headless": False, "fission": True, "sessionHistoryInParent": True, "swgl": False, "privateBrowsing": False, "win11_2009": False, "domstreams": True, "isolated_process": False, "display": "x11", }, "linux-debug": { "os": "linux", "processor": "x86_64", "version": "Ubuntu 18.04", "os_version": "18.04", "bits": 64, "has_sandbox": True, "automation": True, "linux_distro": "Ubuntu", "apple_silicon": False, "appname": "firefox", "artifact": False, "asan": False, "bin_suffix": "", "buildapp": "browser", "buildtype_guess": "debug", "cc_type": "clang", "ccov": False, "crashreporter": True, "datareporting": True, "debug": True, "devedition": False, "early_beta_or_earlier": True, "healthreport": True, "nightly_build": True, "normandy": True, "official": True, "pgo": False, "platform_guess": "linux64", "release_or_beta": False, "require_signing": False, "stylo": True, "sync": True, "telemetry": False, "tests_enabled": True, "toolkit": "gtk", "tsan": False, "ubsan": False, "updater": True, "python_version": 3, "product": "firefox", "verify": False, "wasm": True, "e10s": True, "headless": False, "fission": False, "sessionHistoryInParent": False, "swgl": False, "privateBrowsing": False, "win11_2009": False, "domstreams": True, "isolated_process": False, "display": "x11", }, "win-opt": { "os": "win", "processor": "x86_64", "version": "10.0.17134", "os_version": "10.0", "bits": 64, "has_sandbox": True, "automation": True, "apple_silicon": False, "appname": "firefox", "artifact": False, "asan": False, "bin_suffix": ".exe", "buildapp": "browser", "buildtype_guess": "pgo", "cc_type": "clang-cl", "ccov": False, "crashreporter": True, "datareporting": True, "debug": False, "devedition": False, "early_beta_or_earlier": True, "healthreport": True, "nightly_build": True, "normandy": True, "official": True, "pgo": True, "platform_guess": "win64", "release_or_beta": False, "require_signing": False, "stylo": True, "sync": True, "telemetry": False, "tests_enabled": True, "toolkit": "windows", "tsan": False, "ubsan": False, "updater": True, "python_version": 3, "product": "firefox", "verify": False, "wasm": True, "e10s": True, "headless": False, "fission": False, "sessionHistoryInParent": False, "swgl": False, "privateBrowsing": False, "win11_2009": False, "domstreams": True, "isolated_process": False, "display": None, }, } # RE that checks for anything containing a three+ digit number maybe_bug_re = re.compile(r".*\d\d\d+") def get_parser(): parser = argparse.ArgumentParser() parser.add_argument( "--all-json", type=os.path.abspath, help="Path to write json output to" ) parser.add_argument( "--untriaged", type=os.path.abspath, help="Path to write list of regressions with no associated bug", ) parser.add_argument( "--platform", dest="platforms", action="append", choices=list(run_infos.keys()), help="Configurations to compute fission changes for", ) commandline.add_logging_group(parser) return parser def allowed_results(test, subtest=None): return test.expected(subtest), test.known_intermittent(subtest) def is_worse(baseline_result, new_result): if new_result == baseline_result: return False if new_result in ("PASS", "OK"): return False if baseline_result in ("PASS", "OK"): return True # A crash -> not crash isn't a regression if baseline_result == "CRASH": return False return True def is_regression(baseline_result, new_result): if baseline_result == new_result: return False baseline_expected, baseline_intermittent = baseline_result new_expected, new_intermittent = new_result baseline_all = {baseline_expected} | set(baseline_intermittent) new_all = {new_expected} | set(new_intermittent) if baseline_all == new_all: return False if not baseline_intermittent and not new_intermittent: return is_worse(baseline_expected, new_expected) # If it was intermittent and isn't now, check if the new result is # worse than any of the previous results so that [PASS, FAIL] -> FAIL # looks like a regression if baseline_intermittent and not new_intermittent: return any(is_worse(result, new_expected) for result in baseline_all) # If it was a perma and is now intermittent, check if any new result is # worse than the previous result. if not baseline_intermittent and new_intermittent: return any(is_worse(baseline_expected, result) for result in new_all) # If it was an intermittent and is still an intermittent # check if any new result not in the old results is worse than # any old result new_results = new_all - baseline_all return any( is_worse(baseline_result, new_result) for new_result in new_results for baseline_result in baseline_all ) def get_meta_prop(test, subtest, name): for meta in test.itermeta(subtest): try: value = meta.get(name) except KeyError: pass else: return value return None def include_result(result): if result.disabled or result.regressions: return True if isinstance(result, TestResult): for subtest_result in result.subtest_results.values(): if subtest_result.disabled or subtest_result.regressions: return True return False class Result: def __init__(self): self.bugs = set() self.disabled = set() self.regressions = {} def add_regression(self, platform, baseline_results, fission_results): self.regressions[platform] = { "baseline": [baseline_results[0]] + baseline_results[1], "fission": [fission_results[0]] + fission_results[1], } def to_json(self): raise NotImplementedError def is_triaged(self): raise NotImplementedError class TestResult(Result): def __init__(self): super().__init__() self.subtest_results = {} def add_subtest(self, name): self.subtest_results[name] = SubtestResult(self) def to_json(self): rv = {} include_subtests = { name: item.to_json() for name, item in self.subtest_results.items() if include_result(item) } if include_subtests: rv["subtest_results"] = include_subtests if self.regressions: rv["regressions"] = self.regressions if self.disabled: rv["disabled"] = list(self.disabled) if self.bugs: rv["bugs"] = list(self.bugs) return rv def is_triaged(self): return bool(self.bugs) or ( not self.regressions and all( subtest_result.is_triaged() for subtest_result in self.subtest_results.values() ) ) class SubtestResult(Result): def __init__(self, parent): super().__init__() self.parent = parent def to_json(self): rv = {} if self.regressions: rv["regressions"] = self.regressions if self.disabled: rv["disabled"] = list(self.disabled) bugs = self.bugs - self.parent.bugs if bugs: rv["bugs"] = list(bugs) return rv def is_triaged(self): return bool(not self.regressions or self.parent.bugs or self.bugs) def run(logger, src_root, obj_root, **kwargs): commandline.setup_logging( logger, {key: value for key, value in kwargs.items() if key.startswith("log_")} ) import manifestupdate sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")), ) from wptrunner import testloader, wpttest logger.info("Loading test manifest") test_manifests = manifestupdate.run(src_root, obj_root, logger) test_results = {} platforms = kwargs["platforms"] if platforms is None: platforms = run_infos.keys() for platform in platforms: platform_run_info = run_infos[platform] run_info_baseline = platform_run_info.copy() run_info_baseline["fission"] = False tests = {} for kind in ("baseline", "fission"): logger.info("Loading tests %s %s" % (platform, kind)) run_info = platform_run_info.copy() run_info["fission"] = kind == "fission" subsuites = testloader.load_subsuites(logger, run_info, None, set()) test_loader = testloader.TestLoader( test_manifests, wpttest.enabled_tests, run_info, subsuites=subsuites, manifest_filters=[], ) tests[kind] = { test.id: test for _, _, test in test_loader.iter_tests( run_info, test_loader.manifest_filters ) if test._test_metadata is not None } for test_id, baseline_test in tests["baseline"].items(): fission_test = tests["fission"][test_id] if test_id not in test_results: test_results[test_id] = TestResult() test_result = test_results[test_id] baseline_bug = get_meta_prop(baseline_test, None, "bug") fission_bug = get_meta_prop(fission_test, None, "bug") if fission_bug and fission_bug != baseline_bug: test_result.bugs.add(fission_bug) if fission_test.disabled() and not baseline_test.disabled(): test_result.disabled.add(platform) reason = get_meta_prop(fission_test, None, "disabled") if reason and maybe_bug_re.match(reason): test_result.bugs.add(reason) baseline_results = allowed_results(baseline_test) fission_results = allowed_results(fission_test) result_is_regression = is_regression(baseline_results, fission_results) if baseline_results != fission_results: logger.debug( " %s %s %s %s" % (test_id, baseline_results, fission_results, result_is_regression) ) if result_is_regression: test_result.add_regression(platform, baseline_results, fission_results) for ( name, baseline_subtest_meta, ) in baseline_test._test_metadata.subtests.items(): fission_subtest_meta = baseline_test._test_metadata.subtests[name] if name not in test_result.subtest_results: test_result.add_subtest(name) subtest_result = test_result.subtest_results[name] baseline_bug = get_meta_prop(baseline_test, name, "bug") fission_bug = get_meta_prop(fission_test, name, "bug") if fission_bug and fission_bug != baseline_bug: subtest_result.bugs.add(fission_bug) if bool(fission_subtest_meta.disabled) and not bool( baseline_subtest_meta.disabled ): subtest_result.disabled.add(platform) if maybe_bug_re.match(fission_subtest_meta.disabled): subtest_result.bugs.add(fission_subtest_meta.disabled) baseline_results = allowed_results(baseline_test, name) fission_results = allowed_results(fission_test, name) result_is_regression = is_regression(baseline_results, fission_results) if baseline_results != fission_results: logger.debug( " %s %s %s %s %s" % ( test_id, name, baseline_results, fission_results, result_is_regression, ) ) if result_is_regression: subtest_result.add_regression( platform, baseline_results, fission_results ) test_results = { test_id: result for test_id, result in test_results.items() if include_result(result) } if kwargs["all_json"] is not None: write_all(test_results, kwargs["all_json"]) if kwargs["untriaged"] is not None: write_untriaged(test_results, kwargs["untriaged"]) def write_all(test_results, path): json_data = {test_id: result.to_json() for test_id, result in test_results.items()} dir_name = os.path.dirname(path) if not os.path.exists(dir_name): os.makedirs(dir_name) with open(path, "w") as f: json.dump(json_data, f, indent=2) def write_untriaged(test_results, path): dir_name = os.path.dirname(path) if not os.path.exists(dir_name): os.makedirs(dir_name) data = sorted( (test_id, result) for test_id, result in test_results.items() if not result.is_triaged() ) with open(path, "w") as f: for test_id, result in data: f.write(test_id + "\n") for name, subtest_result in sorted(result.subtest_results.items()): if not subtest_result.is_triaged(): f.write(" %s\n" % name)