summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/fissionregressions.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /testing/web-platform/fissionregressions.py
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/web-platform/fissionregressions.py')
-rw-r--r--testing/web-platform/fissionregressions.py519
1 files changed, 519 insertions, 0 deletions
diff --git a/testing/web-platform/fissionregressions.py b/testing/web-platform/fissionregressions.py
new file mode 100644
index 0000000000..409ba291e0
--- /dev/null
+++ b/testing/web-platform/fissionregressions.py
@@ -0,0 +1,519 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+import os
+import re
+import sys
+
+from mozlog import commandline
+
+run_infos = {
+ "linux-opt": {
+ "os": "linux",
+ "processor": "x86_64",
+ "version": "Ubuntu 18.04",
+ "os_version": "18.04",
+ "bits": 64,
+ "has_sandbox": True,
+ "automation": True,
+ "linux_distro": "Ubuntu",
+ "apple_silicon": False,
+ "appname": "firefox",
+ "artifact": False,
+ "asan": False,
+ "bin_suffix": "",
+ "buildapp": "browser",
+ "buildtype_guess": "pgo",
+ "cc_type": "clang",
+ "ccov": False,
+ "crashreporter": True,
+ "datareporting": True,
+ "debug": False,
+ "devedition": False,
+ "early_beta_or_earlier": True,
+ "healthreport": True,
+ "nightly_build": True,
+ "normandy": True,
+ "official": True,
+ "pgo": True,
+ "platform_guess": "linux64",
+ "release_or_beta": False,
+ "require_signing": False,
+ "stylo": True,
+ "sync": True,
+ "telemetry": False,
+ "tests_enabled": True,
+ "toolkit": "gtk",
+ "tsan": False,
+ "ubsan": False,
+ "updater": True,
+ "python_version": 3,
+ "product": "firefox",
+ "verify": False,
+ "wasm": True,
+ "e10s": True,
+ "headless": False,
+ "fission": True,
+ "sessionHistoryInParent": True,
+ "swgl": False,
+ "editorLegacyDirectionMode": False,
+ "win10_2004": False,
+ "win11_2009": False,
+ "domstreams": True,
+ "isolated_process": False,
+ },
+ "linux-debug": {
+ "os": "linux",
+ "processor": "x86_64",
+ "version": "Ubuntu 18.04",
+ "os_version": "18.04",
+ "bits": 64,
+ "has_sandbox": True,
+ "automation": True,
+ "linux_distro": "Ubuntu",
+ "apple_silicon": False,
+ "appname": "firefox",
+ "artifact": False,
+ "asan": False,
+ "bin_suffix": "",
+ "buildapp": "browser",
+ "buildtype_guess": "debug",
+ "cc_type": "clang",
+ "ccov": False,
+ "crashreporter": True,
+ "datareporting": True,
+ "debug": True,
+ "devedition": False,
+ "early_beta_or_earlier": True,
+ "healthreport": True,
+ "nightly_build": True,
+ "normandy": True,
+ "official": True,
+ "pgo": False,
+ "platform_guess": "linux64",
+ "release_or_beta": False,
+ "require_signing": False,
+ "stylo": True,
+ "sync": True,
+ "telemetry": False,
+ "tests_enabled": True,
+ "toolkit": "gtk",
+ "tsan": False,
+ "ubsan": False,
+ "updater": True,
+ "python_version": 3,
+ "product": "firefox",
+ "verify": False,
+ "wasm": True,
+ "e10s": True,
+ "headless": False,
+ "fission": False,
+ "sessionHistoryInParent": False,
+ "swgl": False,
+ "editorLegacyDirectionMode": False,
+ "win10_2004": False,
+ "win11_2009": False,
+ "domstreams": True,
+ "isolated_process": False,
+ },
+ "win-opt": {
+ "os": "win",
+ "processor": "x86_64",
+ "version": "10.0.17134",
+ "os_version": "10.0",
+ "bits": 64,
+ "has_sandbox": True,
+ "automation": True,
+ "service_pack": "",
+ "apple_silicon": False,
+ "appname": "firefox",
+ "artifact": False,
+ "asan": False,
+ "bin_suffix": ".exe",
+ "buildapp": "browser",
+ "buildtype_guess": "pgo",
+ "cc_type": "clang-cl",
+ "ccov": False,
+ "crashreporter": True,
+ "datareporting": True,
+ "debug": False,
+ "devedition": False,
+ "early_beta_or_earlier": True,
+ "healthreport": True,
+ "nightly_build": True,
+ "normandy": True,
+ "official": True,
+ "pgo": True,
+ "platform_guess": "win64",
+ "release_or_beta": False,
+ "require_signing": False,
+ "stylo": True,
+ "sync": True,
+ "telemetry": False,
+ "tests_enabled": True,
+ "toolkit": "windows",
+ "tsan": False,
+ "ubsan": False,
+ "updater": True,
+ "python_version": 3,
+ "product": "firefox",
+ "verify": False,
+ "wasm": True,
+ "e10s": True,
+ "headless": False,
+ "fission": False,
+ "sessionHistoryInParent": False,
+ "swgl": False,
+ "editorLegacyDirectionMode": False,
+ "win10_2004": False,
+ "win11_2009": False,
+ "domstreams": True,
+ "isolated_process": False,
+ },
+}
+
+
+# RE that checks for anything containing a three+ digit number
+maybe_bug_re = re.compile(r".*\d\d\d+")
+
+
+def get_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--all-json", type=os.path.abspath, help="Path to write json output to"
+ )
+ parser.add_argument(
+ "--untriaged",
+ type=os.path.abspath,
+ help="Path to write list of regressions with no associated bug",
+ )
+ parser.add_argument(
+ "--platform",
+ dest="platforms",
+ action="append",
+ choices=list(run_infos.keys()),
+ help="Configurations to compute fission changes for",
+ )
+ commandline.add_logging_group(parser)
+ return parser
+
+
+def allowed_results(test, subtest=None):
+ return test.expected(subtest), test.known_intermittent(subtest)
+
+
+def is_worse(baseline_result, new_result):
+ if new_result == baseline_result:
+ return False
+
+ if new_result in ("PASS", "OK"):
+ return False
+
+ if baseline_result in ("PASS", "OK"):
+ return True
+
+ # A crash -> not crash isn't a regression
+ if baseline_result == "CRASH":
+ return False
+
+ return True
+
+
+def is_regression(baseline_result, new_result):
+ if baseline_result == new_result:
+ return False
+
+ baseline_expected, baseline_intermittent = baseline_result
+ new_expected, new_intermittent = new_result
+
+ baseline_all = {baseline_expected} | set(baseline_intermittent)
+ new_all = {new_expected} | set(new_intermittent)
+
+ if baseline_all == new_all:
+ return False
+
+ if not baseline_intermittent and not new_intermittent:
+ return is_worse(baseline_expected, new_expected)
+
+ # If it was intermittent and isn't now, check if the new result is
+ # worse than any of the previous results so that [PASS, FAIL] -> FAIL
+ # looks like a regression
+ if baseline_intermittent and not new_intermittent:
+ return any(is_worse(result, new_expected) for result in baseline_all)
+
+ # If it was a perma and is now intermittent, check if any new result is
+ # worse than the previous result.
+ if not baseline_intermittent and new_intermittent:
+ return any(is_worse(baseline_expected, result) for result in new_all)
+
+ # If it was an intermittent and is still an intermittent
+ # check if any new result not in the old results is worse than
+ # any old result
+ new_results = new_all - baseline_all
+ return any(
+ is_worse(baseline_result, new_result)
+ for new_result in new_results
+ for baseline_result in baseline_all
+ )
+
+
+def get_meta_prop(test, subtest, name):
+ for meta in test.itermeta(subtest):
+ try:
+ value = meta.get(name)
+ except KeyError:
+ pass
+ else:
+ return value
+ return None
+
+
+def include_result(result):
+ if result.disabled or result.regressions:
+ return True
+
+ if isinstance(result, TestResult):
+ for subtest_result in result.subtest_results.values():
+ if subtest_result.disabled or subtest_result.regressions:
+ return True
+
+ return False
+
+
+class Result:
+ def __init__(self):
+ self.bugs = set()
+ self.disabled = set()
+ self.regressions = {}
+
+ def add_regression(self, platform, baseline_results, fission_results):
+ self.regressions[platform] = {
+ "baseline": [baseline_results[0]] + baseline_results[1],
+ "fission": [fission_results[0]] + fission_results[1],
+ }
+
+ def to_json(self):
+ raise NotImplementedError
+
+ def is_triaged(self):
+ raise NotImplementedError
+
+
+class TestResult(Result):
+ def __init__(self):
+ super().__init__()
+ self.subtest_results = {}
+
+ def add_subtest(self, name):
+ self.subtest_results[name] = SubtestResult(self)
+
+ def to_json(self):
+ rv = {}
+ include_subtests = {
+ name: item.to_json()
+ for name, item in self.subtest_results.items()
+ if include_result(item)
+ }
+ if include_subtests:
+ rv["subtest_results"] = include_subtests
+ if self.regressions:
+ rv["regressions"] = self.regressions
+ if self.disabled:
+ rv["disabled"] = list(self.disabled)
+ if self.bugs:
+ rv["bugs"] = list(self.bugs)
+ return rv
+
+ def is_triaged(self):
+ return bool(self.bugs) or (
+ not self.regressions
+ and all(
+ subtest_result.is_triaged()
+ for subtest_result in self.subtest_results.values()
+ )
+ )
+
+
+class SubtestResult(Result):
+ def __init__(self, parent):
+ super().__init__()
+ self.parent = parent
+
+ def to_json(self):
+ rv = {}
+ if self.regressions:
+ rv["regressions"] = self.regressions
+ if self.disabled:
+ rv["disabled"] = list(self.disabled)
+ bugs = self.bugs - self.parent.bugs
+ if bugs:
+ rv["bugs"] = list(bugs)
+ return rv
+
+ def is_triaged(self):
+ return bool(not self.regressions or self.parent.bugs or self.bugs)
+
+
+def run(logger, src_root, obj_root, **kwargs):
+ commandline.setup_logging(
+ logger, {key: value for key, value in kwargs.items() if key.startswith("log_")}
+ )
+
+ import manifestupdate
+
+ sys.path.insert(
+ 0,
+ os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
+ )
+ from wptrunner import testloader, wpttest
+
+ logger.info("Loading test manifest")
+ test_manifests = manifestupdate.run(src_root, obj_root, logger)
+
+ test_results = {}
+
+ platforms = kwargs["platforms"]
+ if platforms is None:
+ platforms = run_infos.keys()
+
+ for platform in platforms:
+ platform_run_info = run_infos[platform]
+ run_info_baseline = platform_run_info.copy()
+ run_info_baseline["fission"] = False
+
+ tests = {}
+
+ for kind in ("baseline", "fission"):
+ logger.info("Loading tests %s %s" % (platform, kind))
+ run_info = platform_run_info.copy()
+ run_info["fission"] = kind == "fission"
+
+ test_loader = testloader.TestLoader(
+ test_manifests, wpttest.enabled_tests, run_info, manifest_filters=[]
+ )
+ tests[kind] = {
+ test.id: test
+ for _, _, test in test_loader.iter_tests()
+ if test._test_metadata is not None
+ }
+
+ for test_id, baseline_test in tests["baseline"].items():
+ fission_test = tests["fission"][test_id]
+
+ if test_id not in test_results:
+ test_results[test_id] = TestResult()
+
+ test_result = test_results[test_id]
+
+ baseline_bug = get_meta_prop(baseline_test, None, "bug")
+ fission_bug = get_meta_prop(fission_test, None, "bug")
+ if fission_bug and fission_bug != baseline_bug:
+ test_result.bugs.add(fission_bug)
+
+ if fission_test.disabled() and not baseline_test.disabled():
+ test_result.disabled.add(platform)
+ reason = get_meta_prop(fission_test, None, "disabled")
+ if reason and maybe_bug_re.match(reason):
+ test_result.bugs.add(reason)
+
+ baseline_results = allowed_results(baseline_test)
+ fission_results = allowed_results(fission_test)
+ result_is_regression = is_regression(baseline_results, fission_results)
+
+ if baseline_results != fission_results:
+ logger.debug(
+ " %s %s %s %s"
+ % (test_id, baseline_results, fission_results, result_is_regression)
+ )
+
+ if result_is_regression:
+ test_result.add_regression(platform, baseline_results, fission_results)
+
+ for (
+ name,
+ baseline_subtest_meta,
+ ) in baseline_test._test_metadata.subtests.items():
+ fission_subtest_meta = baseline_test._test_metadata.subtests[name]
+ if name not in test_result.subtest_results:
+ test_result.add_subtest(name)
+
+ subtest_result = test_result.subtest_results[name]
+
+ baseline_bug = get_meta_prop(baseline_test, name, "bug")
+ fission_bug = get_meta_prop(fission_test, name, "bug")
+ if fission_bug and fission_bug != baseline_bug:
+ subtest_result.bugs.add(fission_bug)
+
+ if bool(fission_subtest_meta.disabled) and not bool(
+ baseline_subtest_meta.disabled
+ ):
+ subtest_result.disabled.add(platform)
+ if maybe_bug_re.match(fission_subtest_meta.disabled):
+ subtest_result.bugs.add(fission_subtest_meta.disabled)
+
+ baseline_results = allowed_results(baseline_test, name)
+ fission_results = allowed_results(fission_test, name)
+
+ result_is_regression = is_regression(baseline_results, fission_results)
+
+ if baseline_results != fission_results:
+ logger.debug(
+ " %s %s %s %s %s"
+ % (
+ test_id,
+ name,
+ baseline_results,
+ fission_results,
+ result_is_regression,
+ )
+ )
+
+ if result_is_regression:
+ subtest_result.add_regression(
+ platform, baseline_results, fission_results
+ )
+
+ test_results = {
+ test_id: result
+ for test_id, result in test_results.items()
+ if include_result(result)
+ }
+
+ if kwargs["all_json"] is not None:
+ write_all(test_results, kwargs["all_json"])
+
+ if kwargs["untriaged"] is not None:
+ write_untriaged(test_results, kwargs["untriaged"])
+
+
+def write_all(test_results, path):
+ json_data = {test_id: result.to_json() for test_id, result in test_results.items()}
+
+ dir_name = os.path.dirname(path)
+ if not os.path.exists(dir_name):
+ os.makedirs(dir_name)
+
+ with open(path, "w") as f:
+ json.dump(json_data, f, indent=2)
+
+
+def write_untriaged(test_results, path):
+ dir_name = os.path.dirname(path)
+ if not os.path.exists(dir_name):
+ os.makedirs(dir_name)
+
+ data = sorted(
+ (test_id, result)
+ for test_id, result in test_results.items()
+ if not result.is_triaged()
+ )
+
+ with open(path, "w") as f:
+ for test_id, result in data:
+ f.write(test_id + "\n")
+ for name, subtest_result in sorted(result.subtest_results.items()):
+ if not subtest_result.is_triaged():
+ f.write(" %s\n" % name)