diff options
Diffstat (limited to 'testing/web-platform/interop.py')
| -rw-r--r-- | testing/web-platform/interop.py | 302 |
1 files changed, 302 insertions, 0 deletions
diff --git a/testing/web-platform/interop.py b/testing/web-platform/interop.py new file mode 100644 index 0000000000..658d368162 --- /dev/null +++ b/testing/web-platform/interop.py @@ -0,0 +1,302 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import csv +import math +import os +import re +import shutil +import sys +import tempfile +from typing import Callable, Iterable, List, Mapping, Optional, Set, Tuple + +repos = ["autoland", "mozilla-central", "try", "mozilla-central", "mozilla-beta", "wpt"] + +default_fetch_task_filters = ["-web-platform-tests-|-spidermonkey-"] +default_interop_task_filters = { + "wpt": ["-firefox-"], + None: [ + "web-platform-tests", + "linux.*-64", + "/opt", + "!-nofis|-headless|-asan|-tsan|-ccov", + ], +} + + +def get_parser_fetch_logs() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--log-dir", action="store", help="Directory into which to download logs" + ) + parser.add_argument( + "--task-filter", + dest="task_filters", + action="append", + help="Regex filter applied to task names. Filters starting ! must not match. Filters starting ^ (after any !) match the entire task name, otherwise any substring can match. Multiple filters must all match", + ) + parser.add_argument( + "--check-complete", + action="store_true", + help="Only download logs if the task is complete", + ) + parser.add_argument( + "commits", + nargs="+", + help="repo:commit e.g. mozilla-central:fae24810aef1 for the runs to include", + ) + return parser + + +def get_parser_interop_score() -> argparse.Namespace: + parser = get_parser_fetch_logs() + parser.add_argument( + "--year", + action="store", + default=2023, + type=int, + help="Interop year to score against", + ) + parser.add_argument( + "--category-filter", + action="append", + dest="category_filters", + help="Regex filter applied to category names. Filters starting ! must not match. Filters starting ^ (after any !) match the entire task name, otherwise any substring can match. Multiple filters must all match", + ) + parser.add_argument( + "--expected-failures", + help="Path to a file containing a list of tests which are not expected to pass", + ) + return parser + + +def print_scores( + runs: Iterable[Tuple[str, str]], + results_by_category: Mapping[str, List[int]], + expected_failures_by_category: Optional[Mapping[str, List[Tuple[int, int]]]], + include_total: bool, +): + include_expected_failures = expected_failures_by_category is not None + + writer = csv.writer(sys.stdout, delimiter="\t") + + headers = ["Category"] + for repo, commit in runs: + prefix = f"{repo}:{commit}" + headers.append(f"{prefix}-score") + if include_expected_failures: + headers.append(f"{prefix}-expected-failures") + headers.append(f"{prefix}-adjusted-score") + + writer.writerow(headers) + + totals = {"score": [0.0] * len(runs)} + if include_expected_failures: + totals["expected_failures"] = [0.0] * len(runs) + totals["adjusted_score"] = [0.0] * len(runs) + + for category, category_results in results_by_category.items(): + category_row = [] + category_row.append(category) + for category_index, result in enumerate(category_results): + for run_index, run_score in enumerate(category_results): + category_row.append(f"{run_score / 10:.1f}") + totals["score"][run_index] += run_score + if include_expected_failures: + expected_failures, adjusted_score = expected_failures_by_category[ + category + ][run_index] + category_row.append(f"{expected_failures / 10:.1f}") + category_row.append(f"{adjusted_score / 10:.1f}") + totals["expected_failures"][run_index] += expected_failures + totals["adjusted_score"][run_index] += adjusted_score + writer.writerow(category_row) + + if include_total: + + def get_total(score, floor=True): + total = float(score) / (len(results_by_category)) + if floor: + total = math.floor(total) + total /= 10.0 + return total + + totals_row = ["Total"] + for i in range(len(runs)): + totals_row.append(f"{get_total(totals['score'][i]):.1f}") + if include_expected_failures: + totals_row.append( + f"{get_total(totals['expected_failures'][i], floor=False):.1f}" + ) + totals_row.append(f"{get_total(totals['adjusted_score'][i]):.1f}") + writer.writerow(totals_row) + + +def get_wptreports( + repo: str, commit: str, task_filters: List[str], log_dir: str, check_complete: bool +) -> List[str]: + import tcfetch + + return tcfetch.download_artifacts( + repo, + commit, + task_filters=task_filters, + check_complete=check_complete, + out_dir=log_dir, + ) + + +def get_runs(commits: List[str]) -> List[Tuple[str, str]]: + runs = [] + for item in commits: + if ":" not in item: + raise ValueError(f"Expected commits of the form repo:commit, got {item}") + repo, commit = item.split(":", 1) + if repo not in repos: + raise ValueError(f"Unsupported repo {repo}") + runs.append((repo, commit)) + return runs + + +def get_category_filter( + category_filters: Optional[List[str]], +) -> Optional[Callable[[str], bool]]: + if category_filters is None: + return None + + filters = [] + for item in category_filters: + if not item: + continue + invert = item[0] == "!" + if invert: + item = item[1:] + if item[0] == "^": + regex = re.compile(item) + else: + regex = re.compile(f"^(.*)(?:{item})") + filters.append((regex, invert)) + + def match_filters(category): + for regex, invert in filters: + matches = regex.match(category) is not None + if invert: + matches = not matches + if not matches: + return False + return True + + return match_filters + + +def fetch_logs( + commits: List[str], + task_filters: List[str], + log_dir: Optional[str], + check_complete: bool, + **kwargs, +): + runs = get_runs(commits) + + if not task_filters: + task_filters = default_fetch_task_filters + + if log_dir is None: + log_dir = os.path.abspath(os.curdir) + + for repo, commit in runs: + get_wptreports(repo, commit, task_filters, log_dir, check_complete) + + +def get_expected_failures(path: str) -> Mapping[str, Set[Optional[str]]]: + expected_failures = {} + with open(path) as f: + for i, entry in enumerate(csv.reader(f)): + entry = [item.strip() for item in entry] + if not any(item for item in entry) or entry[0][0] == "#": + continue + if len(entry) > 2: + raise ValueError( + f"{path}:{i+1} expected at most two columns, got {len(entry)}" + ) + if entry[0][0] != "/": + raise ValueError( + f'{path}:{i+1} "{entry[0]}" is not a valid test id (must start with "/")' + ) + test_id = entry[0] + if test_id not in expected_failures: + expected_failures[test_id] = set() + if len(entry) == 2: + subtest_id = entry[1] + if subtest_id == "": + print( + f"Warning: {path}:{i+1} got empty string subtest id, remove the trailing comma to make this apply to the full test" + ) + expected_failures[test_id].add(subtest_id) + else: + expected_failures[test_id].add(None) + return expected_failures + + +def score_runs( + commits: List[str], + task_filters: List[str], + log_dir: Optional[str], + year: int, + check_complete: bool, + category_filters: Optional[List[str]], + expected_failures: Optional[str], + **kwargs, +): + from wpt_interop import score + + runs = get_runs(commits) + + temp_dir = None + if log_dir is None: + temp_dir = tempfile.mkdtemp() + log_dir = temp_dir + + try: + if expected_failures is not None: + expected_failures_data = get_expected_failures(expected_failures) + else: + expected_failures_data = None + + run_logs = [] + for repo, commit in runs: + if not task_filters: + if repo in default_interop_task_filters: + filters = default_interop_task_filters[repo] + else: + filters = default_interop_task_filters[None] + else: + filters = task_filters + + log_paths = get_wptreports(repo, commit, filters, log_dir, check_complete) + if not log_paths: + print(f"Failed to get any logs for {repo}:{commit}", file=sys.stderr) + else: + run_logs.append(log_paths) + + if not run_logs: + print("No logs to process", file=sys.stderr) + + include_total = category_filters is None + + category_filter = ( + get_category_filter(category_filters) if category_filters else None + ) + + scores, expected_failure_scores = score.score_wptreports( + run_logs, + year=year, + category_filter=category_filter, + expected_failures=expected_failures_data, + ) + print_scores(runs, scores, expected_failure_scores, include_total) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir, True) |