diff options
Diffstat (limited to 'testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py')
| -rw-r--r-- | testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py | 217 |
1 files changed, 217 insertions, 0 deletions
diff --git a/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py b/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py new file mode 100644 index 0000000000..f1a514ccd0 --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy +import re + +import filters +from base_python_support import BasePythonSupport +from utils import bool_from_str + +DOMAIN_MATCHER = re.compile(r"(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n]+)") +VARIANCE_THRESHOLD = 0.2 + + +def extract_domain(link): + match = DOMAIN_MATCHER.search(link) + if match: + return match.group(1) + raise Exception(f"Could not find domain for {link}") + + +class TP6BenchSupport(BasePythonSupport): + def __init__(self, **kwargs): + self._load_times = [] + self._total_times = [] + self._geomean_load_times = [] + self._sites_tested = 0 + self._test_pages = {} + + def setup_test(self, test, args): + from cmdline import DESKTOP_APPS + from manifest import get_browser_test_list + from utils import transform_subtest + + all_tests = get_browser_test_list(args.app, args.run_local) + + manifest_to_find = "browsertime-tp6.toml" + if args.app not in DESKTOP_APPS: + manifest_to_find = "browsertime-tp6m.toml" + + test_urls = [] + playback_pageset_manifests = [] + for parsed_test in all_tests: + if manifest_to_find in parsed_test["manifest"]: + if not bool_from_str(parsed_test.get("benchmark_page", "false")): + continue + test_url = parsed_test["test_url"] + test_urls.append(test_url) + playback_pageset_manifests.append( + transform_subtest( + parsed_test["playback_pageset_manifest"], parsed_test["name"] + ) + ) + self._test_pages[test_url] = parsed_test + + if len(playback_pageset_manifests) == 0: + raise Exception("Could not find any manifests for testing.") + + test["test_url"] = ",".join(test_urls) + test["playback_pageset_manifest"] = ",".join(playback_pageset_manifests) + + def handle_result(self, bt_result, raw_result, last_result=False, **kwargs): + measurements = {"totalTime": []} + + # Find new results to add + for res in raw_result["extras"]: + if "pageload-benchmark" in res: + total_time = int( + round(res["pageload-benchmark"].get("totalTime", 0), 0) + ) + measurements["totalTime"].append(total_time) + self._total_times.append(total_time) + + # Gather the load times of each page/cycle to help with diagnosing issues + load_times = [] + result_name = None + for cycle in raw_result["browserScripts"]: + if not result_name: + # When the test name is unknown, we use the url TLD combined + # with the page title to differentiate pages with similar domains, and + # limit it to 35 characters + page_url = cycle["pageinfo"].get("url", "") + if self._test_pages.get(page_url, None) is not None: + result_name = self._test_pages[page_url]["name"] + else: + page_name = extract_domain(page_url) + page_title = ( + cycle["pageinfo"].get("documentTitle", "")[:35].replace(" ", "") + ) + result_name = f"{page_name} - {page_title}" + + load_time = cycle["timings"]["loadEventEnd"] + fcp = cycle["timings"]["paintTiming"]["first-contentful-paint"] + lcp = ( + cycle["timings"] + .get("largestContentfulPaint", {}) + .get("renderTime", None) + ) + measurements.setdefault(f"{result_name} - loadTime", []).append(load_time) + + measurements.setdefault(f"{result_name} - fcp", []).append(fcp) + + if lcp is not None: + measurements.setdefault(f"{result_name} - lcp", []).append(lcp) + + load_times.append(load_time) + + cur_load_times = self._load_times or [0] * len(load_times) + self._load_times = list(map(sum, zip(cur_load_times, load_times))) + self._sites_tested += 1 + + for measurement, values in measurements.items(): + bt_result["measurements"].setdefault(measurement, []).extend(values) + if last_result: + bt_result["measurements"]["totalLoadTime"] = self._load_times + bt_result["measurements"]["totalLoadTimePerSite"] = [ + round(total_load_time / self._sites_tested, 2) + for total_load_time in self._load_times + ] + bt_result["measurements"]["totalTimePerSite"] = [ + round(total_time / self._sites_tested, 2) + for total_time in self._total_times + ] + + def _build_subtest(self, measurement_name, replicates, test): + unit = test.get("unit", "ms") + if test.get("subtest_unit"): + unit = test.get("subtest_unit") + + return { + "name": measurement_name, + "lowerIsBetter": test.get("lower_is_better", True), + "alertThreshold": float(test.get("alert_threshold", 2.0)), + "unit": unit, + "replicates": replicates, + "value": round(filters.geometric_mean(replicates), 3), + } + + def summarize_test(self, test, suite, **kwargs): + suite["type"] = "pageload" + if suite["subtests"] == {}: + suite["subtests"] = [] + for measurement_name, replicates in test["measurements"].items(): + if not replicates: + continue + suite["subtests"].append( + self._build_subtest(measurement_name, replicates, test) + ) + suite["subtests"].sort(key=lambda subtest: subtest["name"]) + + def _produce_suite_alts(self, suite_base, subtests, suite_name_prefix): + geomean_suite = copy.deepcopy(suite_base) + geomean_suite["subtests"] = copy.deepcopy(subtests) + median_suite = copy.deepcopy(geomean_suite) + median_suite["subtests"] = copy.deepcopy(subtests) + + subtest_values = [] + for subtest in subtests: + subtest_values.extend(subtest["replicates"]) + + geomean_suite["name"] = suite_name_prefix + "-geomean" + geomean_suite["value"] = filters.geometric_mean(subtest_values) + for subtest in geomean_suite["subtests"]: + subtest["value"] = filters.geometric_mean(subtest["replicates"]) + + median_suite["name"] = suite_name_prefix + "-median" + median_suite["value"] = filters.median(subtest_values) + for subtest in median_suite["subtests"]: + subtest["value"] = filters.median(subtest["replicates"]) + + return [ + geomean_suite, + median_suite, + ] + + def summarize_suites(self, suites): + fcp_subtests = [] + lcp_subtests = [] + load_time_subtests = [] + + for suite in suites: + for subtest in suite["subtests"]: + if "- fcp" in subtest["name"]: + fcp_subtests.append(subtest) + elif "- lcp" in subtest["name"]: + lcp_subtests.append(subtest) + elif "- loadTime" in subtest["name"]: + load_time_subtests.append(subtest) + + fcp_bench_suites = self._produce_suite_alts( + suites[0], fcp_subtests, "fcp-bench" + ) + + lcp_bench_suites = self._produce_suite_alts( + suites[0], lcp_subtests, "lcp-bench" + ) + + load_time_bench_suites = self._produce_suite_alts( + suites[0], load_time_subtests, "loadtime-bench" + ) + + overall_suite = copy.deepcopy(suites[0]) + new_subtests = [ + subtest + for subtest in suites[0]["subtests"] + if subtest["name"].startswith("total") + ] + overall_suite["subtests"] = new_subtests + + new_suites = [ + overall_suite, + *fcp_bench_suites, + *lcp_bench_suites, + *load_time_bench_suites, + ] + suites.pop() + suites.extend(new_suites) |