diff options
Diffstat (limited to 'testing/performance')
32 files changed, 2598 insertions, 0 deletions
diff --git a/testing/performance/.eslintrc.js b/testing/performance/.eslintrc.js new file mode 100644 index 0000000000..f040032509 --- /dev/null +++ b/testing/performance/.eslintrc.js @@ -0,0 +1,12 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + browser: true, + node: true, + }, +}; diff --git a/testing/performance/README.txt b/testing/performance/README.txt new file mode 100644 index 0000000000..7cce5088c4 --- /dev/null +++ b/testing/performance/README.txt @@ -0,0 +1 @@ +This directory contains perftests owned by the performance team. diff --git a/testing/performance/fxrecord/perfdocs/config.yml b/testing/performance/fxrecord/perfdocs/config.yml new file mode 100644 index 0000000000..59f10455fa --- /dev/null +++ b/testing/performance/fxrecord/perfdocs/config.yml @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +name: fxrecord +manifest: None +static-only: True +suites: {} diff --git a/testing/performance/fxrecord/perfdocs/index.rst b/testing/performance/fxrecord/perfdocs/index.rst new file mode 100644 index 0000000000..313d4b74a3 --- /dev/null +++ b/testing/performance/fxrecord/perfdocs/index.rst @@ -0,0 +1,8 @@ +======== +Fxrecord +======== + +`Fxrecord <https://github.com/mozilla/fxrecord>`__ is a tool for measuring the +startup performance of Firefox for Desktop. It captures a video of Firefox for +desktop starting on a laptop and computes visual metrics using the same manner +as Raptor using Browsertime. diff --git a/testing/performance/hooks.py b/testing/performance/hooks.py new file mode 100644 index 0000000000..1a02f1ab3c --- /dev/null +++ b/testing/performance/hooks.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os + +from mozperftest.test.browsertime import add_option, add_options + +sites = os.path.join(os.path.dirname(__file__), "sites.txt") +with open(sites) as f: + sites = [site for site in f.read().split("\n") if site.strip()] + + +def next_site(): + for site in sites: + yield site + + +get_site = next_site() + +options = [ + ("firefox.preference", "network.http.speculative-parallel-limit:6"), + # XXX potentially move those as first class options in mozperf? + ("pageCompleteWaitTime", "10000"), + ("visualMetrics", "true"), + ("video", "true"), + ("firefox.windowRecorder", "false"), + ("videoParams.addTimer", "false"), + ("videoParams.createFilmstrip", "false"), + ("videoParams.keepOriginalVideo", "true"), +] + + +def before_runs(env, **kw): + env.set_arg("cycles", len(sites)) + add_options(env, options) + + +def before_cycle(env, **kw): + url = next(get_site) + add_option(env, "browsertime.url", url) diff --git a/testing/performance/hooks_android_main.py b/testing/performance/hooks_android_main.py new file mode 100644 index 0000000000..59c1628dab --- /dev/null +++ b/testing/performance/hooks_android_main.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mozperftest.test.browsertime import add_options +from mozperftest.test.browsertime.runner import NodeException + +common_options = [ + ("firefox.disableBrowsertimeExtension", "true"), + # The webdriver session is not created in the MAIN test so reduce the timeout + # Bug 1640638 + ("timeouts.browserStart", "20000"), + ("browserRestartTries", "1"), + # Explicitly disable the Onboarding flow + ("firefox.android.intentArgument", "'--ez'"), + ("firefox.android.intentArgument", "'performancetest'"), + ("firefox.android.intentArgument", "'true'"), +] + + +def on_exception(env, layer, exc): + if not isinstance(exc, NodeException): + raise exc + return True + + +def logcat_processor(): + pass + + +def before_runs(env, **kw): + add_options(env, common_options) diff --git a/testing/performance/hooks_android_startup.py b/testing/performance/hooks_android_startup.py new file mode 100644 index 0000000000..863df74447 --- /dev/null +++ b/testing/performance/hooks_android_startup.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pathlib +import re +import subprocess +from datetime import datetime, timedelta + +import requests +from mozperftest.system.android_startup import ( + BASE_URL_DICT, + DATETIME_FORMAT, + KEY_ARCHITECTURE, + KEY_COMMIT, + KEY_DATETIME, + KEY_NAME, + KEY_PRODUCT, +) + +HTTP_200_OKAY = 200 + + +def before_iterations(kw): + product = kw["AndroidStartUp_product"] + architecture = "arm64-v8a" + if product == "geckoview_example": + architecture = "aarch64" + commit_info = subprocess.getoutput("hg log -l 1") + commit_date = re.search(r"date:\s+([:\s\w]+)\s+", str(commit_info)).group(1) + download_date = ( + datetime.strptime(commit_date, "%a %b %d %H:%M:%S %Y") - timedelta(days=1) + ).strftime(DATETIME_FORMAT) + + nightly_url = BASE_URL_DICT[product].format( + date=download_date, architecture=architecture + ) + filename = f"{product}_nightly_{architecture}.apk" + print("Fetching {}...".format(filename), end="", flush=True) + download_apk_as_date(nightly_url, download_date, filename) + print(f"Downloaded {product} for date: {download_date}") + + kw["apk_metadata"] = { + KEY_NAME: filename, + KEY_DATETIME: download_date, + KEY_COMMIT: "", + KEY_ARCHITECTURE: architecture, + KEY_PRODUCT: product, + } + + +def download_apk_as_date(nightly_url, download_date_string, filename): + apk = requests.get(nightly_url) + if apk.status_code != HTTP_200_OKAY: + raise Exception( + f"Something went wrong downloading the apk check to make sure you have entered" + f" a date that is valid and that the apk for the date you have requested(" + f"{download_date_string}) is available and that the URL({nightly_url}) is also " + f"valid" + ) + pathlib.Path(filename).write_bytes(apk.content) diff --git a/testing/performance/hooks_android_view.py b/testing/performance/hooks_android_view.py new file mode 100644 index 0000000000..667d78f9ed --- /dev/null +++ b/testing/performance/hooks_android_view.py @@ -0,0 +1,154 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import pathlib +import tempfile +import time + +from mozperftest.system.android import _ROOT_URL +from mozperftest.test.browsertime import add_options +from mozperftest.utils import ( + download_file, + get_multi_tasks_url, + get_revision_namespace_url, + install_package, +) + +# We specifically select this URL because: +# - we access Mozilla URLs in tests so any connections to Mozilla URLs may re-use +# existing connections and have different perf characteristics. We can mostly +# avoid this problem by using a non-Mozilla URL +# - we authored the site so we can guarantee it doesn't change or change it if +# needed +# - we're not directing traffic to a site we don't own +URL = "'https://mozilla-mobile.github.io/perf-tools/mozperftest-test-page.html'" + +COMMON_OPTIONS = [ + ("processStartTime", "true"), + ("firefox.disableBrowsertimeExtension", "true"), + ("firefox.android.intentArgument", "'-a'"), + ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), + ("firefox.android.intentArgument", "'-d'"), + ("firefox.android.intentArgument", URL), +] + +NIGHTLY_SIM_ROUTE = "mobile.v3.firefox-android.apks.fenix-nightly-simulation" +ROUTE_SUFFIX = "artifacts/public/build/fenix/{architecture}/target.apk" + +build_generator = None + + +def before_iterations(kw): + global build_generator + + install_list = kw.get("android_install_apk") + if len(install_list) == 0 or all( + ["fenix_nightlysim_multicommit" not in apk for apk in install_list] + ): + return + + # Install gitpython + install_package(kw["virtualenv"], "gitpython==3.1.0") + import git + + class _GitProgress(git.RemoteProgress): + def update(self, op_code, cur_count, max_count=None, message=""): + if message: + print(message) + + # Setup the local fenix github repo + print("Cloning fenix repo...") + fenix_repo = git.Repo.clone_from( + "https://github.com/mozilla-mobile/fenix", + tempfile.mkdtemp(), + branch="master", + progress=_GitProgress(), + ) + + # Get the builds to test + architecture = ( + "arm64-v8a" if "arm64_v8a" in kw.get("android_install_apk") else "armeabi-v7a" + ) + json_ = _fetch_json( + get_revision_namespace_url, NIGHTLY_SIM_ROUTE, day=kw["test_date"] + ) + namespaces = json_["namespaces"] + revisions = [namespace["name"] for namespace in namespaces] + + tasks = [] + for revision in revisions: + try: + commit = fenix_repo.commit(revision) + name_rev = str(commit.name_rev) + if ( + "remotes/origin" not in name_rev + or "release" in name_rev + or "tag" in name_rev + ): + print( + "Commit %s is a release-branch commit, it won't be tested." + % revision + ) + continue + + commitdate = commit.committed_date + except ValueError: + print("Commit %s is not from the Fenix master branch" % revision) + continue + + json_ = _fetch_json( + get_multi_tasks_url, NIGHTLY_SIM_ROUTE, revision, day=kw["test_date"] + ) + for task in json_["tasks"]: + route = task["namespace"] + task_architecture = route.split(".")[-1] + if task_architecture == architecture: + tasks.append( + { + "timestamp": commitdate, + "revision": revision, + "route": route, + "route_suffix": ROUTE_SUFFIX.format( + architecture=task_architecture + ), + } + ) + + # Set the number of test-iterations to the number of builds + kw["test_iterations"] = len(tasks) + + def _build_iterator(): + for task in tasks: + revision = task["revision"] + timestamp = task["timestamp"] + + humandate = time.ctime(int(timestamp)) + print(f"Testing revision {revision} from {humandate}") + + download_url = f'{_ROOT_URL}{task["route"]}/{task["route_suffix"]}' + yield revision, timestamp, [download_url] + + build_generator = _build_iterator() + + return kw + + +def _fetch_json(get_url_function, *args, **kwargs): + build_url = get_url_function(*args, **kwargs) + tmpfile = pathlib.Path(tempfile.mkdtemp(), "temp.json") + download_file(build_url, tmpfile) + + with tmpfile.open() as f: + return json.load(f) + + +def before_runs(env, **kw): + global build_generator + + add_options(env, COMMON_OPTIONS) + if build_generator: + revision, timestamp, build = next(build_generator) + env.set_arg("android-install-apk", build) + env.set_arg("perfherder-prefix", revision) + env.set_arg("perfherder-timestamp", timestamp) diff --git a/testing/performance/hooks_perfstats.py b/testing/performance/hooks_perfstats.py new file mode 100644 index 0000000000..fe6d79c5af --- /dev/null +++ b/testing/performance/hooks_perfstats.py @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mozperftest.test.browsertime import add_options +from mozperftest.test.browsertime.runner import NodeException + +common_options = [("firefox.perfStats", "true")] + + +def on_exception(env, layer, exc): + if not isinstance(exc, NodeException): + raise exc + return True + + +def logcat_processor(): + pass + + +def before_runs(env, **kw): + add_options(env, common_options) diff --git a/testing/performance/hooks_recording.py b/testing/performance/hooks_recording.py new file mode 100644 index 0000000000..63cc65caa0 --- /dev/null +++ b/testing/performance/hooks_recording.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import os +import platform +from pathlib import Path + +from mozperftest.test.browsertime import add_option, add_options + +# Uncomment the manual_login line if you need to do a manual login. +# The extra browsertime arguments get overwritten below so they +# need to be set here. The value is the time you need to do a login. +options = [ + ("pageCompleteWaitTime", "10000"), + # ("browsertime.manual_login", 100000), +] + +next_site = None + +RECORDING_LIST = Path(Path(__file__).parent, "pageload_sites.json") + +SCM_1_LOGIN_SITES = ("facebook", "netflix") + + +def before_iterations(kw): + + global next_site + print("Setting up next site to record.") + + with RECORDING_LIST.open() as f: + site_list = json.load(f) + # currently can't record websites that require user interactions(Logins) + if kw.get("android"): + site_list = site_list["mobile"] + else: + site_list = site_list["desktop"] + + def __should_record(test): + # If a test page selection was provided, only select those + # tests and exclude all the others + specified_tests = kw["proxy_perftest_page"] + if specified_tests is not None: + if test.get("name") in specified_tests: + if test.get("login"): + print(f"WARNING: You selected a login test: {test.get('name')}") + return True + else: + return False + + # Only perform login recordings in automation or when + # RAPTOR_LOGINS is defined + record = False + if not test.get("login") or test.get("login-test"): + record = True + if not ( + "MOZ_AUTOMATION" in os.environ or "RAPTOR_LOGINS" in os.environ + ) and test.get("login-test"): + record = False + print( + f"Skipping login test `{test.get('name')}` " + f"because login info cannot be obtained." + ) + + # When pushing to Try, only attempt login recording using the + # taskcluster secrets that are associated with SCM level 1 as defined + # in `SCM_LVL_1_SITES`. + if test.get("login"): + if "MOZ_AUTOMATION" in os.environ.keys(): + if ( + os.environ.get("MOZ_SCM_LEVEL") == 1 + and test.get("name") not in SCM_1_LOGIN_SITES + ): + print( + f"Skipping login test `{test.get('name')}` " + f"Because SCM = `{os.environ.get('MOZ_SCM_LEVEL') }`" + f"and there is no secret available at this level" + ) + return False + return True + elif "RAPTOR_LOGINS" in os.environ: + # Leave it to the user to have properly set up a local json file with + # the login websites of interest + return True + + return record + + sites = [test_site for test_site in site_list if __should_record(test_site)] + + if not sites: + raise Exception("No tests were selected for recording!") + + def next_site(): + for site in sites: + yield site + + next_site = next_site() + + # Set the number of test-iterations to the number of builds + kw["test_iterations"] = len(sites) + return kw + + +def before_runs(env): + global next_site + print("Running before_runs") + add_options(env, options) + + if next_site: + test_site = next(next_site) + print("Next site: %s" % test_site) + + if env.get_arg("android"): + platform_name = "android" + app_name = env.get_arg("android-app-name").split(".")[-1] + else: + platform_name = platform.system().lower() + app_name = "firefox" + + name = [ + "mitm8", + platform_name, + "gve" if app_name == "geckoview_example" else app_name, + test_site["name"], + ] + + recording_file = "%s.zip" % "-".join(name) + + env.set_arg("proxy-mode", "record") + env.set_arg( + "proxy-file", + recording_file, + ) + + add_options(env, options, overwrite=True) + add_option(env, "browsertime.url", test_site.get("test_url")) + add_option(env, "browsertime.screenshot", "true") + add_option(env, "browsertime.testName", test_site.get("name")) + add_option(env, "browsertime.testType", test_site.get("type", "pageload")) + add_option( + env, "browsertime.login", "true" if test_site.get("login") else "false" + ) + + prefs = test_site.get("preferences", {}) + for pref, val in prefs.items(): + add_option(env, "firefox.preference", f"{pref}:{val}") + + second_url = test_site.get("secondary_url", None) + if second_url: + add_option(env, "browsertime.secondary_url", second_url) + + inject_deterministic = test_site.get("inject_deterministic", True) + env.set_arg("proxy-deterministic", inject_deterministic) + + dismiss_cookie_prompt = test_site.get("dismiss_cookie_prompt", []) + if dismiss_cookie_prompt: + parsed_cmds = [ + ":::".join([str(i) for i in item]) + for item in dismiss_cookie_prompt + if item + ] + add_option( + env, "browsertime.dismiss_cookie_prompt", ";;;".join(parsed_cmds) + ) + + cmds = test_site.get("test_cmds", []) + if cmds: + parsed_cmds = [":::".join([str(i) for i in item]) for item in cmds if item] + add_option(env, "browsertime.commands", ";;;".join(parsed_cmds)) + + print("Recording %s to file: %s" % (test_site.get("test_url"), recording_file)) diff --git a/testing/performance/mach-try-perf/perfdocs/android-chrome-try-perf.png b/testing/performance/mach-try-perf/perfdocs/android-chrome-try-perf.png Binary files differnew file mode 100644 index 0000000000..f9b1172ea5 --- /dev/null +++ b/testing/performance/mach-try-perf/perfdocs/android-chrome-try-perf.png diff --git a/testing/performance/mach-try-perf/perfdocs/config.yml b/testing/performance/mach-try-perf/perfdocs/config.yml new file mode 100644 index 0000000000..597373c854 --- /dev/null +++ b/testing/performance/mach-try-perf/perfdocs/config.yml @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +name: mach-try-perf +manifest: None +static-only: True +suites: {} diff --git a/testing/performance/mach-try-perf/perfdocs/index.rst b/testing/performance/mach-try-perf/perfdocs/index.rst new file mode 100644 index 0000000000..645c7e1dd5 --- /dev/null +++ b/testing/performance/mach-try-perf/perfdocs/index.rst @@ -0,0 +1,268 @@ +############# +Mach Try Perf +############# + +.. contents:: + :depth: 2 + :local: + +To make it easier for developers to find the tests they need to run we built a perf-specific try selector called `./mach try perf`. With this tool, you no longer need to remember the obfuscated platform and test names that you need to target for your tests. Instead, the new interface shows test categories along with a simplified name of the platform that they will run on. + +When you trigger a try run from the perf selector, two try runs will be created. One with your changes, and one without. In your console, after you trigger the try runs, you'll find a PerfCompare link that will bring you directly to a comparison of the two pushes when they have completed. + +The tool is built to be conservative about the number of tests to run, so if you are looking for something that is not listed, it's likely hidden behind a flag found in the `--help`. Here's a small sample of what you'll find there which highlights the most relevant flags:: + + $ ./mach try perf --help + + perf arguments: + --show-all Show all available tasks. + --android Show android test categories (disabled by default). + --chrome Show tests available for Chrome-based browsers (disabled by default). + --safari Show tests available for Safari (disabled by default). + --live-sites Run tasks with live sites (if possible). You can also use the `live-sites` variant. + --profile Run tasks with profiling (if possible). You can also use the `profiling` variant. + --single-run Run tasks without a comparison + --variants [ [ ...]] Select variants to display in the selector from: no-fission, bytecode-cached, live-sites, profiling, swr + --platforms [ [ ...]] + Select specific platforms to target. Android only available with --android. Available platforms: android-a51, android, + windows, linux, macosx, desktop + --apps [ [ ...]] Select specific applications to target from: firefox, chrome, chromium, geckoview, fenix, chrome-m, safari + + task configuration arguments: + --artifact Force artifact builds where possible. + + +Standard Usage +-------------- + +To use mach try perf simply call `./mach try perf`. This will open an interface for test selection like so: + + +.. image:: ./standard-try-perf.png + :alt: Mach try perf with default options + :scale: 75% + :align: center + + +Select the categories you'd like to run, hit enter, and wait for the tool to finish the pushes. **Note that it can take some time to do both pushes, and you might not see logging for some time.** + +Chrome and Android +------------------ + +Android and chrome tests are disabled by default as they are often unneeded and waste our limited resources. If you need either of these, you can add `--chrome` and/or `--android` to the command like so `./mach try perf --android --chrome`: + + +.. image:: ./android-chrome-try-perf.png + :alt: Mach try perf with android, and chrome options + :scale: 75% + :align: center + + +Variants +-------- + +If you are looking for any variants (e.g. no-fission, bytecode-cached, live-sites), use the `--variants` options like so `./mach try perf --variants live-sites`. This will select all possible categories that could have live-sites tests. + + +.. image:: ./variants-try-perf.png + :alt: Mach try perf with variants + :scale: 75% + :align: center + + +Note that it is expected that the offered categories have extra variants (such as bytecode-cached) as we are showing all possible combinations that can include live-sites. + +Platforms +--------- + +To target a particular platform you can use `--platforms` to only show categories with the given platforms. + +Categories +---------- + +In the future, this section will be populated dynamically. If you are wondering what the categories you selected will run, you can use `--no-push` to print out a list of tasks that will run like so:: + + $ ./mach try perf --no-push + + Artifact builds enabled, pass --no-artifact to disable + Gathering tasks for Benchmarks desktop category + Executing queries: 'browsertime 'benchmark, !android 'shippable !-32 !clang, !live, !profil, !chrom + estimates: Runs 66 tasks (54 selected, 12 dependencies) + estimates: Total task duration 8:45:58 + estimates: In the top 62% of durations + estimates: Should take about 1:04:58 (Finished around 2022-11-22 15:08) + Commit message: + Perf selections=Benchmarks desktop (queries='browsertime 'benchmark&!android 'shippable !-32 !clang&!live&!profil&!chrom) + Pushed via `mach try perf` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-macosx1015-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-windows10-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing" + ], + "use-artifact-builds": true, + "version": 1 + } + + +Adding a New Category +--------------------- + +It's very easy to add a new category if needed, and you can do so by modifying the `PerfParser categories attribute here <https://searchfox.org/mozilla-central/source/tools/tryselect/selectors/perf.py#179>`_. The following is an example of a complex category that gives a good idea of what you have available:: + + "Resource Usage": { + "query": { + "talos": ["'talos 'xperf | 'tp5"], + "raptor": ["'power 'osx"], + "awsy": ["'awsy"], + }, + "suites": ["talos", "raptor", "awsy"], + "platform-restrictions": ["desktop"], + "variant-restrictions": { + "raptor": [], + "talos": [], + }, + "app-restrictions": { + "raptor": ["firefox"], + "talos": ["firefox"], + }, + "tasks": [], + }, + +The following fields are available: + * **query**: Set the queries to use for each suite you need. + * **suites**: The suites that are needed for this category. + * **tasks**: A hard-coded list of tasks to select. + * **platform-restrictions**: The platforms that it can run on. + * **app-restrictions**: A list of apps that the category can run. + * **variant-restrictions**: A list of variants available for each suite. + +Note that setting the App/Variant-Restriction fields should be used to restrict the available apps and variants, not expand them as the suites, apps, and platforms combined already provide the largest coverage. The restrictions should be used when you know certain things definitely won't work, or will never be implemented for this category of tests. For instance, our `Resource Usage` tests only work on Firefox even though they may exist in Raptor which can run tests with Chrome. + +Comparators +----------- + +If the standard/default push-to-try comparison is not enough, you can build your own "comparator" that can setup the base, and new revisions. The default comparator `BasePerfComparator` runs the standard mach-try-perf comparison, and there also exists a custom comparator called `BenchmarkComparator` for running custom benchmark comparisons on try (using Github PR links). + +If you'd like to add a custom comparator, you can either create it in a separate file and pass it in the `--comparator`, or add it to the `tools/tryselect/selectors/perfselector/perfcomparators.py` and use the name of the class as the `--comparator` argument (e.g. `--comparator BenchmarkComparator`). You can pass additional arguments to it using the `--comparator-args` option that accepts arguments in the format `NAME=VALUE`. + +The custom comparator needs to be a subclass of `BasePerfComparator`, and optionally overrides its methods. See the comparators file for more information about the interface available. Here's the general interface for it (subject to change), note that the `@comparator` decorator is required when making a builtin comparator:: + + @comparator + class BasePerfComparator: + def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args): + """Initialize the standard/default settings for Comparators. + + :param vcs object: Used for updating the local repo. + :param compare_commit str: The base revision found for the local repo. + :param current_revision_ref str: The current revision of the local repo. + :param comparator_args list: List of comparator args in the format NAME=VALUE. + """ + + def setup_base_revision(self, extra_args): + """Setup the base try run/revision. + + The extra_args can be used to set additional + arguments for Raptor (not available for other harnesses). + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + + def teardown_base_revision(self): + """Teardown the setup for the base revision.""" + + def setup_new_revision(self, extra_args): + """Setup the new try run/revision. + + Note that the extra_args are reset between the base, and new revision runs. + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + + def teardown_new_revision(self): + """Teardown the new run/revision setup.""" + + def teardown(self): + """Teardown for failures. + + This method can be used for ensuring that the repo is cleaned up + when a failure is hit at any point in the process of doing the + new/base revision setups, or the pushes to try. + """ + +Frequently Asked Questions (FAQ) +-------------------------------- + +If you have any questions which aren't already answered below please reach out to us in the `perftest matrix channel <https://matrix.to/#/#perftest:mozilla.org>`_. + + * **How can I tell what a category or a set of selections will run?** + + At the moment, you need to run your command with an additional option to see what will be run: `./mach try perf --no-push`. See the `Categories`_ section for more information about this. In the future, we plan on having an dynamically updated list for the tasks in the `Categories`_ section of this document. + + * **What's the difference between `Pageload desktop`, and `Pageload desktop firefox`?** + + If you simply ran `./mach try perf` with no additional options, then there is no difference. If you start adding additional browsers to the try run with commands like `./mach try perf --chrome`, then `Pageload desktop` will select all tests available for ALL browsers available, and `Pageload desktop firefox` will only select Firefox tests. When `--chrome` is provided, you'll also see a `Pageload desktop chrome` option. + + * **Help! I can't find a test in any of the categories. What should I do?** + + Use the option `--show-all`. This will let you select tests from the `./mach try fuzzy --full` interface directly instead of the categories. You will always be able to find your tests this way. Please be careful with your task selections though as it's easy to run far too many tests in this way! + +Future Work +----------- + +The future work for this tool can be `found in this bug <https://bugzilla.mozilla.org/show_bug.cgi?id=1799178>`_. Feel free to file improvments, and bugs against it. diff --git a/testing/performance/mach-try-perf/perfdocs/standard-try-perf.png b/testing/performance/mach-try-perf/perfdocs/standard-try-perf.png Binary files differnew file mode 100644 index 0000000000..d1168665fb --- /dev/null +++ b/testing/performance/mach-try-perf/perfdocs/standard-try-perf.png diff --git a/testing/performance/mach-try-perf/perfdocs/variants-try-perf.png b/testing/performance/mach-try-perf/perfdocs/variants-try-perf.png Binary files differnew file mode 100644 index 0000000000..54050db962 --- /dev/null +++ b/testing/performance/mach-try-perf/perfdocs/variants-try-perf.png diff --git a/testing/performance/pageload_sites.json b/testing/performance/pageload_sites.json new file mode 100644 index 0000000000..07a28be0b8 --- /dev/null +++ b/testing/performance/pageload_sites.json @@ -0,0 +1,606 @@ +{ + "mobile": [ + { + "login": false, + "name": "allrecipes", + "test_url": "https://www.allrecipes.com/", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='onetrust-reject-all-handler']"] + ] + }, + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com" + }, + { + "login": false, + "name": "amazon-search", + "test_url": "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone" + }, + { + "login": false, + "name": "bing", + "test_url": "https://www.bing.com/", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='bnp_btn_reject']"] + ] + }, + { + "login": false, + "name": "bing-search-restaurants", + "test_url": "https://www.bing.com/search?q=restaurants+in+exton+pa+19341", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='bnp_btn_reject']"] + ] + }, + { + "login": false, + "name": "booking", + "test_url": "https://www.booking.com/", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[5]/div[3]/div[3]/div[1]/button" + ] + ] + }, + { + "login": false, + "name": "cnn", + "test_url": "https://cnn.com", + "secondary_url": "https://www.cnn.com/weather", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[14]/div[3]/div[3]/div[1]/button[2]" + ] + ] + }, + { + "login": false, + "name": "cnn-ampstories", + "test_url": "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[14]/div[3]/div[3]/div[1]/button[2]" + ] + ] + }, + { + "login": false, + "name": "dailymail", + "test_url": "https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html" + }, + { + "login": false, + "name": "ebay-kleinanzeigen", + "test_url": "https://m.ebay-kleinanzeigen.de" + }, + { + "login": false, + "name": "ebay-kleinanzeigen-search", + "test_url": "https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331" + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://m.facebook.com" + }, + { + "login": false, + "name": "facebook-cristiano", + "test_url": "https://m.facebook.com/Cristiano" + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com" + }, + { + "login": false, + "name": "google-maps", + "test_url": "https://www.google.com/maps?force=pwa" + }, + { + "login": true, + "name": "google-search-restaurants", + "test_url": "https://www.google.com/search?q=restaurants+near+me" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://m.imdb.com/" + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com" + }, + { + "login": false, + "name": "sina", + "test_url": "https://www.sina.com.cn/" + }, + { + "login": false, + "name": "microsoft-support", + "test_url": "https://support.microsoft.com/en-us" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com" + }, + { + "login": false, + "name": "stackoverflow", + "test_url": "https://stackoverflow.com/" + }, + { + "login": false, + "name": "web-de", + "test_url": "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='edit-purpose-settings']"], + ["click.byXpathAndWait", "//*[@id='save-purpose-settings']"] + ] + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.m.wikipedia.org/wiki/Main_Page" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://m.youtube.com", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[2]/ytm-consent-bump-v2-renderer/div/div[2]/div[3]/c3-material-button[2]/button/div/div" + ] + ] + }, + { + "login": false, + "name": "youtube-watch", + "test_url": "https://www.youtube.com/watch?v=COU5T-Wafa4", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[2]/ytm-consent-bump-v2-renderer/div/div[2]/div[3]/c3-material-button[2]/button/div/div" + ] + ] + } + ], + "desktop": [ + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1", + "secondary_url": "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3" + }, + { + "login": false, + "name": "bing-search", + "test_url": "https://www.bing.com/search?q=barack+obama", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div[1]/div/div[2]/div[2]/button[2]/a" + ] + ] + }, + { + "login": false, + "name": "buzzfeed", + "test_url": "https://www.buzzfeed.com/", + "secondary_url": "https://www.buzzfeed.com/quizzes", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[2]/div/button[2]" + ] + ] + }, + { + "login": false, + "name": "cnn", + "test_url": "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html", + "secondary_url": "https://www.cnn.com/weather", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[13]/div[3]/div[3]/div[1]/button[2]" + ] + ] + }, + { + "login": false, + "name": "ebay", + "test_url": "https://www.ebay.com/", + "secondary_url": "https://www.ebay.com/deals", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "//*[@id='gdpr-banner-decline']"] + ] + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[10]/div[2]/div/div/div[3]/button" + ] + ] + }, + { + "login": false, + "name": "expedia", + "test_url": "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2", + "secondary_url": "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://www.facebook.com", + "secondary_url": "https://www.facebook.com/marketplace/?ref=bookmark" + }, + { + "login": true, + "login-test": true, + "name": "facebook-login", + "test_url": "https://www.facebook.com", + "type": "interactive", + "test_cmds": [ + ["setup_login", "https://www.facebook.com"], + ["wait.byTime", 1000], + ["login", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "fandom", + "test_url": "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[2]"] + ] + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com/search?hl=en&q=barack+obama&cad=h", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[3]/div[3]/span/div/div/div/div[3]/button[1]/div" + ], + [ + "click.byXpathAndWait", + "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]" + ] + ] + }, + { + "login": false, + "name": "google-docs", + "test_url": "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": true, + "name": "google-mail", + "test_url": "https://mail.google.com/", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "/html/body/div/div/span[2]/a[2]"] + ] + }, + { + "login": false, + "name": "google-slides", + "test_url": "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2", + "secondary_url": "https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl" + }, + { + "login": false, + "name": "imgur", + "test_url": "https://imgur.com/gallery/m5tYJL6", + "inject_deterministic": false, + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[2]/div/button[2]" + ] + ], + "secondary_url": "https://imgur.com/gallery/rCXZUil", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + } + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com/" + }, + { + "login": true, + "name": "linkedin", + "test_url": "https://www.linkedin.com/in/thommy-harris-hk-385723106/", + "secondary_url": "https://www.linkedin.com/company/github?trk=affiliated-pages" + }, + { + "login": false, + "name": "microsoft", + "test_url": "https://www.microsoft.com/en-us/", + "secondary_url": "https://support.microsoft.com/en-us", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[3]/div[1]/div/div/div[2]/div/div/div/div[2]/button[2]" + ] + ] + }, + { + "login": true, + "name": "netflix", + "test_url": "https://www.netflix.com/title/80117263", + "secondary_url": "https://www.netflix.com/title/699257" + }, + { + "login": false, + "name": "nytimes", + "test_url": "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html", + "secondary_url": "https://www.nytimes.com/section/opinion/columnists", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div/div[2]/main/div[3]/div[2]/div/div[2]/button[2]" + ] + ] + }, + { + "login": true, + "name": "office", + "test_url": "https://www.office.com/launch/powerpoint/", + "secondary_url": "https://www.office.com/" + }, + { + "login": true, + "name": "outlook", + "test_url": "https://outlook.live.com/mail/inbox" + }, + { + "login": true, + "name": "paypal", + "test_url": "https://www.paypal.com/myaccount/summary/" + }, + { + "login": true, + "name": "pinterest", + "test_url": "https://pinterest.com/", + "secondary_url": "https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/", + "secondary_url": "https://www.reddit.com/r/technology/", + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[3]/div[1]/section/div/section/section/form[1]/button" + ] + ] + }, + { + "login": true, + "name": "tumblr", + "test_url": "https://www.tumblr.com/dashboard", + "secondary_url": "https://www.tumblr.com/tagged/funny+cats?sort=top" + }, + { + "login": false, + "name": "twitch", + "test_url": "https://www.twitch.tv/videos/894226211", + "secondary_url": "https://www.twitch.tv/gmashley", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "dismiss_cookie_prompt": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[1]/div/div/div/div[3]/div/button" + ], + [ + "click.byXpathAndWait", + "/html/body/div[3]/div/div/div/div/div/div[1]/div[2]/div[2]/button" + ] + ] + }, + { + "login": true, + "name": "twitter", + "test_url": "https://twitter.com/BarackObama" + }, + { + "login": false, + "name": "wikia", + "test_url": "https://marvel.fandom.com/wiki/Black_Panther", + "secondary_url": "https://marvel.fandom.com/wiki/Celestials", + "dismiss_cookie_prompt": [ + ["click.byXpathAndWait", "/html/body/div[6]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[6]/div/div/div[2]/div[2]"] + ] + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.wikipedia.org/wiki/Barack_Obama", + "secondary_url": "https://en.wikipedia.org/wiki/Joe_Biden" + }, + { + "login": true, + "name": "yahoo-mail", + "test_url": "https://mail.yahoo.com/" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://www.youtube.com", + "secondary_url": "https://www.youtube.com/watch?v=JrdEMERq8MA", + "dismiss_cookie_prompt ": [ + [ + "click.byXpathAndWait", + "/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div[4]/div[2]/div[6]/div[1]/ytd-button-renderer[1]/a" + ] + ] + }, + { + "login": false, + "name": "cnn-nav", + "test_url": "https://www.cnn.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "landing"], + ["navigate", "https://www.cnn.com"], + ["wait.byTime", 4000], + ["measure.stop", ""], + ["measure.start", "world"], + [ + "click.byXpathAndWait", + "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a" + ], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-ama", + "test_url": "https://www.reddit.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg-ama"], + [ + "navigate", + "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/" + ], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "members"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div" + ], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-post", + "test_url": "https://www.reddit.com/user/thisisbillgates/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg"], + ["navigate", "https://www.reddit.com/user/thisisbillgates/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "posts"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "comments"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "hot"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "top"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "facebook-nav", + "test_url": "https://www.facebook.com/", + "type": "interactive", + "test_cmds": [ + ["navigate", "https://www.facebook.com/login"], + ["wait.byTime", 30000], + ["measure.start", "landing"], + ["navigate", "https://www.facebook.com/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "groups"], + ["navigate", "https://www.facebook.com/groups/discover/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "friends"], + ["navigate", "https://www.facebook.com/friends/"], + ["wait.byTime", 5000], + ["measure.stop", ""] + ] + } + ] +} diff --git a/testing/performance/perftest.ini b/testing/performance/perftest.ini new file mode 100644 index 0000000000..ae27492ca7 --- /dev/null +++ b/testing/performance/perftest.ini @@ -0,0 +1,13 @@ +[perftest_android_main.js] +[perftest_android_startup.js] +[perftest_android_view.js] +[perftest_bbc_link.js] +[perftest_facebook.js] +[perftest_jsconf_cold.js] +[perftest_jsconf_warm.js] +[perftest_pageload.js] +[perftest_perfstats.js] +[perftest_politico_link.js] +[perftest_youtube_link.js] +[perftest_WPT_firefox_init_file.js] +[perftest_WPT_chrome_init_file.js] diff --git a/testing/performance/perftest_WPT_chrome_init_file.js b/testing/performance/perftest_WPT_chrome_init_file.js new file mode 100644 index 0000000000..f74e714242 --- /dev/null +++ b/testing/performance/perftest_WPT_chrome_init_file.js @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; +function setUp() {} + +function tearDown() {} + +function test() {} + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Testing Team", + name: "webpagetest-chrome", + description: + "Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites", + longDescription: `This mozperftest gets webpagetest to run pageload tests on Chrome against the 50 most popular + websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, + visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut, + TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, + loadEventStart, loadEventEnd`, + options: { + test_parameters: { + location: "ec2-us-east-1", + browser: "Chrome", + connection: "Cable", + timeout_limit: 21600, + wait_between_requests: 5, + statistics: ["average", "median", "standardDeviation"], + label: "", + runs: 3, + fvonly: 0, + private: 0, + web10: 0, + script: "", + block: "", + video: 1, + tcpdump: 0, + noimages: 0, + keepua: 1, + uastring: "", + htmlbody: 0, + custom: "", + ignoreSSL: 0, + appendua: "", + injectScript: "", + disableAVIF: 0, + disableWEBP: 0, + disableJXL: 0, + }, + test_list: [ + "google.com", + "youtube.com", + "facebook.com", + "qq.com", + "baidu.com", + "sohu.com", + "360.cn", + "jd.com", + "amazon.com", + "yahoo.com", + "zoom.us", + "sina.com.cn", + "live.com", + "reddit.com", + "netflix.com", + "microsoft.com", + "instagram.com", + "google.com.hk", + "csdn.net", + "bing.com", + "vk.com", + "yahoo.co.jp", + "twitter.com", + "naver.com", + "canva.com", + "ebay.com", + "force.com", + "amazon.in", + "adobe.com", + "aliexpress.com", + "linkedin.com", + "tianya.cn", + "yy.com", + "huanqiu.com", + "amazon.co.jp", + "okezone.com", + ], + browser_metrics: [ + "firstContentfulPaint", + "visualComplete90", + "firstPaint", + "visualComplete99", + "visualComplete", + "SpeedIndex", + "bytesIn", + "bytesOut", + "TTFB", + "fullyLoadedCPUms", + "fullyLoadedCPUpct", + "domElements", + "domContentLoadedEventStart", + "domContentLoadedEventEnd", + "loadEventStart", + "loadEventEnd", + ], + }, +}; diff --git a/testing/performance/perftest_WPT_firefox_init_file.js b/testing/performance/perftest_WPT_firefox_init_file.js new file mode 100644 index 0000000000..e35d62fb34 --- /dev/null +++ b/testing/performance/perftest_WPT_firefox_init_file.js @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; +function setUp() {} + +function tearDown() {} + +function test() {} + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Testing Team", + name: "webpagetest-firefox", + description: + "Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites", + longDescription: `This mozperftest gets webpagetest to run pageload tests on Firefox against the 50 most popular + websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, + timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn, bytesOut, + TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, + loadEventStart, loadEventEnd`, + options: { + test_parameters: { + location: "ec2-us-east-1", + browser: "Firefox", + connection: "Cable", + timeout_limit: 21600, + wait_between_requests: 5, + statistics: ["average", "median", "standardDeviation"], + label: "", + runs: 3, + fvonly: 0, + private: 0, + web10: 0, + script: "", + block: "", + video: 1, + tcpdump: 0, + noimages: 0, + keepua: 1, + uastring: "", + htmlbody: 0, + custom: "", + ignoreSSL: 0, + appendua: "", + injectScript: "", + disableAVIF: 0, + disableWEBP: 0, + disableJXL: 0, + }, + test_list: [ + "google.com", + "youtube.com", + "facebook.com", + "qq.com", + "baidu.com", + "sohu.com", + "360.cn", + "jd.com", + "amazon.com", + "yahoo.com", + "zoom.us", + "sina.com.cn", + "live.com", + "reddit.com", + "netflix.com", + "microsoft.com", + "instagram.com", + "google.com.hk", + "csdn.net", + "bing.com", + "vk.com", + "yahoo.co.jp", + "twitter.com", + "naver.com", + "canva.com", + "ebay.com", + "force.com", + "amazon.in", + "adobe.com", + "aliexpress.com", + "linkedin.com", + "tianya.cn", + "yy.com", + "huanqiu.com", + "amazon.co.jp", + "okezone.com", + ], + browser_metrics: [ + "firstContentfulPaint", + "timeToContentfulPaint", + "visualComplete90", + "firstPaint", + "visualComplete99", + "visualComplete", + "SpeedIndex", + "bytesIn", + "bytesOut", + "TTFB", + "fullyLoadedCPUms", + "fullyLoadedCPUpct", + "domElements", + "domContentLoadedEventStart", + "domContentLoadedEventEnd", + "loadEventStart", + "loadEventEnd", + ], + }, +}; diff --git a/testing/performance/perftest_android_main.js b/testing/performance/perftest_android_main.js new file mode 100644 index 0000000000..2a10cbcb86 --- /dev/null +++ b/testing/performance/perftest_android_main.js @@ -0,0 +1,35 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +async function test(context, commands) { + "use strict"; + // Nothing to do -- the timing is captured via logcat + return true; +} + +module.exports = { + test, + owner: "Performance Team", + name: "main", + description: + "Measures the time from process start until the Fenix main activity (HomeActivity) reports Fully Drawn", + longDescription: ` + This test launches Fenix to its main activity (HomeActivity). + The application logs "Fully Drawn" when the activity is drawn. + Using the android log transformer we measure the time from process start to this event. + `, + usage: ` + ./mach perftest testing/performance/perftest_android_main.js --android --flavor mobile-browser \ + --hooks testing/performance/hooks_home_activity.py --perfherder --android-app-name org.mozilla.fenix \ + --android-activity .App --android-install-apk ~/Downloads/fenix.apk --android-clear-logcat \ + --android-capture-logcat logcat \ + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\.fenix.*\.App.*" \ + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\.fenix.*" \ + --androidlog-subtest-name "MAIN" --androidlog + `, + supportedBrowsers: ["Fenix nightly"], + supportedPlatforms: ["Android"], +}; diff --git a/testing/performance/perftest_android_startup.js b/testing/performance/perftest_android_startup.js new file mode 100644 index 0000000000..4dfcd32af0 --- /dev/null +++ b/testing/performance/perftest_android_startup.js @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; + +async function test(context, commands) {} + +module.exports = { + test, + owner: "Performance Team", + name: "android-startup", + description: "Measures android startup times", + longDescription: ` + This test consists of 2 main tests, cold main first frame(cmff) and cold view nav start(cvns). + cold main first frame is the measurement from when you click the app icon & get duration to first frame from 'am start -W'. + cold view nav start is the measurement from when you send a VIEW intent & get duration from logcat: START proc to PageStart. + `, + usage: ` + ./mach perftest --flavor mobile-browser --AndroidStartUp testing/performance/perftest_android_startup.js + --browsertime-cycles=0 --AndroidStartUp-test-name=cold_view_nav_start --perfherder --hooks + testing/performance/hooks_android_startup.py --AndroidStartUp-product=fenix + --AndroidStartUp-release-channel=nightly + `, + options: { + test_parameters: { + single_date: null, // Dates in YYYY.MM.DD format + date_range: [], // 2 Dates in YYYY.MM.DD format the first and last date(inclusive) + startup_cache: true, + test_cycles: 50, + }, + }, +}; diff --git a/testing/performance/perftest_android_view.js b/testing/performance/perftest_android_view.js new file mode 100644 index 0000000000..50e4dce85e --- /dev/null +++ b/testing/performance/perftest_android_view.js @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; + +async function test(context, commands) { + await commands.measure.start(); + await commands.measure.browser.wait(commands.measure.pageCompleteCheck); + await commands.measure.stop(); + + const browserScripts = commands.measure.result[0].browserScripts; + + const processLaunchToNavStart = + browserScripts.pageinfo.navigationStartTime - + browserScripts.browser.processStartTime; + + browserScripts.pageinfo.processLaunchToNavStart = processLaunchToNavStart; + console.log("processLaunchToNavStart: " + processLaunchToNavStart); + + return true; +} + +module.exports = { + test, + owner: "Performance Team", + name: "VIEW", + description: "Measures cold process view time", + longDescription: ` + This test launches the appropriate android app, simulating a opening a link through VIEW intent + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_android_view.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_android_view.py \ + --android-app-name org.mozilla.fenix \ + --perfherder-metrics processLaunchToNavStart + `, + supportedBrowsers: ["Fenix nightly", "Geckoview_example", "Fennec"], + supportedPlatforms: ["Android"], +}; diff --git a/testing/performance/perftest_bbc_link.js b/testing/performance/perftest_bbc_link.js new file mode 100644 index 0000000000..3021c44c83 --- /dev/null +++ b/testing/performance/perftest_bbc_link.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.bbc.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("block-link__overlay-link"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "BBC Link", + component: "pageload", + description: "Measures time to load BBC homepage", +}; diff --git a/testing/performance/perftest_facebook.js b/testing/performance/perftest_facebook.js new file mode 100644 index 0000000000..bd607d6be4 --- /dev/null +++ b/testing/performance/perftest_facebook.js @@ -0,0 +1,51 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + await commands.navigate("https://www.example.com"); + await commands.wait.byTime(15000); + + // Fill in: + let username = "some_user@mail.com"; + let passworld = "topsecretpassword"; + + // We start by navigating to the login page. + await commands.navigate("https://www.facebook.com"); + + // When we fill in a input field/click on a link we wanna + // try/catch that if the HTML on the page changes in the feature + // sitespeed.io will automatically log the error in a user friendly + // way, and the error will be re-thrown so you can act on it. + await commands.wait.byTime(5000); + + // Add text into an input field, finding the field by id + await commands.addText.bySelector(username, "input[name=email]"); + await commands.wait.byTime(2000); + await commands.addText.bySelector(passworld, "input[name=pass]"); + await commands.wait.byTime(2000); + + // Start the measurement before we click on the + // submit button. Sitespeed.io will start the video recording + // and prepare everything. + // Find the sumbit button and click it and then wait + // for the pageCompleteCheck to finish + await commands.measure.start("pageload"); + + // There are two variants of the facebook login page: + try { + await commands.click.bySelectorAndWait("button[name=login]"); + } catch (e) { + await commands.click.bySelectorAndWait("input[type=submit]"); + } + + // Stop and collect the measurement before the next page we want to measure + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "Facebook", + description: "Measures time to log in to Facebook", +}; diff --git a/testing/performance/perftest_jsconf_cold.js b/testing/performance/perftest_jsconf_cold.js new file mode 100644 index 0000000000..c6b68a3e55 --- /dev/null +++ b/testing/performance/perftest_jsconf_cold.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://2019.jsconf.eu/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byLinkTextAndWait("Artists"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "JSConf (cold)", + description: "Measures time to load JSConf page (cold)", +}; diff --git a/testing/performance/perftest_jsconf_warm.js b/testing/performance/perftest_jsconf_warm.js new file mode 100644 index 0000000000..be58553ab6 --- /dev/null +++ b/testing/performance/perftest_jsconf_warm.js @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://2019.jsconf.eu/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Click on the link and wait a few seconds + await commands.click.byLinkTextAndWait("Artists"); + await commands.wait.byTime(3000); + + // Back to about + await commands.click.byLinkTextAndWait("About"); + await commands.wait.byTime(3000); + + // Start the measurement + await commands.measure.start("pageload"); + await commands.click.byLinkTextAndWait("Artists"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "JSConf (warm)", + description: "Measures time to load JSConf page (warm)", +}; diff --git a/testing/performance/perftest_pageload.js b/testing/performance/perftest_pageload.js new file mode 100644 index 0000000000..36a0ca8b68 --- /dev/null +++ b/testing/performance/perftest_pageload.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function setUp(context) { + context.log.info("setUp example!"); +} + +async function test(context, commands) { + let url = context.options.browsertime.url; + await commands.navigate("https://www.mozilla.org/en-US/"); + await commands.wait.byTime(100); + await commands.navigate("about:blank"); + await commands.wait.byTime(50); + return commands.measure.start(url); +} + +async function tearDown(context) { + context.log.info("tearDown example!"); +} + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Team", + name: "pageload", + description: "Measures time to load mozilla page", +}; diff --git a/testing/performance/perftest_perfstats.js b/testing/performance/perftest_perfstats.js new file mode 100644 index 0000000000..ad3e49eb79 --- /dev/null +++ b/testing/performance/perftest_perfstats.js @@ -0,0 +1,52 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const urls = [ + "https://accounts.google.com", + "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other", + "https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git", + "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2", + "https://m.imdb.com/title/tt0083943/", +]; + +const idle_delay = 10000; + +async function test(context, commands) { + await commands.wait.byTime(idle_delay); + + await commands.measure.start(); + + // Cold + for (const url of urls) { + await commands.navigate(url); + await commands.wait.byTime(idle_delay); + } + + await commands.navigate("about:blank"); + await commands.wait.byTime(idle_delay); + + // Warm + for (const url of urls) { + await commands.navigate(url); + await commands.wait.byTime(idle_delay); + } + + return commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "perfstats", + description: "Collect perfstats for the given site", + longDescription: ` + This test launches browsertime with the perfStats option (will collect low-overhead timings, see Bug 1553254). + The test currently runs a short user journey. A selection of popular sites are visited, first as cold pageloads, and then as warm. + `, + usage: ` + ./mach perftest --hook testing/performance/hooks_perfstats.py \ + testing/performance/perftest_perfstats.js --browsertime-iterations 10 \ + --perfherder-metrics name:HttpChannelCompletion_Cache name:HttpChannelCompletion_Network + `, +}; diff --git a/testing/performance/perftest_politico_link.js b/testing/performance/perftest_politico_link.js new file mode 100644 index 0000000000..019595afb8 --- /dev/null +++ b/testing/performance/perftest_politico_link.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.politico.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("js-tealium-tracking"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "Politico Link", + description: "Measures time to load Politico homepage", +}; diff --git a/testing/performance/perftest_record.js b/testing/performance/perftest_record.js new file mode 100644 index 0000000000..a0bfa6d134 --- /dev/null +++ b/testing/performance/perftest_record.js @@ -0,0 +1,562 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* eslint-env node */ +"use strict"; + +const fs = require("fs"); +const http = require("http"); + +const URL = "/secrets/v1/secret/project/perftest/gecko/level-"; +const SECRET = "/perftest-login"; +const DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"; + +const SCM_1_LOGIN_SITES = ["facebook", "netflix"]; + +/** + * This function obtains the perftest secret from Taskcluster. + * + * It will NOT work locally. Please see the get_logins function, you + * will need to define a JSON file and set the RAPTOR_LOGINS + * env variable to its path. + */ +async function get_tc_secrets(context) { + const MOZ_AUTOMATION = process.env.MOZ_AUTOMATION; + if (!MOZ_AUTOMATION) { + throw Error( + "Not running in CI. Set RAPTOR_LOGINS to a JSON file containing the logins." + ); + } + + let TASKCLUSTER_PROXY_URL = process.env.TASKCLUSTER_PROXY_URL + ? process.env.TASKCLUSTER_PROXY_URL + : DEFAULT_SERVER; + + let MOZ_SCM_LEVEL = process.env.MOZ_SCM_LEVEL ? process.env.MOZ_SCM_LEVEL : 1; + + const url = TASKCLUSTER_PROXY_URL + URL + MOZ_SCM_LEVEL + SECRET; + + const data = await new Promise((resolve, reject) => { + context.log.info("Obtaining secrets for login..."); + + http.get( + url, + { + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }, + res => { + let data = ""; + context.log.info(`Secret status code: ${res.statusCode}`); + + res.on("data", d => { + data += d.toString(); + }); + + res.on("end", () => { + resolve(data); + }); + + res.on("error", error => { + context.log.error(error); + reject(error); + }); + } + ); + }); + + return JSON.parse(data); +} + +/** + * This function gets the login information required. + * + * It starts by looking for a local file whose path is defined + * within RAPTOR_LOGINS. If we don't find this file, then we'll + * attempt to get the login information from our Taskcluster secret. + * If MOZ_AUTOMATION is undefined, then the test will fail, Taskcluster + * secrets can only be obtained in CI. + */ +async function get_logins(context) { + let logins; + + let RAPTOR_LOGINS = process.env.RAPTOR_LOGINS; + if (RAPTOR_LOGINS) { + // Get logins from a local file + if (!RAPTOR_LOGINS.endsWith(".json")) { + throw Error( + `File given for logins does not end in '.json': ${RAPTOR_LOGINS}` + ); + } + + let logins_file = null; + try { + logins_file = await fs.readFileSync(RAPTOR_LOGINS, "utf8"); + } catch (err) { + throw Error(`Failed to read the file ${RAPTOR_LOGINS}: ${err}`); + } + + logins = await JSON.parse(logins_file); + } else { + // Get logins from a perftest Taskcluster secret + logins = await get_tc_secrets(context); + } + + return logins; +} + +/** + * This function returns the type of login to do. + * + * This function returns "single-form" when we find a single form. If we only + * find a single input field, we assume that there is one page per input + * and return "multi-page". Otherwise, we return null. + */ +async function get_login_type(context, commands) { + /* + Determine if there's a password field visible with this + query selector. Some sites use `tabIndex` to hide the password + field behind other elements. In this case, we are searching + for any password-type field that has a tabIndex of 0 or undefined and + is not hidden. + */ + let input_length = await commands.js.run(` + return document.querySelectorAll( + "input[type=password][tabIndex='0']:not([type=hidden])," + + "input[type=password]:not([tabIndex]):not([type=hidden])" + ).length; + `); + if (input_length == 0) { + context.log.info("Found a multi-page login"); + return multi_page_login; + } else if (input_length == 1) { + context.log.info("Found a single-page login"); + return single_page_login; + } + + if ( + (await commands.js.run( + `return document.querySelectorAll("form").length;` + )) >= 1 + ) { + context.log.info("Found a single-form login"); + return single_form_login; + } + + return null; +} + +/** + * This function sets up the login for a single form. + * + * The username field is defined as the field which immediately precedes + * the password field. We have to do this in two steps because we need + * to make sure that the event we emit from the change has the `isTrusted` + * field set to `true`. Otherwise, some websites will ignore the input and + * the form submission. + */ +async function single_page_login(login_info, context, commands, prefix = "") { + // Get the first input field in the form that is not hidden and add the + // username. Assumes that email/username is always the first input field. + await commands.addText.bySelector( + login_info.username, + `${prefix}input:not([type=hidden]):not([type=password])` + ); + + // Get the password field and ensure it's not hidden. + await commands.addText.bySelector( + login_info.password, + `${prefix}input[type=password]:not([type=hidden])` + ); + + return undefined; +} + +/** + * See single_page_login. + */ +async function single_form_login(login_info, context, commands) { + return single_page_login(login_info, context, commands, "form "); +} + +/** + * Login to a website that uses multiple pages for the login. + * + * WARNING: Assumes that the first page is for the username. + */ +async function multi_page_login(login_info, context, commands) { + const driver = context.selenium.driver; + const webdriver = context.selenium.webdriver; + + const username_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await username_field.sendKeys(login_info.username); + await username_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + + let password_field; + try { + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } catch (err) { + if (err.toString().includes("NoSuchElementError")) { + // Sometimes we're suspicious (i.e. they think we're a bot/bad-actor) + let name_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await name_field.sendKeys(login_info.suspicious_answer); + await name_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + + // Try getting the password field again + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } else { + throw err; + } + } + + await password_field.sendKeys(login_info.password); + + return async function () { + password_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + }; +} + +/** + * This function sets up the login. + * + * This is done by first the login type, and then performing the + * actual login setup. The return is a possible button to click + * to perform the login. + */ +async function setup_login(login_info, context, commands) { + let login_func = await get_login_type(context, commands); + if (!login_func) { + throw Error("Could not determine the type of login page."); + } + + try { + return await login_func(login_info, context, commands); + } catch (err) { + throw Error(`Could not setup login information: ${err}`); + } +} + +/** + * This function performs the login. + * + * It does this by either clicking on a button with a type + * of "sumbit", or running a final_button function that was + * obtained from the setup_login function. Some pages also ask + * questions about setting up 2FA or other information. Generally, + * these contain the "skip" text. + */ +async function login(context, commands, final_button) { + try { + if (!final_button) { + // The mouse double click emits an event with `evt.isTrusted=true` + await commands.mouse.doubleClick.bySelector("button[type=submit]"); + await commands.wait.byTime(10000); + } else { + // In some cases, it's preferable to be given a function for the final button + await final_button(); + } + + // Some pages ask to setup 2FA, skip this based on the text + const XPATHS = [ + "//a[contains(text(), 'skip')]", + "//button[contains(text(), 'skip')]", + "//input[contains(text(), 'skip')]", + "//div[contains(text(), 'skip')]", + ]; + + for (let xpath of XPATHS) { + try { + await commands.mouse.doubleClick.byXpath(xpath); + } catch (err) { + if (err.toString().includes("not double click")) { + context.log.info(`Can't find a button with the text: ${xpath}`); + } else { + throw err; + } + } + } + } catch (err) { + throw Error( + `Could not login to website as we could not find the submit button/input: ${err}` + ); + } +} + +/** + * Grab the base URL from the browsertime url. + * + * This is a necessary step for getting the login values from the Taskcluster + * secrets, which are hashed by the base URL. + * + * The first entry is the protocal, third is the top-level domain (or host) + */ +function get_base_URL(fullUrl) { + let pathAsArray = fullUrl.split("/"); + return pathAsArray[0] + "//" + pathAsArray[2]; +} + +/** + * This function attempts the login-login sequence for a live pageload recording + */ +async function perform_live_login(context, commands) { + let testUrl = context.options.browsertime.url; + + let logins = await get_logins(context); + const baseUrl = get_base_URL(testUrl); + + await commands.navigate("about:blank"); + + let login_info = logins.secret[baseUrl]; + try { + await commands.navigate(login_info.login_url); + } catch (err) { + context.log.info("Unable to acquire login information"); + throw err; + } + await commands.wait.byTime(5000); + + let final_button = await setup_login(login_info, context, commands); + await login(context, commands, final_button); +} + +async function dismissCookiePrompt(input_cmds, context, commands) { + context.log.info("Searching for cookie prompt elements..."); + let cmds = input_cmds.split(";;;"); + for (let cmdstr of cmds) { + let [cmd, ...args] = cmdstr.split(":::"); + context.log.info(cmd, args); + let result = await commands.js.run( + `return document.evaluate("` + + args + + `", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;` + ); + if (result) { + context.log.info("Element found, clicking on it."); + await run_command(cmdstr, context, commands); + } else { + context.log.info( + "Element not found! The cookie prompt may have not appeared, please check the screenshots." + ); + } + } +} + +async function pageload_test(context, commands) { + let testUrl = context.options.browsertime.url; + let secondaryUrl = context.options.browsertime.secondary_url; + let testName = context.options.browsertime.testName; + let dismissPrompt = context.options.browsertime.dismiss_cookie_prompt || ""; + context.log.info(context.options.browsertime); + + // Wait for browser to settle + await commands.wait.byTime(1000); + + // If the user has RAPTOR_LOGINS configured correctly, a local login pageload + // test can be attempted. Otherwise if attempting it in CI, only sites with the + // associated MOZ_SCM_LEVEL will be attempted (e.g. Try = 1, autoland = 3) + if (context.options.browsertime.login) { + if (context.options.browsertime.manual_login) { + // Perform a manual login using the value given in manual_login + // as the amount of time to wait + await commands.navigate(testUrl); + context.log.info( + `Waiting ${context.options.browsertime.manual_login}ms for login...` + ); + await commands.wait.byTime(context.options.browsertime.manual_login); + } else if ( + process.env.RAPTOR_LOGINS || + process.env.MOZ_SCM_LEVEL == 3 || + SCM_1_LOGIN_SITES.includes(testName) + ) { + try { + await perform_live_login(context, commands); + } catch (err) { + context.log.info( + "Unable to login. Acquiring a recording without logging in" + ); + context.log.info("Error:" + err); + } + } else { + context.log.info(` + NOTE: This is a login test but a manual login was not requested, and + we cannot find any logins defined in RAPTOR_LOGINS. + `); + } + } + + await commands.measure.start(testUrl); + await commands.wait.byTime(40000); + if (dismissPrompt) { + await dismissCookiePrompt(dismissPrompt, context, commands); + } + commands.screenshot.take("test_url_" + testName); + + if (secondaryUrl !== null) { + // Wait for browser to settle + await commands.wait.byTime(1000); + + await commands.measure.start(secondaryUrl); + commands.screenshot.take("secondary_url_" + testName); + } + + // Wait for browser to settle + await commands.wait.byTime(1000); +} + +/** + * Converts a string such as `measure.start` into the + * actual function that is found in the `commands` module. + * + * XX: Find a way to share this function between + * perftest_record.js and browsertime_interactive.js + */ +async function get_command_function(cmd, commands) { + if (cmd == "") { + throw new Error("A blank command was given."); + } else if (cmd.endsWith(".")) { + throw new Error( + "An extra `.` was found at the end of this command: " + cmd + ); + } + + // `func` will hold the actual method that needs to be called, + // and the `parent_mod` is the context required to run the `func` + // method. Without that context, `this` becomes undefined in the browsertime + // classes. + let func = null; + let parent_mod = null; + for (let func_part of cmd.split(".")) { + if (func_part == "") { + throw new Error( + "An empty function part was found in the command: " + cmd + ); + } + + if (func === null) { + parent_mod = commands; + func = commands[func_part]; + } else if (func !== undefined) { + parent_mod = func; + func = func[func_part]; + } else { + break; + } + } + + if (func == undefined) { + throw new Error( + "The given command could not be found as a function: " + cmd + ); + } + + return [func, parent_mod]; +} + +/** + * Performs an interactive test. + * + * These tests are interactive as the entire test is defined + * through a set of browsertime commands. This allows users + * to build arbitrary tests. Furthermore, interactive tests + * provide the ability to login to websites. + */ +async function interactive_test(input_cmds, context, commands) { + let cmds = input_cmds.split(";;;"); + + let logins; + if (context.options.browsertime.login) { + logins = await get_logins(context); + } + + await commands.navigate("about:blank"); + + let user_setup = false; + let final_button = null; + for (let cmdstr of cmds) { + let [cmd, ...args] = cmdstr.split(":::"); + + if (cmd == "setup_login") { + if (!logins) { + throw Error( + "This test is not specified as a `login` test so no login information is available." + ); + } + if (args.length < 1 || args[0] == "") { + throw Error( + `No URL given, can't tell where to setup the login. We only accept: ${logins.keys()}` + ); + } + /* Structure for logins is: + { + "username": ..., + "password": ..., + "suspicious_answer": ..., + "login_url": ..., + } + */ + let login_info = logins.secret[args[0]]; + + await commands.navigate(login_info.login_url); + await commands.wait.byTime(5000); + + final_button = await setup_login(login_info, context, commands); + user_setup = true; + } else if (cmd == "login") { + if (!user_setup) { + throw Error("setup_login needs to be called before the login command"); + } + await login(context, commands, final_button); + } else { + await run_command(cmdstr, context, commands); + } + } +} + +async function run_command(cmdstr, context, commands) { + let [cmd, ...args] = cmdstr.split(":::"); + let [func, parent_mod] = await get_command_function(cmd, commands); + + try { + await func.call(parent_mod, ...args); + } catch (e) { + context.log.info( + `Exception found while running \`commands.${cmd}(${args})\`: ` + e + ); + } +} + +async function test(context, commands) { + let input_cmds = context.options.browsertime.commands; + let test_type = context.options.browsertime.testType; + if (test_type == "interactive") { + await interactive_test(input_cmds, context, commands); + } else { + await pageload_test(context, commands); + } + return true; +} + +module.exports = { + test, + owner: "Bebe fstrugariu@mozilla.com", + name: "Mozproxy recording generator", + component: "raptor", + description: ` This test generates fresh MozProxy recordings. It iterates through a list of + websites provided in *_sites.json and for each one opens a browser and + records all the associated HTTP traffic`, + usage: + "mach perftest --proxy --hooks testing/raptor/recorder/hooks.py testing/raptor/recorder/perftest_record.js", +}; diff --git a/testing/performance/perftest_youtube_link.js b/testing/performance/perftest_youtube_link.js new file mode 100644 index 0000000000..be34841741 --- /dev/null +++ b/testing/performance/perftest_youtube_link.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.politico.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("js-tealium-tracking"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "YouTube Link", + description: "Measures time to load YouTube video", +}; diff --git a/testing/performance/sites.txt b/testing/performance/sites.txt new file mode 100644 index 0000000000..2f6afdd686 --- /dev/null +++ b/testing/performance/sites.txt @@ -0,0 +1,21 @@ +https://accounts.google.com +https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other +https://discordapp.com/ +https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 +https://fashionbeans.com/article/coolest-menswear-stores-in-the-world +https://m.360.cn +https://m.facebook.com/Cristiano +https://m.imdb.com/title/tt0083943/ +https://m.ranker.com/list/hunger-games-book-vs-movie-comparisons/lisa-waugh?ref=browse_list_5&l=2&pos=1 +https://marvel.wikia.com/wiki/Black_Panther +https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git +https://support.microsoft.com/en-us +https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone +https://www.bbc.com/news/business-47245877 +https://www.booking.com/hotel/us/edwardian-san-francisco.html +https://www.google.com +https://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words +https://www.jianshu.com/ +https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html +https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/ +https://www.youtube.com/watch?v=COU5T-Wafa4 |
