From 2aa4a82499d4becd2284cdb482213d541b8804dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 16:29:10 +0200 Subject: Adding upstream version 86.0.1. Signed-off-by: Daniel Baumann --- testing/performance/.eslintrc.js | 12 +++ testing/performance/README.txt | 1 + testing/performance/hooks.py | 42 ++++++++ testing/performance/hooks_android_main.py | 33 ++++++ testing/performance/hooks_android_view.py | 149 ++++++++++++++++++++++++++ testing/performance/perftest.ini | 9 ++ testing/performance/perftest_android_main.js | 35 ++++++ testing/performance/perftest_android_view.js | 44 ++++++++ testing/performance/perftest_bbc_link.js | 29 +++++ testing/performance/perftest_facebook.js | 51 +++++++++ testing/performance/perftest_jsconf_cold.js | 28 +++++ testing/performance/perftest_jsconf_warm.js | 34 ++++++ testing/performance/perftest_pageload.js | 29 +++++ testing/performance/perftest_politico_link.js | 28 +++++ testing/performance/perftest_youtube_link.js | 28 +++++ testing/performance/sites.txt | 21 ++++ 16 files changed, 573 insertions(+) create mode 100644 testing/performance/.eslintrc.js create mode 100644 testing/performance/README.txt create mode 100644 testing/performance/hooks.py create mode 100644 testing/performance/hooks_android_main.py create mode 100644 testing/performance/hooks_android_view.py create mode 100644 testing/performance/perftest.ini create mode 100644 testing/performance/perftest_android_main.js create mode 100644 testing/performance/perftest_android_view.js create mode 100644 testing/performance/perftest_bbc_link.js create mode 100644 testing/performance/perftest_facebook.js create mode 100644 testing/performance/perftest_jsconf_cold.js create mode 100644 testing/performance/perftest_jsconf_warm.js create mode 100644 testing/performance/perftest_pageload.js create mode 100644 testing/performance/perftest_politico_link.js create mode 100644 testing/performance/perftest_youtube_link.js create mode 100644 testing/performance/sites.txt (limited to 'testing/performance') diff --git a/testing/performance/.eslintrc.js b/testing/performance/.eslintrc.js new file mode 100644 index 0000000000..f040032509 --- /dev/null +++ b/testing/performance/.eslintrc.js @@ -0,0 +1,12 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + browser: true, + node: true, + }, +}; diff --git a/testing/performance/README.txt b/testing/performance/README.txt new file mode 100644 index 0000000000..7cce5088c4 --- /dev/null +++ b/testing/performance/README.txt @@ -0,0 +1 @@ +This directory contains perftests owned by the performance team. diff --git a/testing/performance/hooks.py b/testing/performance/hooks.py new file mode 100644 index 0000000000..4c92d38304 --- /dev/null +++ b/testing/performance/hooks.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +import os +from mozperftest.test.browsertime import add_options, add_option + + +sites = os.path.join(os.path.dirname(__file__), "sites.txt") +with open(sites) as f: + sites = [site for site in f.read().split("\n") if site.strip()] + + +def next_site(): + for site in sites: + yield site + + +get_site = next_site() + +options = [ + ("firefox.preference", "network.http.speculative-parallel-limit:6"), + ("firefox.preference", "gfx.webrender.force-disabled:true"), + # XXX potentially move those as first class options in mozperf? + ("pageCompleteWaitTime", "10000"), + ("visualMetrics", "true"), + ("video", "true"), + ("firefox.windowRecorder", "false"), + ("videoParams.addTimer", "false"), + ("videoParams.createFilmstrip", "false"), + ("videoParams.keepOriginalVideo", "true"), +] + + +def before_runs(env, **kw): + env.set_arg("cycles", len(sites)) + add_options(env, options) + + +def before_cycle(env, **kw): + url = next(get_site) + add_option(env, "browsertime.url", url) diff --git a/testing/performance/hooks_android_main.py b/testing/performance/hooks_android_main.py new file mode 100644 index 0000000000..ef9d4cae91 --- /dev/null +++ b/testing/performance/hooks_android_main.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +from mozperftest.test.browsertime import add_options +from mozperftest.test.browsertime.runner import NodeException + + +common_options = [ + ("firefox.disableBrowsertimeExtension", "true"), + # The webdriver session is not created in the MAIN test so reduce the timeout + # Bug 1640638 + ("timeouts.browserStart", "20000"), + ("browserRestartTries", "1"), + # Explicitly disable the Onboarding flow + ("firefox.android.intentArgument", "'--ez'"), + ("firefox.android.intentArgument", "'performancetest'"), + ("firefox.android.intentArgument", "'true'"), +] + + +def on_exception(env, layer, exc): + if not isinstance(exc, NodeException): + raise exc + return True + + +def logcat_processor(): + pass + + +def before_runs(env, **kw): + add_options(env, common_options) diff --git a/testing/performance/hooks_android_view.py b/testing/performance/hooks_android_view.py new file mode 100644 index 0000000000..c896c2ee1f --- /dev/null +++ b/testing/performance/hooks_android_view.py @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import json +import tempfile +import time +import pathlib + +from mozperftest.test.browsertime import add_options +from mozperftest.system.android import _ROOT_URL +from mozperftest.utils import ( + download_file, + get_multi_tasks_url, + get_revision_namespace_url, + install_package, +) + +URL = "'https://www.example.com'" + +COMMON_OPTIONS = [ + ("processStartTime", "true"), + ("firefox.disableBrowsertimeExtension", "true"), + ("firefox.android.intentArgument", "'-a'"), + ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), + ("firefox.android.intentArgument", "'-d'"), + ("firefox.android.intentArgument", URL), +] + +NIGHTLY_SIM_ROUTE = "mobile.v2.fenix.nightly-simulation" +ROUTE_SUFFIX = "artifacts/public/build/{architecture}/target.apk" + +build_generator = None + + +def before_iterations(kw): + global build_generator + + install_list = kw.get("android_install_apk") + if len(install_list) == 0 or all( + ["fenix_nightlysim_multicommit" not in apk for apk in install_list] + ): + return + + # Install gitpython + install_package(kw["virtualenv"], "gitpython==3.1.0") + import git + + class _GitProgress(git.RemoteProgress): + def update(self, op_code, cur_count, max_count=None, message=""): + if message: + print(message) + + # Setup the local fenix github repo + print("Cloning fenix repo...") + fenix_repo = git.Repo.clone_from( + "https://github.com/mozilla-mobile/fenix", + tempfile.mkdtemp(), + branch="master", + progress=_GitProgress(), + ) + + # Get the builds to test + architecture = ( + "arm64-v8a" if "arm64_v8a" in kw.get("android_install_apk") else "armeabi-v7a" + ) + json_ = _fetch_json( + get_revision_namespace_url, NIGHTLY_SIM_ROUTE, day=kw["test_date"] + ) + namespaces = json_["namespaces"] + revisions = [namespace["name"] for namespace in namespaces] + + tasks = [] + for revision in revisions: + try: + commit = fenix_repo.commit(revision) + name_rev = str(commit.name_rev) + if ( + "remotes/origin" not in name_rev + or "release" in name_rev + or "tag" in name_rev + ): + print( + "Commit %s is a release-branch commit, it won't be tested." + % revision + ) + continue + + commitdate = commit.committed_date + except ValueError: + print("Commit %s is not from the Fenix master branch" % revision) + continue + + json_ = _fetch_json( + get_multi_tasks_url, NIGHTLY_SIM_ROUTE, revision, day=kw["test_date"] + ) + for task in json_["tasks"]: + route = task["namespace"] + task_architecture = route.split(".")[-1] + if task_architecture == architecture: + tasks.append( + { + "timestamp": commitdate, + "revision": revision, + "route": route, + "route_suffix": ROUTE_SUFFIX.format( + architecture=task_architecture + ), + } + ) + + # Set the number of test-iterations to the number of builds + kw["test_iterations"] = len(tasks) + + def _build_iterator(): + for task in tasks: + revision = task["revision"] + timestamp = task["timestamp"] + + humandate = time.ctime(int(timestamp)) + print(f"Testing revision {revision} from {humandate}") + + download_url = f'{_ROOT_URL}{task["route"]}/{task["route_suffix"]}' + yield revision, timestamp, [download_url] + + build_generator = _build_iterator() + + return kw + + +def _fetch_json(get_url_function, *args, **kwargs): + build_url = get_url_function(*args, **kwargs) + tmpfile = pathlib.Path(tempfile.mkdtemp(), "temp.json") + download_file(build_url, tmpfile) + + with tmpfile.open() as f: + return json.load(f) + + +def before_runs(env, **kw): + global build_generator + + add_options(env, COMMON_OPTIONS) + if build_generator: + revision, timestamp, build = next(build_generator) + env.set_arg("android-install-apk", build) + env.set_arg("perfherder-prefix", revision) + env.set_arg("perfherder-timestamp", timestamp) diff --git a/testing/performance/perftest.ini b/testing/performance/perftest.ini new file mode 100644 index 0000000000..deb4c65260 --- /dev/null +++ b/testing/performance/perftest.ini @@ -0,0 +1,9 @@ +[perftest_android_main.js] +[perftest_android_view.js] +[perftest_bbc_link.js] +[perftest_facebook.js] +[perftest_jsconf_cold.js] +[perftest_jsconf_warm.js] +[perftest_pageload.js] +[perftest_politico_link.js] +[perftest_youtube_link.js] diff --git a/testing/performance/perftest_android_main.js b/testing/performance/perftest_android_main.js new file mode 100644 index 0000000000..2a10cbcb86 --- /dev/null +++ b/testing/performance/perftest_android_main.js @@ -0,0 +1,35 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +async function test(context, commands) { + "use strict"; + // Nothing to do -- the timing is captured via logcat + return true; +} + +module.exports = { + test, + owner: "Performance Team", + name: "main", + description: + "Measures the time from process start until the Fenix main activity (HomeActivity) reports Fully Drawn", + longDescription: ` + This test launches Fenix to its main activity (HomeActivity). + The application logs "Fully Drawn" when the activity is drawn. + Using the android log transformer we measure the time from process start to this event. + `, + usage: ` + ./mach perftest testing/performance/perftest_android_main.js --android --flavor mobile-browser \ + --hooks testing/performance/hooks_home_activity.py --perfherder --android-app-name org.mozilla.fenix \ + --android-activity .App --android-install-apk ~/Downloads/fenix.apk --android-clear-logcat \ + --android-capture-logcat logcat \ + --androidlog-first-timestamp ".*Start proc.*org\.mozilla\.fenix.*\.App.*" \ + --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\.fenix.*" \ + --androidlog-subtest-name "MAIN" --androidlog + `, + supportedBrowsers: ["Fenix nightly"], + supportedPlatforms: ["Android"], +}; diff --git a/testing/performance/perftest_android_view.js b/testing/performance/perftest_android_view.js new file mode 100644 index 0000000000..50e4dce85e --- /dev/null +++ b/testing/performance/perftest_android_view.js @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; + +async function test(context, commands) { + await commands.measure.start(); + await commands.measure.browser.wait(commands.measure.pageCompleteCheck); + await commands.measure.stop(); + + const browserScripts = commands.measure.result[0].browserScripts; + + const processLaunchToNavStart = + browserScripts.pageinfo.navigationStartTime - + browserScripts.browser.processStartTime; + + browserScripts.pageinfo.processLaunchToNavStart = processLaunchToNavStart; + console.log("processLaunchToNavStart: " + processLaunchToNavStart); + + return true; +} + +module.exports = { + test, + owner: "Performance Team", + name: "VIEW", + description: "Measures cold process view time", + longDescription: ` + This test launches the appropriate android app, simulating a opening a link through VIEW intent + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_android_view.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_android_view.py \ + --android-app-name org.mozilla.fenix \ + --perfherder-metrics processLaunchToNavStart + `, + supportedBrowsers: ["Fenix nightly", "Geckoview_example", "Fennec"], + supportedPlatforms: ["Android"], +}; diff --git a/testing/performance/perftest_bbc_link.js b/testing/performance/perftest_bbc_link.js new file mode 100644 index 0000000000..3021c44c83 --- /dev/null +++ b/testing/performance/perftest_bbc_link.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.bbc.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("block-link__overlay-link"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "BBC Link", + component: "pageload", + description: "Measures time to load BBC homepage", +}; diff --git a/testing/performance/perftest_facebook.js b/testing/performance/perftest_facebook.js new file mode 100644 index 0000000000..bd607d6be4 --- /dev/null +++ b/testing/performance/perftest_facebook.js @@ -0,0 +1,51 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + await commands.navigate("https://www.example.com"); + await commands.wait.byTime(15000); + + // Fill in: + let username = "some_user@mail.com"; + let passworld = "topsecretpassword"; + + // We start by navigating to the login page. + await commands.navigate("https://www.facebook.com"); + + // When we fill in a input field/click on a link we wanna + // try/catch that if the HTML on the page changes in the feature + // sitespeed.io will automatically log the error in a user friendly + // way, and the error will be re-thrown so you can act on it. + await commands.wait.byTime(5000); + + // Add text into an input field, finding the field by id + await commands.addText.bySelector(username, "input[name=email]"); + await commands.wait.byTime(2000); + await commands.addText.bySelector(passworld, "input[name=pass]"); + await commands.wait.byTime(2000); + + // Start the measurement before we click on the + // submit button. Sitespeed.io will start the video recording + // and prepare everything. + // Find the sumbit button and click it and then wait + // for the pageCompleteCheck to finish + await commands.measure.start("pageload"); + + // There are two variants of the facebook login page: + try { + await commands.click.bySelectorAndWait("button[name=login]"); + } catch (e) { + await commands.click.bySelectorAndWait("input[type=submit]"); + } + + // Stop and collect the measurement before the next page we want to measure + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "Facebook", + description: "Measures time to log in to Facebook", +}; diff --git a/testing/performance/perftest_jsconf_cold.js b/testing/performance/perftest_jsconf_cold.js new file mode 100644 index 0000000000..c6b68a3e55 --- /dev/null +++ b/testing/performance/perftest_jsconf_cold.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://2019.jsconf.eu/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byLinkTextAndWait("Artists"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "JSConf (cold)", + description: "Measures time to load JSConf page (cold)", +}; diff --git a/testing/performance/perftest_jsconf_warm.js b/testing/performance/perftest_jsconf_warm.js new file mode 100644 index 0000000000..be58553ab6 --- /dev/null +++ b/testing/performance/perftest_jsconf_warm.js @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://2019.jsconf.eu/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Click on the link and wait a few seconds + await commands.click.byLinkTextAndWait("Artists"); + await commands.wait.byTime(3000); + + // Back to about + await commands.click.byLinkTextAndWait("About"); + await commands.wait.byTime(3000); + + // Start the measurement + await commands.measure.start("pageload"); + await commands.click.byLinkTextAndWait("Artists"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "JSConf (warm)", + description: "Measures time to load JSConf page (warm)", +}; diff --git a/testing/performance/perftest_pageload.js b/testing/performance/perftest_pageload.js new file mode 100644 index 0000000000..0adbc6f741 --- /dev/null +++ b/testing/performance/perftest_pageload.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function setUp(context) { + context.log.info("setUp example!"); +} + +async function test(context, commands) { + let url = context.options.browsertime.url; + await commands.navigate("https://www.mozilla.org/en-US/"); + await commands.wait.byTime(100); + await commands.navigate("about:blank"); + await commands.wait.byTime(50); + return commands.measure.start(url); +} + +async function tearDown(context) { + context.log.info("tearDown example!"); +} + +module.exports = { // eslint-disable-line + setUp, + tearDown, + test, + owner: "Performance Team", + name: "pageload", + description: "Measures time to load mozilla page", +}; diff --git a/testing/performance/perftest_politico_link.js b/testing/performance/perftest_politico_link.js new file mode 100644 index 0000000000..019595afb8 --- /dev/null +++ b/testing/performance/perftest_politico_link.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.politico.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("js-tealium-tracking"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "Politico Link", + description: "Measures time to load Politico homepage", +}; diff --git a/testing/performance/perftest_youtube_link.js b/testing/performance/perftest_youtube_link.js new file mode 100644 index 0000000000..be34841741 --- /dev/null +++ b/testing/performance/perftest_youtube_link.js @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +async function test(context, commands) { + let rootUrl = "https://www.politico.com/"; + + await commands.navigate(rootUrl); + + // Wait for browser to settle + await commands.wait.byTime(10000); + + // Start the measurement + await commands.measure.start("pageload"); + + // Click on the link and wait for page complete check to finish. + await commands.click.byClassNameAndWait("js-tealium-tracking"); + + // Stop and collect the measurement + await commands.measure.stop(); +} + +module.exports = { + test, + owner: "Performance Team", + name: "YouTube Link", + description: "Measures time to load YouTube video", +}; diff --git a/testing/performance/sites.txt b/testing/performance/sites.txt new file mode 100644 index 0000000000..2f6afdd686 --- /dev/null +++ b/testing/performance/sites.txt @@ -0,0 +1,21 @@ +https://accounts.google.com +https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other +https://discordapp.com/ +https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 +https://fashionbeans.com/article/coolest-menswear-stores-in-the-world +https://m.360.cn +https://m.facebook.com/Cristiano +https://m.imdb.com/title/tt0083943/ +https://m.ranker.com/list/hunger-games-book-vs-movie-comparisons/lisa-waugh?ref=browse_list_5&l=2&pos=1 +https://marvel.wikia.com/wiki/Black_Panther +https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git +https://support.microsoft.com/en-us +https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone +https://www.bbc.com/news/business-47245877 +https://www.booking.com/hotel/us/edwardian-san-francisco.html +https://www.google.com +https://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words +https://www.jianshu.com/ +https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html +https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/ +https://www.youtube.com/watch?v=COU5T-Wafa4 -- cgit v1.2.3