Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

16 files changed, 573 insertions, 0 deletions

diff --git a/testing/performance/.eslintrc.js b/testing/performance/.eslintrc.js
new file mode 100644
index 0000000000..f040032509
--- /dev/null
+++ b/testing/performance/.eslintrc.js
@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+module.exports = {
+  env: {
+    browser: true,
+    node: true,
+  },
+};
diff --git a/testing/performance/README.txt b/testing/performance/README.txt
new file mode 100644
index 0000000000..7cce5088c4
--- /dev/null
+++ b/testing/performance/README.txt
@@ -0,0 +1 @@
+This directory contains perftests owned by the performance team.
diff --git a/testing/performance/hooks.py b/testing/performance/hooks.py
new file mode 100644
index 0000000000..4c92d38304
--- /dev/null
+++ b/testing/performance/hooks.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+import os
+from mozperftest.test.browsertime import add_options, add_option
+
+
+sites = os.path.join(os.path.dirname(__file__), "sites.txt")
+with open(sites) as f:
+    sites = [site for site in f.read().split("\n") if site.strip()]
+
+
+def next_site():
+    for site in sites:
+        yield site
+
+
+get_site = next_site()
+
+options = [
+    ("firefox.preference", "network.http.speculative-parallel-limit:6"),
+    ("firefox.preference", "gfx.webrender.force-disabled:true"),
+    # XXX potentially move those as first class options in mozperf?
+    ("pageCompleteWaitTime", "10000"),
+    ("visualMetrics", "true"),
+    ("video", "true"),
+    ("firefox.windowRecorder", "false"),
+    ("videoParams.addTimer", "false"),
+    ("videoParams.createFilmstrip", "false"),
+    ("videoParams.keepOriginalVideo", "true"),
+]
+
+
+def before_runs(env, **kw):
+    env.set_arg("cycles", len(sites))
+    add_options(env, options)
+
+
+def before_cycle(env, **kw):
+    url = next(get_site)
+    add_option(env, "browsertime.url", url)
diff --git a/testing/performance/hooks_android_main.py b/testing/performance/hooks_android_main.py
new file mode 100644
index 0000000000..ef9d4cae91
--- /dev/null
+++ b/testing/performance/hooks_android_main.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from mozperftest.test.browsertime import add_options
+from mozperftest.test.browsertime.runner import NodeException
+
+
+common_options = [
+    ("firefox.disableBrowsertimeExtension", "true"),
+    # The webdriver session is not created in the MAIN test so reduce the timeout
+    # Bug 1640638
+    ("timeouts.browserStart", "20000"),
+    ("browserRestartTries", "1"),
+    # Explicitly disable the Onboarding flow
+    ("firefox.android.intentArgument", "'--ez'"),
+    ("firefox.android.intentArgument", "'performancetest'"),
+    ("firefox.android.intentArgument", "'true'"),
+]
+
+
+def on_exception(env, layer, exc):
+    if not isinstance(exc, NodeException):
+        raise exc
+    return True
+
+
+def logcat_processor():
+    pass
+
+
+def before_runs(env, **kw):
+    add_options(env, common_options)
diff --git a/testing/performance/hooks_android_view.py b/testing/performance/hooks_android_view.py
new file mode 100644
index 0000000000..c896c2ee1f
--- /dev/null
+++ b/testing/performance/hooks_android_view.py
@@ -0,0 +1,149 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import json
+import tempfile
+import time
+import pathlib
+
+from mozperftest.test.browsertime import add_options
+from mozperftest.system.android import _ROOT_URL
+from mozperftest.utils import (
+    download_file,
+    get_multi_tasks_url,
+    get_revision_namespace_url,
+    install_package,
+)
+
+URL = "'https://www.example.com'"
+
+COMMON_OPTIONS = [
+    ("processStartTime", "true"),
+    ("firefox.disableBrowsertimeExtension", "true"),
+    ("firefox.android.intentArgument", "'-a'"),
+    ("firefox.android.intentArgument", "'android.intent.action.VIEW'"),
+    ("firefox.android.intentArgument", "'-d'"),
+    ("firefox.android.intentArgument", URL),
+]
+
+NIGHTLY_SIM_ROUTE = "mobile.v2.fenix.nightly-simulation"
+ROUTE_SUFFIX = "artifacts/public/build/{architecture}/target.apk"
+
+build_generator = None
+
+
+def before_iterations(kw):
+    global build_generator
+
+    install_list = kw.get("android_install_apk")
+    if len(install_list) == 0 or all(
+        ["fenix_nightlysim_multicommit" not in apk for apk in install_list]
+    ):
+        return
+
+    # Install gitpython
+    install_package(kw["virtualenv"], "gitpython==3.1.0")
+    import git
+
+    class _GitProgress(git.RemoteProgress):
+        def update(self, op_code, cur_count, max_count=None, message=""):
+            if message:
+                print(message)
+
+    # Setup the local fenix github repo
+    print("Cloning fenix repo...")
+    fenix_repo = git.Repo.clone_from(
+        "https://github.com/mozilla-mobile/fenix",
+        tempfile.mkdtemp(),
+        branch="master",
+        progress=_GitProgress(),
+    )
+
+    # Get the builds to test
+    architecture = (
+        "arm64-v8a" if "arm64_v8a" in kw.get("android_install_apk") else "armeabi-v7a"
+    )
+    json_ = _fetch_json(
+        get_revision_namespace_url, NIGHTLY_SIM_ROUTE, day=kw["test_date"]
+    )
+    namespaces = json_["namespaces"]
+    revisions = [namespace["name"] for namespace in namespaces]
+
+    tasks = []
+    for revision in revisions:
+        try:
+            commit = fenix_repo.commit(revision)
+            name_rev = str(commit.name_rev)
+            if (
+                "remotes/origin" not in name_rev
+                or "release" in name_rev
+                or "tag" in name_rev
+            ):
+                print(
+                    "Commit %s is a release-branch commit, it won't be tested."
+                    % revision
+                )
+                continue
+
+            commitdate = commit.committed_date
+        except ValueError:
+            print("Commit %s is not from the Fenix master branch" % revision)
+            continue
+
+        json_ = _fetch_json(
+            get_multi_tasks_url, NIGHTLY_SIM_ROUTE, revision, day=kw["test_date"]
+        )
+        for task in json_["tasks"]:
+            route = task["namespace"]
+            task_architecture = route.split(".")[-1]
+            if task_architecture == architecture:
+                tasks.append(
+                    {
+                        "timestamp": commitdate,
+                        "revision": revision,
+                        "route": route,
+                        "route_suffix": ROUTE_SUFFIX.format(
+                            architecture=task_architecture
+                        ),
+                    }
+                )
+
+    # Set the number of test-iterations to the number of builds
+    kw["test_iterations"] = len(tasks)
+
+    def _build_iterator():
+        for task in tasks:
+            revision = task["revision"]
+            timestamp = task["timestamp"]
+
+            humandate = time.ctime(int(timestamp))
+            print(f"Testing revision {revision} from {humandate}")
+
+            download_url = f'{_ROOT_URL}{task["route"]}/{task["route_suffix"]}'
+            yield revision, timestamp, [download_url]
+
+    build_generator = _build_iterator()
+
+    return kw
+
+
+def _fetch_json(get_url_function, *args, **kwargs):
+    build_url = get_url_function(*args, **kwargs)
+    tmpfile = pathlib.Path(tempfile.mkdtemp(), "temp.json")
+    download_file(build_url, tmpfile)
+
+    with tmpfile.open() as f:
+        return json.load(f)
+
+
+def before_runs(env, **kw):
+    global build_generator
+
+    add_options(env, COMMON_OPTIONS)
+    if build_generator:
+        revision, timestamp, build = next(build_generator)
+        env.set_arg("android-install-apk", build)
+        env.set_arg("perfherder-prefix", revision)
+        env.set_arg("perfherder-timestamp", timestamp)
diff --git a/testing/performance/perftest.ini b/testing/performance/perftest.ini
new file mode 100644
index 0000000000..deb4c65260
--- /dev/null
+++ b/testing/performance/perftest.ini
@@ -0,0 +1,9 @@
+[perftest_android_main.js]
+[perftest_android_view.js]
+[perftest_bbc_link.js]
+[perftest_facebook.js]
+[perftest_jsconf_cold.js]
+[perftest_jsconf_warm.js]
+[perftest_pageload.js]
+[perftest_politico_link.js]
+[perftest_youtube_link.js]
diff --git a/testing/performance/perftest_android_main.js b/testing/performance/perftest_android_main.js
new file mode 100644
index 0000000000..2a10cbcb86
--- /dev/null
+++ b/testing/performance/perftest_android_main.js
@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+async function test(context, commands) {
+  "use strict";
+  // Nothing to do -- the timing is captured via logcat
+  return true;
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "main",
+  description:
+    "Measures the time from process start until the Fenix main activity (HomeActivity) reports Fully Drawn",
+  longDescription: `
+  This test launches Fenix to its main activity (HomeActivity).
+  The application logs "Fully Drawn" when the activity is drawn.
+  Using the android log transformer we measure the time from process start to this event.
+  `,
+  usage: `
+    ./mach perftest testing/performance/perftest_android_main.js --android --flavor mobile-browser \
+    --hooks testing/performance/hooks_home_activity.py --perfherder --android-app-name  org.mozilla.fenix \
+    --android-activity .App --android-install-apk ~/Downloads/fenix.apk --android-clear-logcat \
+    --android-capture-logcat logcat \
+    --androidlog-first-timestamp ".*Start proc.*org\.mozilla\.fenix.*\.App.*" \
+    --androidlog-second-timestamp ".*Fully drawn.*org\.mozilla\.fenix.*" \
+    --androidlog-subtest-name "MAIN" --androidlog
+  `,
+  supportedBrowsers: ["Fenix nightly"],
+  supportedPlatforms: ["Android"],
+};
diff --git a/testing/performance/perftest_android_view.js b/testing/performance/perftest_android_view.js
new file mode 100644
index 0000000000..50e4dce85e
--- /dev/null
+++ b/testing/performance/perftest_android_view.js
@@ -0,0 +1,44 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+/* eslint-env node */
+"use strict";
+
+async function test(context, commands) {
+  await commands.measure.start();
+  await commands.measure.browser.wait(commands.measure.pageCompleteCheck);
+  await commands.measure.stop();
+
+  const browserScripts = commands.measure.result[0].browserScripts;
+
+  const processLaunchToNavStart =
+    browserScripts.pageinfo.navigationStartTime -
+    browserScripts.browser.processStartTime;
+
+  browserScripts.pageinfo.processLaunchToNavStart = processLaunchToNavStart;
+  console.log("processLaunchToNavStart: " + processLaunchToNavStart);
+
+  return true;
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "VIEW",
+  description: "Measures cold process view time",
+  longDescription: `
+  This test launches the appropriate android app, simulating a opening a link through VIEW intent
+  workflow. The application is launched with the intent action
+  android.intent.action.VIEW loading a trivially simple website. The reported
+  metric is the time from process start to navigationStart, reported as processLaunchToNavStart
+  `,
+  usage: `
+  ./mach perftest testing/performance/perftest_android_view.js \
+    --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \
+    --hooks testing/performance/hooks_android_view.py \
+    --android-app-name org.mozilla.fenix \
+    --perfherder-metrics processLaunchToNavStart
+  `,
+  supportedBrowsers: ["Fenix nightly", "Geckoview_example", "Fennec"],
+  supportedPlatforms: ["Android"],
+};
diff --git a/testing/performance/perftest_bbc_link.js b/testing/performance/perftest_bbc_link.js
new file mode 100644
index 0000000000..3021c44c83
--- /dev/null
+++ b/testing/performance/perftest_bbc_link.js
@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  let rootUrl = "https://www.bbc.com/";
+
+  await commands.navigate(rootUrl);
+
+  // Wait for browser to settle
+  await commands.wait.byTime(10000);
+
+  // Start the measurement
+  await commands.measure.start("pageload");
+
+  // Click on the link and wait for page complete check to finish.
+  await commands.click.byClassNameAndWait("block-link__overlay-link");
+
+  // Stop and collect the measurement
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "BBC Link",
+  component: "pageload",
+  description: "Measures time to load BBC homepage",
+};
diff --git a/testing/performance/perftest_facebook.js b/testing/performance/perftest_facebook.js
new file mode 100644
index 0000000000..bd607d6be4
--- /dev/null
+++ b/testing/performance/perftest_facebook.js
@@ -0,0 +1,51 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  await commands.navigate("https://www.example.com");
+  await commands.wait.byTime(15000);
+
+  // Fill in:
+  let username = "some_user@mail.com";
+  let passworld = "topsecretpassword";
+
+  // We start by navigating to the login page.
+  await commands.navigate("https://www.facebook.com");
+
+  // When we fill in a input field/click on a link we wanna
+  // try/catch that if the HTML on the page changes in the feature
+  // sitespeed.io will automatically log the error in a user friendly
+  // way, and the error will be re-thrown so you can act on it.
+  await commands.wait.byTime(5000);
+
+  // Add text into an input field, finding the field by id
+  await commands.addText.bySelector(username, "input[name=email]");
+  await commands.wait.byTime(2000);
+  await commands.addText.bySelector(passworld, "input[name=pass]");
+  await commands.wait.byTime(2000);
+
+  // Start the measurement before we click on the
+  // submit button. Sitespeed.io will start the video recording
+  // and prepare everything.
+  // Find the sumbit button and click it and then wait
+  // for the pageCompleteCheck to finish
+  await commands.measure.start("pageload");
+
+  // There are two variants of the facebook login page:
+  try {
+    await commands.click.bySelectorAndWait("button[name=login]");
+  } catch (e) {
+    await commands.click.bySelectorAndWait("input[type=submit]");
+  }
+
+  // Stop and collect the measurement before the next page we want to measure
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "Facebook",
+  description: "Measures time to log in to Facebook",
+};
diff --git a/testing/performance/perftest_jsconf_cold.js b/testing/performance/perftest_jsconf_cold.js
new file mode 100644
index 0000000000..c6b68a3e55
--- /dev/null
+++ b/testing/performance/perftest_jsconf_cold.js
@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  let rootUrl = "https://2019.jsconf.eu/";
+
+  await commands.navigate(rootUrl);
+
+  // Wait for browser to settle
+  await commands.wait.byTime(10000);
+
+  // Start the measurement
+  await commands.measure.start("pageload");
+
+  // Click on the link and wait for page complete check to finish.
+  await commands.click.byLinkTextAndWait("Artists");
+
+  // Stop and collect the measurement
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "JSConf (cold)",
+  description: "Measures time to load JSConf page (cold)",
+};
diff --git a/testing/performance/perftest_jsconf_warm.js b/testing/performance/perftest_jsconf_warm.js
new file mode 100644
index 0000000000..be58553ab6
--- /dev/null
+++ b/testing/performance/perftest_jsconf_warm.js
@@ -0,0 +1,34 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  let rootUrl = "https://2019.jsconf.eu/";
+
+  await commands.navigate(rootUrl);
+
+  // Wait for browser to settle
+  await commands.wait.byTime(10000);
+
+  // Click on the link and wait a few seconds
+  await commands.click.byLinkTextAndWait("Artists");
+  await commands.wait.byTime(3000);
+
+  // Back to about
+  await commands.click.byLinkTextAndWait("About");
+  await commands.wait.byTime(3000);
+
+  // Start the measurement
+  await commands.measure.start("pageload");
+  await commands.click.byLinkTextAndWait("Artists");
+
+  // Stop and collect the measurement
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "JSConf (warm)",
+  description: "Measures time to load JSConf page (warm)",
+};
diff --git a/testing/performance/perftest_pageload.js b/testing/performance/perftest_pageload.js
new file mode 100644
index 0000000000..0adbc6f741
--- /dev/null
+++ b/testing/performance/perftest_pageload.js
@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function setUp(context) {
+  context.log.info("setUp example!");
+}
+
+async function test(context, commands) {
+  let url = context.options.browsertime.url;
+  await commands.navigate("https://www.mozilla.org/en-US/");
+  await commands.wait.byTime(100);
+  await commands.navigate("about:blank");
+  await commands.wait.byTime(50);
+  return commands.measure.start(url);
+}
+
+async function tearDown(context) {
+  context.log.info("tearDown example!");
+}
+
+module.exports = {   // eslint-disable-line
+  setUp,
+  tearDown,
+  test,
+  owner: "Performance Team",
+  name: "pageload",
+  description: "Measures time to load mozilla page",
+};
diff --git a/testing/performance/perftest_politico_link.js b/testing/performance/perftest_politico_link.js
new file mode 100644
index 0000000000..019595afb8
--- /dev/null
+++ b/testing/performance/perftest_politico_link.js
@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  let rootUrl = "https://www.politico.com/";
+
+  await commands.navigate(rootUrl);
+
+  // Wait for browser to settle
+  await commands.wait.byTime(10000);
+
+  // Start the measurement
+  await commands.measure.start("pageload");
+
+  // Click on the link and wait for page complete check to finish.
+  await commands.click.byClassNameAndWait("js-tealium-tracking");
+
+  // Stop and collect the measurement
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "Politico Link",
+  description: "Measures time to load Politico homepage",
+};
diff --git a/testing/performance/perftest_youtube_link.js b/testing/performance/perftest_youtube_link.js
new file mode 100644
index 0000000000..be34841741
--- /dev/null
+++ b/testing/performance/perftest_youtube_link.js
@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function test(context, commands) {
+  let rootUrl = "https://www.politico.com/";
+
+  await commands.navigate(rootUrl);
+
+  // Wait for browser to settle
+  await commands.wait.byTime(10000);
+
+  // Start the measurement
+  await commands.measure.start("pageload");
+
+  // Click on the link and wait for page complete check to finish.
+  await commands.click.byClassNameAndWait("js-tealium-tracking");
+
+  // Stop and collect the measurement
+  await commands.measure.stop();
+}
+
+module.exports = {
+  test,
+  owner: "Performance Team",
+  name: "YouTube Link",
+  description: "Measures time to load YouTube video",
+};
diff --git a/testing/performance/sites.txt b/testing/performance/sites.txt
new file mode 100644
index 0000000000..2f6afdd686
--- /dev/null
+++ b/testing/performance/sites.txt
@@ -0,0 +1,21 @@
+https://accounts.google.com
+https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
+https://discordapp.com/
+https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2
+https://fashionbeans.com/article/coolest-menswear-stores-in-the-world
+https://m.360.cn
+https://m.facebook.com/Cristiano
+https://m.imdb.com/title/tt0083943/
+https://m.ranker.com/list/hunger-games-book-vs-movie-comparisons/lisa-waugh?ref=browse_list_5&l=2&pos=1
+https://marvel.wikia.com/wiki/Black_Panther
+https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git
+https://support.microsoft.com/en-us
+https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
+https://www.bbc.com/news/business-47245877
+https://www.booking.com/hotel/us/edwardian-san-francisco.html
+https://www.google.com
+https://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
+https://www.jianshu.com/
+https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html
+https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/
+https://www.youtube.com/watch?v=COU5T-Wafa4