Merging upstream version 127.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

23 files changed, 104 insertions, 62 deletions

diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py
index 0de7b1a239..f06763e2ca 100644
--- a/testing/raptor/raptor/browsertime/base.py
+++ b/testing/raptor/raptor/browsertime/base.py
@@ -186,7 +186,7 @@ class Browsertime(Perftest):
                 # setup once all chrome versions use the new artifact setup.
                 cd_extracted_names_115 = {
                     "windows": str(
-                        pathlib.Path("{}chromedriver-win32", "chromedriver.exe")
+                        pathlib.Path("{}chromedriver-win64", "chromedriver.exe")
                     ),
                     "mac-x86_64": str(
                         pathlib.Path("{}chromedriver-mac-x64", "chromedriver")
@@ -212,7 +212,8 @@ class Browsertime(Perftest):
                     elif "win" in self.config["platform"]:
                         self.browsertime_chromedriver = (
                             self.browsertime_chromedriver.replace(
-                                "{}chromedriver.exe", cd_extracted_names_115["windows"]
+                                "{}chromedriver.exe",
+                                cd_extracted_names_115["windows"],
                             )
                         )
                     else:
@@ -598,13 +599,19 @@ class Browsertime(Perftest):
                 browsertime_options=browsertime_options, test=test
             )
 
-        return (
+        cmd = (
             [self.browsertime_node, self.browsertime_browsertimejs]
             + self.driver_paths
             + [browsertime_script]
             + browsertime_options
         )
 
+        if test.get("support_class", None):
+            LOG.info("Test support class is modifying the command...")
+            test.get("support_class").modify_command(cmd, test)
+
+        return cmd
+
     def _compose_gecko_profiler_cmds(self, test, priority1_options):
         """Modify the command line options for running the gecko profiler
         in the Firefox application"""
@@ -623,7 +630,7 @@ class Browsertime(Perftest):
             (
                 "gecko_profile_features",
                 "--firefox.geckoProfilerParams.features",
-                "js,stackwalk,cpu,screenshots",
+                "js,stackwalk,cpu,screenshots,memory",
             ),
             (
                 "gecko_profile_threads",
@@ -916,10 +923,6 @@ class Browsertime(Perftest):
         # this will be used for btime --timeouts.pageLoad
         cmd = self._compose_cmd(test, timeout)
 
-        if test.get("support_class", None):
-            LOG.info("Test support class is modifying the command...")
-            test.get("support_class").modify_command(cmd, test)
-
         output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT
         if test.get("type", "") == "scenario":
             # Change the timeout for scenarios since they
diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py
index 3a79e65341..5542fdfdd8 100644
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -198,7 +198,7 @@ def create_parser(mach_interface=False):
     add_arg(
         "--gecko-profile-interval",
         dest="gecko_profile_interval",
-        type=int,
+        type=float,
         help="How frequently to take samples (milliseconds)",
     )
     add_arg(
diff --git a/testing/raptor/raptor/filters.py b/testing/raptor/raptor/filters.py
index 87b0d827e6..08a3241621 100644
--- a/testing/raptor/raptor/filters.py
+++ b/testing/raptor/raptor/filters.py
@@ -4,7 +4,6 @@
 
 # originally taken from /testing/talos/talos/filter.py
 
-import math
 
 """
 data filters:
@@ -175,11 +174,10 @@ def geometric_mean(series):
     """
     geometric_mean: http://en.wikipedia.org/wiki/Geometric_mean
     """
-    total = 0
-    for i in series:
-        total += math.log(i + 1)
-    # pylint --py3k W1619
-    return math.exp(total / len(series)) - 1
+
+    from scipy.stats.mstats import gmean
+
+    return gmean(series)
 
 
 # filters that return a list
diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py
index 509f74afb4..cae7af7c26 100644
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -275,7 +275,7 @@ def write_test_settings_json(args, test_details, oskey):
                 "gecko_profile_entries": int(
                     test_details.get("gecko_profile_entries", 1000000)
                 ),
-                "gecko_profile_interval": int(
+                "gecko_profile_interval": float(
                     test_details.get("gecko_profile_interval", 1)
                 ),
                 "gecko_profile_threads": ",".join(set(threads)),
diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml
index 723da06287..b4d61bd10a 100644
--- a/testing/raptor/raptor/perfdocs/config.yml
+++ b/testing/raptor/raptor/perfdocs/config.yml
@@ -12,7 +12,7 @@ metrics:
             - First Contentful Paint
             - First Contentful Composite
         description: >
-            Denotes the first time the browser performs a paint that has content in it.
+            Denotes the first time the browser performs a paint that has content in it (in ms).
     "Youtube Playback Metrics":
         aliases:
             - VP9
@@ -20,10 +20,51 @@ metrics:
         matcher: VP9.*|H264.*
         description: >
             Metrics starting with VP9/H264 give the number of frames dropped, and painted.
-    "In Progress":
-        aliases: []
-        matcher: .*
-        description: Many metrics have not been documented yet, this is a placeholder.
+    "First Visual Change":
+        aliases:
+            - FirstVisualChange
+        description: The first visual change detected in the test (in ms).
+    "Last Visual Change":
+        aliases:
+            - LastVisualChange
+        description: The last visual change detected in the test (in ms).
+    "Load Time":
+        aliases:
+            - loadtime
+        description: The time it took for the page to complete loading (in ms).
+    "Largest Contentful Paint":
+        aliases:
+            - largestContentfulPaint
+            - lcp
+        description: >
+            The time (in ms) at which the largest piece of content on the page was rendered/painted.
+    "Speed Index":
+        aliases:
+            - SpeedIndex
+        description: >
+            A metric used to denote the speed at which a page loaded. Lower values indicate faster
+            pageloads. Units are in (Visually-Complete x Milliseconds). Calculated by using videos
+            of the pageload which provide a measure of visual completeness. Visual completeness is
+            calculated by comparing the histogram of a given frame to the final frame of the pageload.
+            The SpeedIndex is calculated as the area between the curves of a constant line at y=1,
+            and the graph of the visual completeness from 0ms to when visual completeness reaches
+            100% (or hits the y=1 line).
+    "Contentful Speed Index":
+        aliases:
+            - ContentfulSpeedIndex
+        description: >
+            Similar to SpeedIndex, except that it uses the contentfulness of a frame to determine
+            visual completeness instead of the histogram differences. The contentfulness is determined
+            by calculating the number of edges found in the frame. A lower number of edges detected
+            gives a smaller contentfulness value (and a smaller visually complete value).
+    "Perceptual Speed Index":
+        aliases:
+            - PerceptualSpeedIndex
+        description: >
+            Similar to SpeedIndex, except that it uses the structural similarity index measure (ssim)
+            to determine visual completeness. This technique compares the luminance, contrast, and
+            structure of the frames (a given frame vs. a final frame) to determine the completeness.
+
 suites:
     desktop:
         description: "Tests for page-load performance. The links direct to the actual websites that are being tested."
diff --git a/testing/raptor/raptor/perfdocs/debugging.rst b/testing/raptor/raptor/perfdocs/debugging.rst
index 8cc799e820..87165765f8 100644
--- a/testing/raptor/raptor/perfdocs/debugging.rst
+++ b/testing/raptor/raptor/perfdocs/debugging.rst
@@ -14,9 +14,9 @@ As of now, there is no easy way to do this. Raptor was not built for debugging f
 To debug a functional failure in Raptor you can follow these steps:
 
 #. If bug 1653617 has not landed yet, apply the patch.
-#. Add the --verbose flag to the extra-options list `here <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml#98-101>`__.
+#. Add the --verbose flag to the extra-options list `here <https://searchfox.org/mozilla-central/source/taskcluster/kinds/test/raptor.yml#98-101>`__.
 #. If the --setenv doesn't exist yet (`bug 1494669 <https://bugzilla.mozilla.org/show_bug.cgi?id=1494669>`_), then add your MOZ_LOG environment variables to give you additional logging `here <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/webextension/desktop.py#42>`_.
-#. If the flag does exist, then you can add the MOZ_LOG variables to the `raptor.yml <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml>`_ configuration file.
+#. If the flag does exist, then you can add the MOZ_LOG variables to the `raptor.yml <https://searchfox.org/mozilla-central/source/taskcluster/kinds/test/raptor.yml>`_ configuration file.
 #. Push to try if you can't reproduce the failure locally.
 
 You can follow `bug 1655554 <https://bugzilla.mozilla.org/show_bug.cgi?id=1655554>`_ as we work on improving this workflow.
diff --git a/testing/raptor/raptor/perfdocs/index.rst b/testing/raptor/raptor/perfdocs/index.rst
index 88c1702c00..082fa05940 100644
--- a/testing/raptor/raptor/perfdocs/index.rst
+++ b/testing/raptor/raptor/perfdocs/index.rst
@@ -23,7 +23,7 @@ Locally, Raptor can be invoked with the following command:
    browsertime
    debugging
    contributing
-   metrics
+   {metrics_rst_name}
 
 .. contents::
    :depth: 2
diff --git a/testing/raptor/raptor/perfdocs/metrics.rst b/testing/raptor/raptor/perfdocs/metrics.rst
index 931456467c..106f62b750 100644
--- a/testing/raptor/raptor/perfdocs/metrics.rst
+++ b/testing/raptor/raptor/perfdocs/metrics.rst
@@ -80,3 +80,10 @@ Benchmarks
 ----------
 
 Benchmarks gather their own custom metrics unlike the pageload tests above. Please ping the owners of those benchmarks to determine what they mean and how they are produced, or reach out to the Performance Test and Tooling team in #perftest on Element.
+
+Metric Definitions
+------------------
+
+The following documents all available metrics that current alert in Raptor Browsertime tests.
+
+{metrics_documentation}
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml
index a480054f7a..2001cd3f7c 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "firefox, chrome, safari, custom-car"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_browser_profiler = true
 expose_chrome_trace = true
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml
index f68f418608..46470c4535 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "fenix, geckoview, chrome-m, cstm-car-m"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_browser_profiler = true
 lower_is_better = false
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml
index 78ffae7811..c82c197355 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "firefox, chrome, safari"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_browser_profiler = true
 lower_is_better = false
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml
index 4c178dc47f..6899d0ff42 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "fenix, chrome-m, geckoview"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_browser_profiler = true
 lower_is_better = false
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml
index f62f1d993a..b52e97339d 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "firefox, chrome, safari"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_gecko_profiler = true
 expose_chrome_trace = true
diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml
index a617735305..d02dc5b0f5 100644
--- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml
+++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml
@@ -2,7 +2,7 @@
 alert_threshold = 2.0
 apps = "fenix, chrome-m, geckoview"
 gecko_profile_interval = 1
-gecko_profile_features = "stackwalk,js,cpu,java,processcpu"
+gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory"
 gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate"
 expose_gecko_profiler = true
 expose_chrome_trace = true
diff --git a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml
index 4e510b3d12..83262e5f24 100644
--- a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml
+++ b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml
@@ -4,7 +4,7 @@ alert_threshold = 2.0
 browser_cycles = 1
 custom_data = true
 gecko_profile_entries = 131072000 # 1GB
-gecko_profile_features = "js,stackwalk,cpu"
+gecko_profile_features = "js,stackwalk,cpu,memory"
 gecko_profile_threads = "GeckoMain,DOM Worker,IndexedDB"
 lower_is_better = true
 measure = "cpuTime"
diff --git a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml
index 9091ec05ce..9bddb0c0b9 100644
--- a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml
+++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml
@@ -48,6 +48,7 @@ test_url = "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comp
 secondary_url = "https://www.espn.com/nba/draft/news"
 
 ["expedia"]
+playback_pageset_manifest = "mitm8-darwin-firefox-expedia.manifest"
 secondary_url = "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1"
 test_url = "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2"
 
@@ -108,6 +109,7 @@ secondary_url = "https://www.linkedin.com/in/thommy-harris-hk-385723106/"
 test_url = "https://www.linkedin.com/feed/"
 
 ["microsoft"]
+playback_pageset_manifest = "mitm8-linux-firefox-microsoft.manifest"
 secondary_url = "https://support.microsoft.com/en-us"
 test_url = "https://www.microsoft.com/en-us/"
 
diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest
deleted file mode 100644
index e1ef76a43d..0000000000
--- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest
+++ /dev/null
@@ -1,9 +0,0 @@
-[
-  {
-    "filename": "mitm6-linux-firefox-expedia.zip",
-    "size": 9447080,
-    "algorithm": "sha512",
-    "digest": "c9665a107e90b9b57de7784230f71184356cde2beaf022f96c4629ebc43712d2214fb83131424faec25b95eee5c7b88ca26b28817230040bf98156e96776720f",
-    "visibility": "public"
-  }
-]
diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest
deleted file mode 100644
index a54b860243..0000000000
--- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest
+++ /dev/null
@@ -1,9 +0,0 @@
-[
-  {
-    "size": 4595339,
-    "visibility": "public",
-    "digest": "0cfd03b0cd8b4a33656da1a6e7651fd2ea584ae4182f46646f61d7e018f4b04223ed65574d884598328dd1d34c629294836375c16798e4cf3c78533accdf2a53",
-    "algorithm": "sha512",
-    "filename": "mitm6-linux-firefox-microsoft.zip"
-  }
-]
diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest
deleted file mode 100644
index 092958e287..0000000000
--- a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest
+++ /dev/null
@@ -1,9 +0,0 @@
-[
-  {
-    "size": 5078634,
-    "visibility": "public",
-    "digest": "315399f0773837ef13b9ba768a8a72a001ba103e8c09992ebef0421b6639beb5d48bb71bf47c29bfe8484bed19bfc4f9b97b9513f6593c9afb01f078a3a9b7f4",
-    "algorithm": "sha512",
-    "filename": "mitm7-linux-firefox-microsoft.zip"
-  }
-]
diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest
new file mode 100644
index 0000000000..03281ba612
--- /dev/null
+++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest
@@ -0,0 +1,9 @@
+[
+  {
+    "filename": "mitm8-darwin-firefox-expedia.zip",
+    "size": 15755960,
+    "algorithm": "sha512",
+    "digest": "239dc9438455417ca49a38278632efd72283f845dd421f8739449c5eb270418627507068bdd0c9363e3529cc93053999cda9cf2d5cfd509d5c937c561c9cdb84",
+    "visibility": "public"
+  }
+]
\ No newline at end of file
diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest
new file mode 100644
index 0000000000..0b627539fa
--- /dev/null
+++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest
@@ -0,0 +1,9 @@
+[
+  {
+    "filename": "mitm8-linux-firefox-microsoft.zip",
+    "size": 6177656,
+    "algorithm": "sha512",
+    "digest": "8b8cc69e0e51bcae9801f266b8c3e0eee18f569b243459767c476a4166875c328dec05d013c6a843bc34dd8a91c72ab3facdcdf41ba2c9cfb7f87886a3dd90a8",
+    "visibility": "public"
+  }
+]
\ No newline at end of file
diff --git a/testing/raptor/requirements.txt b/testing/raptor/requirements.txt
index 2532033cd4..c03db2f6ea 100644
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,6 +1,6 @@
 mozcrash ~= 2.0
-mozrunner ~= 7.0
-mozprofile ~= 2.1
+mozrunner ~= 8.3
+mozprofile ~= 3.0
 manifestparser >= 1.1
 ../tools/wptserve
 ../tools/wpt_third_party/h2
diff --git a/testing/raptor/source_requirements.txt b/testing/raptor/source_requirements.txt
index 9d13e74926..502f3fb2cc 100644
--- a/testing/raptor/source_requirements.txt
+++ b/testing/raptor/source_requirements.txt
@@ -1,8 +1,8 @@
 # These dependencies are required for running raptor tests from a source checkout
 
 mozcrash ~= 2.0
-mozrunner ~= 7.0
-mozprofile ~= 2.1
+mozrunner ~= 8.3
+mozprofile ~= 3.0
 manifestparser >= 1.1
 ../web-platform/tests/tools/wptserve
 ../web-platform/tests/tools/third_party/h2