diff options
Diffstat (limited to 'testing/raptor')
23 files changed, 104 insertions, 62 deletions
diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index 0de7b1a239..f06763e2ca 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -186,7 +186,7 @@ class Browsertime(Perftest): # setup once all chrome versions use the new artifact setup. cd_extracted_names_115 = { "windows": str( - pathlib.Path("{}chromedriver-win32", "chromedriver.exe") + pathlib.Path("{}chromedriver-win64", "chromedriver.exe") ), "mac-x86_64": str( pathlib.Path("{}chromedriver-mac-x64", "chromedriver") @@ -212,7 +212,8 @@ class Browsertime(Perftest): elif "win" in self.config["platform"]: self.browsertime_chromedriver = ( self.browsertime_chromedriver.replace( - "{}chromedriver.exe", cd_extracted_names_115["windows"] + "{}chromedriver.exe", + cd_extracted_names_115["windows"], ) ) else: @@ -598,13 +599,19 @@ class Browsertime(Perftest): browsertime_options=browsertime_options, test=test ) - return ( + cmd = ( [self.browsertime_node, self.browsertime_browsertimejs] + self.driver_paths + [browsertime_script] + browsertime_options ) + if test.get("support_class", None): + LOG.info("Test support class is modifying the command...") + test.get("support_class").modify_command(cmd, test) + + return cmd + def _compose_gecko_profiler_cmds(self, test, priority1_options): """Modify the command line options for running the gecko profiler in the Firefox application""" @@ -623,7 +630,7 @@ class Browsertime(Perftest): ( "gecko_profile_features", "--firefox.geckoProfilerParams.features", - "js,stackwalk,cpu,screenshots", + "js,stackwalk,cpu,screenshots,memory", ), ( "gecko_profile_threads", @@ -916,10 +923,6 @@ class Browsertime(Perftest): # this will be used for btime --timeouts.pageLoad cmd = self._compose_cmd(test, timeout) - if test.get("support_class", None): - LOG.info("Test support class is modifying the command...") - test.get("support_class").modify_command(cmd, test) - output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT if test.get("type", "") == "scenario": # Change the timeout for scenarios since they diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 3a79e65341..5542fdfdd8 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -198,7 +198,7 @@ def create_parser(mach_interface=False): add_arg( "--gecko-profile-interval", dest="gecko_profile_interval", - type=int, + type=float, help="How frequently to take samples (milliseconds)", ) add_arg( diff --git a/testing/raptor/raptor/filters.py b/testing/raptor/raptor/filters.py index 87b0d827e6..08a3241621 100644 --- a/testing/raptor/raptor/filters.py +++ b/testing/raptor/raptor/filters.py @@ -4,7 +4,6 @@ # originally taken from /testing/talos/talos/filter.py -import math """ data filters: @@ -175,11 +174,10 @@ def geometric_mean(series): """ geometric_mean: http://en.wikipedia.org/wiki/Geometric_mean """ - total = 0 - for i in series: - total += math.log(i + 1) - # pylint --py3k W1619 - return math.exp(total / len(series)) - 1 + + from scipy.stats.mstats import gmean + + return gmean(series) # filters that return a list diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py index 509f74afb4..cae7af7c26 100644 --- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -275,7 +275,7 @@ def write_test_settings_json(args, test_details, oskey): "gecko_profile_entries": int( test_details.get("gecko_profile_entries", 1000000) ), - "gecko_profile_interval": int( + "gecko_profile_interval": float( test_details.get("gecko_profile_interval", 1) ), "gecko_profile_threads": ",".join(set(threads)), diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index 723da06287..b4d61bd10a 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -12,7 +12,7 @@ metrics: - First Contentful Paint - First Contentful Composite description: > - Denotes the first time the browser performs a paint that has content in it. + Denotes the first time the browser performs a paint that has content in it (in ms). "Youtube Playback Metrics": aliases: - VP9 @@ -20,10 +20,51 @@ metrics: matcher: VP9.*|H264.* description: > Metrics starting with VP9/H264 give the number of frames dropped, and painted. - "In Progress": - aliases: [] - matcher: .* - description: Many metrics have not been documented yet, this is a placeholder. + "First Visual Change": + aliases: + - FirstVisualChange + description: The first visual change detected in the test (in ms). + "Last Visual Change": + aliases: + - LastVisualChange + description: The last visual change detected in the test (in ms). + "Load Time": + aliases: + - loadtime + description: The time it took for the page to complete loading (in ms). + "Largest Contentful Paint": + aliases: + - largestContentfulPaint + - lcp + description: > + The time (in ms) at which the largest piece of content on the page was rendered/painted. + "Speed Index": + aliases: + - SpeedIndex + description: > + A metric used to denote the speed at which a page loaded. Lower values indicate faster + pageloads. Units are in (Visually-Complete x Milliseconds). Calculated by using videos + of the pageload which provide a measure of visual completeness. Visual completeness is + calculated by comparing the histogram of a given frame to the final frame of the pageload. + The SpeedIndex is calculated as the area between the curves of a constant line at y=1, + and the graph of the visual completeness from 0ms to when visual completeness reaches + 100% (or hits the y=1 line). + "Contentful Speed Index": + aliases: + - ContentfulSpeedIndex + description: > + Similar to SpeedIndex, except that it uses the contentfulness of a frame to determine + visual completeness instead of the histogram differences. The contentfulness is determined + by calculating the number of edges found in the frame. A lower number of edges detected + gives a smaller contentfulness value (and a smaller visually complete value). + "Perceptual Speed Index": + aliases: + - PerceptualSpeedIndex + description: > + Similar to SpeedIndex, except that it uses the structural similarity index measure (ssim) + to determine visual completeness. This technique compares the luminance, contrast, and + structure of the frames (a given frame vs. a final frame) to determine the completeness. + suites: desktop: description: "Tests for page-load performance. The links direct to the actual websites that are being tested." diff --git a/testing/raptor/raptor/perfdocs/debugging.rst b/testing/raptor/raptor/perfdocs/debugging.rst index 8cc799e820..87165765f8 100644 --- a/testing/raptor/raptor/perfdocs/debugging.rst +++ b/testing/raptor/raptor/perfdocs/debugging.rst @@ -14,9 +14,9 @@ As of now, there is no easy way to do this. Raptor was not built for debugging f To debug a functional failure in Raptor you can follow these steps: #. If bug 1653617 has not landed yet, apply the patch. -#. Add the --verbose flag to the extra-options list `here <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml#98-101>`__. +#. Add the --verbose flag to the extra-options list `here <https://searchfox.org/mozilla-central/source/taskcluster/kinds/test/raptor.yml#98-101>`__. #. If the --setenv doesn't exist yet (`bug 1494669 <https://bugzilla.mozilla.org/show_bug.cgi?id=1494669>`_), then add your MOZ_LOG environment variables to give you additional logging `here <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/webextension/desktop.py#42>`_. -#. If the flag does exist, then you can add the MOZ_LOG variables to the `raptor.yml <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml>`_ configuration file. +#. If the flag does exist, then you can add the MOZ_LOG variables to the `raptor.yml <https://searchfox.org/mozilla-central/source/taskcluster/kinds/test/raptor.yml>`_ configuration file. #. Push to try if you can't reproduce the failure locally. You can follow `bug 1655554 <https://bugzilla.mozilla.org/show_bug.cgi?id=1655554>`_ as we work on improving this workflow. diff --git a/testing/raptor/raptor/perfdocs/index.rst b/testing/raptor/raptor/perfdocs/index.rst index 88c1702c00..082fa05940 100644 --- a/testing/raptor/raptor/perfdocs/index.rst +++ b/testing/raptor/raptor/perfdocs/index.rst @@ -23,7 +23,7 @@ Locally, Raptor can be invoked with the following command: browsertime debugging contributing - metrics + {metrics_rst_name} .. contents:: :depth: 2 diff --git a/testing/raptor/raptor/perfdocs/metrics.rst b/testing/raptor/raptor/perfdocs/metrics.rst index 931456467c..106f62b750 100644 --- a/testing/raptor/raptor/perfdocs/metrics.rst +++ b/testing/raptor/raptor/perfdocs/metrics.rst @@ -80,3 +80,10 @@ Benchmarks ---------- Benchmarks gather their own custom metrics unlike the pageload tests above. Please ping the owners of those benchmarks to determine what they mean and how they are produced, or reach out to the Performance Test and Tooling team in #perftest on Element. + +Metric Definitions +------------------ + +The following documents all available metrics that current alert in Raptor Browsertime tests. + +{metrics_documentation} diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml index a480054f7a..2001cd3f7c 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "firefox, chrome, safari, custom-car" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_browser_profiler = true expose_chrome_trace = true diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml index f68f418608..46470c4535 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "fenix, geckoview, chrome-m, cstm-car-m" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml index 78ffae7811..c82c197355 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "firefox, chrome, safari" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml index 4c178dc47f..6899d0ff42 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "fenix, chrome-m, geckoview" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml index f62f1d993a..b52e97339d 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "firefox, chrome, safari" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_gecko_profiler = true expose_chrome_trace = true diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml index a617735305..d02dc5b0f5 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml @@ -2,7 +2,7 @@ alert_threshold = 2.0 apps = "fenix, chrome-m, geckoview" gecko_profile_interval = 1 -gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_features = "stackwalk,js,cpu,java,processcpu,memory" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" expose_gecko_profiler = true expose_chrome_trace = true diff --git a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml index 4e510b3d12..83262e5f24 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml @@ -4,7 +4,7 @@ alert_threshold = 2.0 browser_cycles = 1 custom_data = true gecko_profile_entries = 131072000 # 1GB -gecko_profile_features = "js,stackwalk,cpu" +gecko_profile_features = "js,stackwalk,cpu,memory" gecko_profile_threads = "GeckoMain,DOM Worker,IndexedDB" lower_is_better = true measure = "cpuTime" diff --git a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml index 9091ec05ce..9bddb0c0b9 100644 --- a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml +++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml @@ -48,6 +48,7 @@ test_url = "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comp secondary_url = "https://www.espn.com/nba/draft/news" ["expedia"] +playback_pageset_manifest = "mitm8-darwin-firefox-expedia.manifest" secondary_url = "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1" test_url = "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2" @@ -108,6 +109,7 @@ secondary_url = "https://www.linkedin.com/in/thommy-harris-hk-385723106/" test_url = "https://www.linkedin.com/feed/" ["microsoft"] +playback_pageset_manifest = "mitm8-linux-firefox-microsoft.manifest" secondary_url = "https://support.microsoft.com/en-us" test_url = "https://www.microsoft.com/en-us/" diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest deleted file mode 100644 index e1ef76a43d..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm6-linux-firefox-expedia.zip", - "size": 9447080, - "algorithm": "sha512", - "digest": "c9665a107e90b9b57de7784230f71184356cde2beaf022f96c4629ebc43712d2214fb83131424faec25b95eee5c7b88ca26b28817230040bf98156e96776720f", - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest deleted file mode 100644 index a54b860243..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "size": 4595339, - "visibility": "public", - "digest": "0cfd03b0cd8b4a33656da1a6e7651fd2ea584ae4182f46646f61d7e018f4b04223ed65574d884598328dd1d34c629294836375c16798e4cf3c78533accdf2a53", - "algorithm": "sha512", - "filename": "mitm6-linux-firefox-microsoft.zip" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest deleted file mode 100644 index 092958e287..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "size": 5078634, - "visibility": "public", - "digest": "315399f0773837ef13b9ba768a8a72a001ba103e8c09992ebef0421b6639beb5d48bb71bf47c29bfe8484bed19bfc4f9b97b9513f6593c9afb01f078a3a9b7f4", - "algorithm": "sha512", - "filename": "mitm7-linux-firefox-microsoft.zip" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest new file mode 100644 index 0000000000..03281ba612 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-expedia.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-darwin-firefox-expedia.zip", + "size": 15755960, + "algorithm": "sha512", + "digest": "239dc9438455417ca49a38278632efd72283f845dd421f8739449c5eb270418627507068bdd0c9363e3529cc93053999cda9cf2d5cfd509d5c937c561c9cdb84", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest new file mode 100644 index 0000000000..0b627539fa --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-microsoft.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-microsoft.zip", + "size": 6177656, + "algorithm": "sha512", + "digest": "8b8cc69e0e51bcae9801f266b8c3e0eee18f569b243459767c476a4166875c328dec05d013c6a843bc34dd8a91c72ab3facdcdf41ba2c9cfb7f87886a3dd90a8", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/requirements.txt b/testing/raptor/requirements.txt index 2532033cd4..c03db2f6ea 100644 --- a/testing/raptor/requirements.txt +++ b/testing/raptor/requirements.txt @@ -1,6 +1,6 @@ mozcrash ~= 2.0 -mozrunner ~= 7.0 -mozprofile ~= 2.1 +mozrunner ~= 8.3 +mozprofile ~= 3.0 manifestparser >= 1.1 ../tools/wptserve ../tools/wpt_third_party/h2 diff --git a/testing/raptor/source_requirements.txt b/testing/raptor/source_requirements.txt index 9d13e74926..502f3fb2cc 100644 --- a/testing/raptor/source_requirements.txt +++ b/testing/raptor/source_requirements.txt @@ -1,8 +1,8 @@ # These dependencies are required for running raptor tests from a source checkout mozcrash ~= 2.0 -mozrunner ~= 7.0 -mozprofile ~= 2.1 +mozrunner ~= 8.3 +mozprofile ~= 3.0 manifestparser >= 1.1 ../web-platform/tests/tools/wptserve ../web-platform/tests/tools/third_party/h2 |