diff options
Diffstat (limited to 'testing/raptor')
46 files changed, 177 insertions, 100 deletions
diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index 1ddf0a91d5..0de7b1a239 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -169,7 +169,6 @@ class Browsertime(Perftest): if self.browsertime_chromedriver and self.config["app"] in ( "chrome", "chrome-m", - "chromium", "custom-car", "cstm-car-m", ): @@ -462,7 +461,6 @@ class Browsertime(Perftest): priority1_options = self.browsertime_args if self.config["app"] in ( "chrome", - "chromium", "chrome-m", "custom-car", "cstm-car-m", @@ -522,7 +520,6 @@ class Browsertime(Perftest): ) if self.browsertime_no_ffwindowrecorder or self.config["app"] in ( - "chromium", "chrome-m", "chrome", "custom-car", diff --git a/testing/raptor/raptor/browsertime/desktop.py b/testing/raptor/raptor/browsertime/desktop.py index 1fb3bacc63..fd9a938f2e 100644 --- a/testing/raptor/raptor/browsertime/desktop.py +++ b/testing/raptor/raptor/browsertime/desktop.py @@ -25,7 +25,6 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime): if self.config["app"] in ( "chrome", - "chromium", "custom-car", ): return args_list + [ diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 3021f080ee..3a79e65341 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -7,10 +7,9 @@ import os import six from mozlog.commandline import add_logging_group -(FIREFOX, CHROME, CHROMIUM, SAFARI, CHROMIUM_RELEASE) = DESKTOP_APPS = [ +(FIREFOX, CHROME, SAFARI, CHROMIUM_RELEASE) = DESKTOP_APPS = [ "firefox", "chrome", - "chromium", "safari", "custom-car", ] @@ -25,11 +24,10 @@ from mozlog.commandline import add_logging_group ] FIREFOX_APPS = FIREFOX_ANDROID_APPS + [FIREFOX] -CHROMIUM_DISTROS = [CHROME, CHROMIUM] +CHROMIUM_DISTROS = [CHROME, CHROMIUM_RELEASE] APPS = { FIREFOX: {"long_name": "Firefox Desktop"}, CHROME: {"long_name": "Google Chrome Desktop"}, - CHROMIUM: {"long_name": "Google Chromium Desktop"}, SAFARI: {"long_name": "Safari Desktop"}, CHROMIUM_RELEASE: {"long_name": "Custom Chromium-as-Release desktop"}, GECKOVIEW: { @@ -62,7 +60,7 @@ INTEGRATED_APPS = list(APPS.keys()) GECKO_PROFILER_APPS = (FIREFOX, GECKOVIEW, REFBROW, FENIX) -TRACE_APPS = (CHROME, CHROMIUM, CHROMIUM_RELEASE) +TRACE_APPS = (CHROME, CHROMIUM_RELEASE) APP_BINARIES = { "fenix": "org.mozilla.fenix", diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py index 8483a3b8be..509f74afb4 100644 --- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -374,7 +374,6 @@ def get_raptor_test_list(args, oskey): if args.collect_perfstats and args.app.lower() not in ( "chrome", - "chromium", "custom-car", ): for next_test in tests_to_run: diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py index 3725e84831..4ddfcf75e8 100644 --- a/testing/raptor/raptor/output.py +++ b/testing/raptor/raptor/output.py @@ -1837,7 +1837,6 @@ class BrowsertimeOutput(PerftestOutput): if self.app in ( "chrome", "chrome-m", - "chromium", "custom-car", "cstm-car-m", ): @@ -1880,7 +1879,6 @@ class BrowsertimeOutput(PerftestOutput): if self.app in ( "chrome", "chrome-m", - "chromium", "custom-car", "cstm-car-m", ): diff --git a/testing/raptor/raptor/perfdocs/browsertime.rst b/testing/raptor/raptor/perfdocs/browsertime.rst index beff3f0ad8..3cde0baca6 100644 --- a/testing/raptor/raptor/perfdocs/browsertime.rst +++ b/testing/raptor/raptor/perfdocs/browsertime.rst @@ -180,6 +180,25 @@ trigger explicit linter errors, :doc:`see condprof-addons linter docs </code-qua .. _webext.json: https://searchfox.org/mozilla-central/rev/bc6a50e6f08db0bb371ef7197c472555499e82c0/testing/condprofile/condprof/customization/webext.json .. _firefox-addons: https://searchfox.org/mozilla-central/rev/bc6a50e6f08db0bb371ef7197c472555499e82c0/taskcluster/ci/fetch/browsertime.yml#169-176 +TP6-Bench Test +-------------- + +The ``tp6-bench`` can either be run on desktop, or locally using the following command: + +:: + + ./mach raptor -t tp6-bench + +This test runs through a selection of our tp6 pageload tests as quickly as possible while only measuring loadtime, FCP (First Contentful Paint), and LCP (Largest Contentful Paint). It does not measure visual metrics like our standard tp6 pageload tests do. The test produces 3 overall metrics (fcp, lcp, and loadtime) across all the pages tested that are summarized in two ways: geomean, and mean. The subtests for each of those metrics are the individual metrics produced in each of the pages tested. + +To run this test on try and get a compare view link, run the following + +:: + + ./mach try perf -q "'Pageload 'Lite" + + + More Examples ------------- diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index a9ab330feb..723da06287 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -32,6 +32,7 @@ suites: bing-search: "" buzzfeed: "" cnn: "" + docomo: "" ebay: "" espn: "" expedia: "" @@ -49,17 +50,22 @@ suites: netflix: "" nytimes: "" office: "" + openai: "" outlook: "" paypal: "" pinterest: "" reddit: "" + samsung: "" + tiktok: "" tumblr: "" twitch: "" twitter: "" + weather: "" wikia: "" wikipedia: "" yahoo-mail: "" youtube: "" + mobile: description: "Page-load performance test suite on Android. The links direct to the actual websites that are being tested." tests: @@ -181,7 +187,10 @@ suites: throttled: "Pageload test using a throttled network environment" sample-python-support: "A sample test that uses a python support file to modify the test command." connect: "Verifies that connection time on a pageload is zero when a speculative connection has first been made" - tp6-bench: "A pageload benchmark test that quickly runs through all of the pages available to produce a single overall metric." + tp6-bench: > + A pageload benchmark test that quickly runs through a selection of the + pages available to produce a single overall metric for fcp, lcp, and + loadtime. The metrics from all pages get summarized with the geomean, and mean. interactive: description: "Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest." tests: diff --git a/testing/raptor/raptor/perfdocs/webextension.rst b/testing/raptor/raptor/perfdocs/webextension.rst index e1883755a4..b688c22f32 100644 --- a/testing/raptor/raptor/perfdocs/webextension.rst +++ b/testing/raptor/raptor/perfdocs/webextension.rst @@ -21,7 +21,7 @@ Warm Page-Load ============== For warm page-load tests, the browser is just started up once; so the browser is warm on each page-load. -**Raptor warm page-load test process when running on Firefox/Chrome/Chromium desktop:** +**Raptor warm page-load test process when running on Firefox/Chrome desktop:** * A new browser profile is created * The desktop browser is started up @@ -45,7 +45,7 @@ Cold Page-Load ============== For cold page-load tests, the browser is shut down and restarted between page load cycles, so the browser is cold on each page-load. This is what happens for Raptor cold page-load tests: -**Raptor cold page-load test process when running on Firefox/Chrome/Chromium desktop:** +**Raptor cold page-load test process when running on Firefox/Chrome desktop:** * A new browser profile is created * The desktop browser is started up diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index 6e21b6d114..ec2fa3625e 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -181,7 +181,6 @@ class Perftest(object): if self.config["app"] in ( "chrome", "chrome-m", - "chromium", "custom-car", "cstm-car-m", ): @@ -811,8 +810,8 @@ class PerftestDesktop(Perftest): def desktop_chrome_args(self, test): """Returns cmd line options required to run pageload tests on Desktop Chrome - and Chromium. Also add the cmd line options to turn on the proxy and - ignore security certificate errors if using host localhost, 127.0.0.1. + and Chromium as Release (CaR). Also add the cmd line options to turn on the + proxy and ignore security certificate errors if using host localhost, 127.0.0.1. """ chrome_args = ["--use-mock-keychain", "--no-default-browser-check"] diff --git a/testing/raptor/raptor/raptor.toml b/testing/raptor/raptor/raptor.toml index 19277e8cff..9cb9d0ff00 100644 --- a/testing/raptor/raptor/raptor.toml +++ b/testing/raptor/raptor/raptor.toml @@ -100,6 +100,9 @@ ["include:tests/tp6/mobile/browsertime-tp6m.toml"] # raptor-browsertime android page-load tests +["include:tests/tp7/desktop/browsertime-tp7.toml"] +# raptor-browsertime desktop page-load tests + ["include:tests/unittests/browsertime-tp6-unittest.toml"] # raptor-browsertime unit tests # this is required for the manifest unit tests diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py index fdb1d9d7a8..48ddfda25d 100644 --- a/testing/raptor/raptor/results.py +++ b/testing/raptor/raptor/results.py @@ -30,7 +30,7 @@ KNOWN_TEST_MODIFIERS = [ "bytecode-cached", ] NON_FIREFOX_OPTS = ("webrender", "bytecode-cached", "fission") -NON_FIREFOX_BROWSERS = ("chrome", "chromium", "custom-car", "safari") +NON_FIREFOX_BROWSERS = ("chrome", "custom-car", "safari") NON_FIREFOX_BROWSERS_MOBILE = ("chrome-m", "cstm-car-m") diff --git a/testing/raptor/raptor/tests/benchmarks/ares6.toml b/testing/raptor/raptor/tests/benchmarks/ares6.toml index 40e6d30e10..617f89653f 100644 --- a/testing/raptor/raptor/tests/benchmarks/ares6.toml +++ b/testing/raptor/raptor/tests/benchmarks/ares6.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/assorted-dom.toml b/testing/raptor/raptor/tests/benchmarks/assorted-dom.toml index 30db96ffd9..6d312be699 100644 --- a/testing/raptor/raptor/tests/benchmarks/assorted-dom.toml +++ b/testing/raptor/raptor/tests/benchmarks/assorted-dom.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/jetstream2.toml b/testing/raptor/raptor/tests/benchmarks/jetstream2.toml index 5d5ef58da5..34759b28ef 100644 --- a/testing/raptor/raptor/tests/benchmarks/jetstream2.toml +++ b/testing/raptor/raptor/tests/benchmarks/jetstream2.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.toml b/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.toml index cef23f17f6..d1a47d054f 100644 --- a/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.toml +++ b/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium" +apps = "firefox, chrome" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml index 3c471d3a17..a480054f7a 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari, custom-car" +apps = "firefox, chrome, safari, custom-car" gecko_profile_interval = 1 gecko_profile_features = "stackwalk,js,cpu,java,processcpu" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml index 98a755a52e..78ffae7811 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 gecko_profile_features = "stackwalk,js,cpu,java,processcpu" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" @@ -16,5 +16,5 @@ unit = "score" ["motionmark-animometer"] ["motionmark-animometer-ramp"] -apps = "firefox, chrome, chromium, safari, custom-car" +apps = "firefox, chrome, safari, custom-car" test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}" diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml index c374560404..f62f1d993a 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 gecko_profile_features = "stackwalk,js,cpu,java,processcpu" gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" @@ -17,5 +17,5 @@ unit = "score" ["motionmark-htmlsuite"] ["motionmark-htmlsuite-ramp"] -apps = "firefox, chrome, chromium, safari, custom-car" +apps = "firefox, chrome, safari, custom-car" test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}" diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml index d3bc83e420..f304493c20 100644 --- a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari, custom-car" +apps = "firefox, chrome, safari, custom-car" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/stylebench.toml b/testing/raptor/raptor/tests/benchmarks/stylebench.toml index a759326e93..b1c4b0e32e 100644 --- a/testing/raptor/raptor/tests/benchmarks/stylebench.toml +++ b/testing/raptor/raptor/tests/benchmarks/stylebench.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/sunspider.toml b/testing/raptor/raptor/tests/benchmarks/sunspider.toml index 316d81b86c..33cff92f7d 100644 --- a/testing/raptor/raptor/tests/benchmarks/sunspider.toml +++ b/testing/raptor/raptor/tests/benchmarks/sunspider.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.toml b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.toml index 6f663703b7..bfeec3971d 100644 --- a/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-godot.toml b/testing/raptor/raptor/tests/benchmarks/wasm-godot.toml index 1fe2b82c65..4cd9926bf9 100644 --- a/testing/raptor/raptor/tests/benchmarks/wasm-godot.toml +++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-misc.toml b/testing/raptor/raptor/tests/benchmarks/wasm-misc.toml index facc2946c8..86ed3b9b82 100644 --- a/testing/raptor/raptor/tests/benchmarks/wasm-misc.toml +++ b/testing/raptor/raptor/tests/benchmarks/wasm-misc.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium" +apps = "firefox, chrome" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/webaudio.toml b/testing/raptor/raptor/tests/benchmarks/webaudio.toml index df8f7c555c..0bf555241c 100644 --- a/testing/raptor/raptor/tests/benchmarks/webaudio.toml +++ b/testing/raptor/raptor/tests/benchmarks/webaudio.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = true diff --git a/testing/raptor/raptor/tests/custom/browsertime-grandprix.toml b/testing/raptor/raptor/tests/custom/browsertime-grandprix.toml index 6c3d0b69d5..d54d434266 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-grandprix.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-grandprix.toml @@ -1,5 +1,5 @@ [DEFAULT] -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" alert_threshold = 2.0 browser_cycles = 1 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml index 745c1b1823..4e510b3d12 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml @@ -1,5 +1,5 @@ [DEFAULT] -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" alert_threshold = 2.0 browser_cycles = 1 custom_data = true diff --git a/testing/raptor/raptor/tests/custom/browsertime-process-switch.toml b/testing/raptor/raptor/tests/custom/browsertime-process-switch.toml index 610003a127..b4d8642e0a 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-process-switch.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-process-switch.toml @@ -1,7 +1,7 @@ [DEFAULT] alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange" alert_threshold = 2.0 -apps = "firefox, chrome, chromium" +apps = "firefox, chrome" browser_cycles = 25 gecko_profile_interval = 1 lower_is_better = true diff --git a/testing/raptor/raptor/tests/custom/browsertime-regression-test.toml b/testing/raptor/raptor/tests/custom/browsertime-regression-test.toml index 814d7bb0f6..7ae6bfc8fd 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-regression-test.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-regression-test.toml @@ -1,5 +1,5 @@ [DEFAULT] -apps = "firefox, chrome, chromium, safari, geckoview, fenix" +apps = "firefox, chrome, safari, geckoview, fenix" alert_threshold = 2.0 browser_cycles = 1 fore_window = 1 diff --git a/testing/raptor/raptor/tests/custom/browsertime-upload.toml b/testing/raptor/raptor/tests/custom/browsertime-upload.toml index 5e297d0053..2dcd1d20bb 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-upload.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-upload.toml @@ -1,5 +1,5 @@ [DEFAULT] -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" alert_threshold = 2.0 browser_cycles = 1 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.toml b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.toml index 2194f96c8e..0502fd577d 100644 --- a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.toml +++ b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.toml @@ -2,7 +2,7 @@ accept_zero_vismet = true alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange" alert_threshold = 2.0 -apps = "firefox, chrome, chromium, safari" +apps = "firefox, chrome, safari" browser_cycles = 25 gecko_profile_interval = 1 interactive = true diff --git a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml index 8c0552fb0c..9091ec05ce 100644 --- a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml +++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml @@ -1,7 +1,7 @@ [DEFAULT] alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint" alert_threshold = "2.0" -apps = "firefox, chrome, chromium, safari, custom-car" +apps = "firefox, chrome, safari, custom-car" browser_cycles = 25 gecko_profile_interval = 1 lower_is_better = true @@ -19,6 +19,7 @@ benchmark_page = true # Keep this list in alphabetical order ["amazon"] +playback_pageset_manifest = "mitm8-linux-firefox-amazon.manifest" benchmark_page = false test_url = "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1" secondary_url = "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3" @@ -97,7 +98,7 @@ secondary_url = "https://imgur.com/gallery/WuOobqV" test_url = "https://imgur.com/gallery/uTI7fKy" ["instagram"] -playback_pageset_manifest = "mitm6-linux-firefox-instagram.manifest" +playback_pageset_manifest = "mitm8-darwin-firefox-instagram.manifest" secondary_url = "https://www.instagram.com/nobelprize_org/" test_url = "https://www.instagram.com/" diff --git a/testing/raptor/raptor/tests/tp6/live/browsertime-live.toml b/testing/raptor/raptor/tests/tp6/live/browsertime-live.toml index ac0613784e..9ac0f53030 100644 --- a/testing/raptor/raptor/tests/tp6/live/browsertime-live.toml +++ b/testing/raptor/raptor/tests/tp6/live/browsertime-live.toml @@ -1,7 +1,7 @@ [DEFAULT] alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange" alert_threshold = 2.0 -apps = "geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium" +apps = "geckoview, fenix, refbrow, chrome-m, firefox, chrome" browser_cycles = 15 lower_is_better = true owner = "PerfTest Team" diff --git a/testing/raptor/raptor/tests/tp7/desktop/browsertime-tp7.toml b/testing/raptor/raptor/tests/tp7/desktop/browsertime-tp7.toml new file mode 100644 index 0000000000..58a2da36dd --- /dev/null +++ b/testing/raptor/raptor/tests/tp7/desktop/browsertime-tp7.toml @@ -0,0 +1,42 @@ +[DEFAULT] +alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint" +alert_threshold = "2.0" +apps = "firefox, chrome" +browser_cycles = 25 +gecko_profile_interval = 1 +lower_is_better = true +owner = "PerfTest Team" +page_cycles = 25 +page_timeout = 60000 +playback = "mitmproxy" +playback_pageset_manifest = "mitm8-linux-firefox-{subtest}.manifest" +playback_version = "8.1.1" +type = "pageload" +unit = "ms" +use_live_sites = false +benchmark_page = false + +# Keep this list in alphabetical order +# TODO: maybe break up into categories + +[docomo] +test_url = "https://www.docomo.ne.jp/" +secondary_url = "https://www.docomo.ne.jp/iphone/?icid=CRP_menu_to_CRP_IPH" + +[openai] +# Bug 1890563, setup logins for tiktok and openai +test_url = "https://openai.com/" +secondary_url = "https://openai.com/blog" + +[samsung] +test_url = "https://www.samsung.com/us" +secondary_url = "https://www.samsung.com/us/mobile" + +[tiktok] +# Bug 1890563, setup logins for tiktok and openai +test_url = "https://www.tiktok.com/en" +secondary_url = "https://www.tiktok.com/explore" + +[weather] +test_url = "https://weather.com/en-CA/weather/today/l/43.67%2C-79.42" +secondary_url = "https://weather.com/en-CA/weather/monthly/l/43.67%2C-79.42" diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest deleted file mode 100644 index fd640e26e1..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm5-linux-firefox-wikia.zip", - "size": 4847101, - "algorithm": "sha512", - "digest": "c53c45c374cd924d03972f56b6a7416451861147f0a14b8c51c2c722a82582ae7ce8927a500981152aa6f4d3617b5ce455b85910981a9aabbd672e1ea47c0ff7", - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest deleted file mode 100644 index f992a624c0..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "size": 6548847, - "visibility": "public", - "digest": "30d26a7bd8343255bf5e9e58f9315a56719be25dcb32214de1604563a9fbd49e17e477286a9d9c4b941ba20591385ac523c153e6f290ef57f679bd9261fa9078", - "algorithm": "sha512", - "filename": "mitm6-linux-firefox-instagram.zip" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest deleted file mode 100644 index 5af2865fc4..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "size": 20477615, - "visibility": "public", - "digest": "4f588d80825cd570a702d060a8e75443f5ba564f33a37fa75179829308b7258c7989cd13966d09e3258e87ca7da18b138041d0723d28f9f0397c72bdac559bf4", - "algorithm": "sha512", - "filename": "mitm7-linux-firefox-amazon.zip" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-instagram.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-instagram.manifest new file mode 100644 index 0000000000..fc9c5451bc --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-darwin-firefox-instagram.manifest @@ -0,0 +1,10 @@ +[ + { + "filename": "mitm8-darwin-firefox-instagram.zip", + "size": 37433674, + "algorithm": "sha512", + "digest": "677826a8d755441e374ab2fa1ef4229add5a60b7c97f1ceb547fd215f366e2dd885d6b3aa46d32664125562add33db48ecaba1e5cd4aec9164959a6ba25e2e2f", + "visibility": "public" + } +] + diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-amazon.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-amazon.manifest new file mode 100644 index 0000000000..4691bddb23 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-amazon.manifest @@ -0,0 +1,10 @@ +[ + { + "filename": "mitm8-linux-firefox-amazon.zip", + "size": 32863692, + "algorithm": "sha512", + "digest": "c8e98e1d294fa58faaab499ccd904e7b8f98ffac7f3e8a835103eb41f54d98fad903175ae43dd1ddbabdfdefd57b1652436b287825f8474eed902b938f3dacd5", + "visibility": "public" + } +] + diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-docomo.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-docomo.manifest new file mode 100644 index 0000000000..6f607e987d --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-docomo.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-docomo.zip", + "size": 15093337, + "algorithm": "sha512", + "digest": "36ed3d113f0011bf071a7e16d58a10864f46379701e6734785456bbc9a328299c08464224e4e615b8bcb0318c76ed634b0f690ce05ee71332af1c086c4459bf5", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-imgur.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-imgur.manifest index 042460e7dd..b3d2a132f2 100644 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-imgur.manifest +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-imgur.manifest @@ -1,9 +1,10 @@ [ { "filename": "mitm8-linux-firefox-imgur.zip", - "size": 11323543, + "size": 13808645, "algorithm": "sha512", - "digest": "5817f3ec63d72658e0b34cc2d50069879ceee80aeaff715ab1370253d8f0ac60e7e58f284ec7b3bd50f562beeeda7d0c1a0cbdf9fd5c960cb8ae1230dcbd08e9", + "digest": "35f2bd5b9b89f0912bb8a1577c1beabb76713cbb5e8cff060529d407ebb4818bf791c4e62e3a9bb4c2e299ec0e60a1ab207e1bdc8316538e5bfc66ca2b8a4aab", "visibility": "public" } ] + diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-openai.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-openai.manifest new file mode 100644 index 0000000000..88bc62bf5b --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-openai.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-openai.zip", + "size": 26770625, + "algorithm": "sha512", + "digest": "d5f3141f9d078b77dc7d258430a922bb8c48a15d84e01cc03edc0bb740445de78430e1c47c8ae88d370b7c24084f2bfbf8b6a0de36393efaca04e16acb1386fa", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-samsung.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-samsung.manifest new file mode 100644 index 0000000000..001fafe900 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-samsung.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-samsung.zip", + "size": 8407529, + "algorithm": "sha512", + "digest": "5f63e190084b3ed65abcee18a60ab801f43b5c1aecc4c96c16a2b18a0619c39b211cbc5ef5fb842162c85878d877cf053d21e84a1283dff2035127a75ce3a4a1", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-tiktok.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-tiktok.manifest new file mode 100644 index 0000000000..bb8e1e305f --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-tiktok.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-tiktok.zip", + "size": 63432854, + "algorithm": "sha512", + "digest": "4ab6393be937f33175d4cae9c70f54ba7e42b34ce4a63a0fa8f4650c33a527acb2064781ff3410beca1a37555f69d52d8f2591fa557afa9e5c9b503908afed8a", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-weather.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-weather.manifest new file mode 100644 index 0000000000..10cfe3db9c --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-linux-firefox-weather.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-linux-firefox-weather.zip", + "size": 14934410, + "algorithm": "sha512", + "digest": "dc9c0ceec902518254592845a698f2346c525ee9e111c784278ec8aaea68d8928d9a6788bad3034badbd7a49a0de2b3608284726d0fc0246715ce1b15f390cf5", + "visibility": "public" + } +]
\ No newline at end of file diff --git a/testing/raptor/test/test_manifest.py b/testing/raptor/test/test_manifest.py index 3b066953e1..ed2ff3a563 100644 --- a/testing/raptor/test/test_manifest.py +++ b/testing/raptor/test/test_manifest.py @@ -76,18 +76,6 @@ VALID_MANIFESTS = [ "type": "benchmark", "unit": "score", }, - { - # benchmark test for chromium - "alert_threshold": 2.0, - "apps": "chromium", - "lower_is_better": False, - "manifest": "valid_details_1", - "measure": "fcp", - "page_cycles": 5, - "test_url": "http://www.test-url/goes/here", - "type": "benchmark", - "unit": "score", - }, ] INVALID_MANIFESTS = [ @@ -116,18 +104,6 @@ INVALID_MANIFESTS = [ "unit": "ms", }, { - "alert_threshold": 2.0, - "apps": "chromium", - "lower_is_better": True, - "manifest": "invalid_details_1", - "measure": "fnbpaint, fcp", - "page_cycles": 25, - "playback": "mitmproxy", - "test_url": "http://www.test-url/goes/here", - "type": "pageload", - "unit": "ms", - }, - { "alert_on": "nope", "alert_threshold": 2.0, "apps": "firefox", @@ -144,9 +120,7 @@ INVALID_MANIFESTS = [ ] -@pytest.mark.parametrize( - "app", ["firefox", "chrome", "chromium", "geckoview", "refbrow", "fenix"] -) +@pytest.mark.parametrize("app", ["firefox", "chrome", "geckoview", "refbrow", "fenix"]) def test_get_browser_test_list(app): test_list = get_browser_test_list(app, run_local=True) assert len(test_list) > 0 |