summaryrefslogtreecommitdiffstats
path: root/taskcluster/ci/test/browsertime-desktop.yml
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--taskcluster/ci/test/browsertime-desktop.yml721
1 files changed, 721 insertions, 0 deletions
diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml
new file mode 100644
index 0000000000..b15fe1ae36
--- /dev/null
+++ b/taskcluster/ci/test/browsertime-desktop.yml
@@ -0,0 +1,721 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+ max-run-time:
+ by-test-platform:
+ .*-qr/.*: 2400
+ default: 1800
+ suite: raptor
+ run-on-projects:
+ by-test-platform:
+ linux.*shippable[^-qr].*: [mozilla-central]
+ default: []
+ limit-platforms:
+ by-app:
+ chrome:
+ - .*shippable.*
+ chromium:
+ - .*shippable.*
+ safari:
+ - .*macos.*shippable.*
+ custom-car:
+ - linux.*shippable.*
+ - windows10-64.*shippable.*
+ default: []
+ test-manifest-loader: null # don't load tests in the taskgraph
+ tier: 3
+ virtualization: hardware
+ mozharness:
+ script: raptor_script.py
+ config:
+ by-test-platform:
+ macosx.*:
+ - raptor/mac_config.py
+ windows.*:
+ - raptor/windows_config.py
+ default:
+ - raptor/linux_config.py
+ extra-options:
+ - --conditioned-profile=settled
+ fetches:
+ toolchain:
+ by-app:
+ custom-car:
+ by-test-platform:
+ # this satisfied taskgraph for now, but once mac/win is implemented
+ # then linux will be the default
+ linux.*:
+ - linux64-custom-car
+ - linux64-minidump-stackwalk
+ - linux64-fix-stacks
+ windows10-64.*:
+ - win64-custom-car
+ - win64-minidump-stackwalk
+ - win64-fix-stacks
+ default: []
+ default:
+ by-test-platform:
+ windows.*-32.*.:
+ - win32-minidump-stackwalk
+ - win32-fix-stacks
+ windows10-64.*:
+ - win64-minidump-stackwalk
+ - win64-fix-stacks
+ macos.*:
+ - macosx64-minidump-stackwalk
+ - macosx64-fix-stacks
+ default:
+ - linux64-minidump-stackwalk
+ - linux64-fix-stacks
+
+browsertime-tp6:
+ <<: &tp6-defaults
+ description: Raptor (browsertime) tp6 page-load tests
+ limit-platforms:
+ by-app:
+ chrome:
+ - .*shippable.*
+ chromium:
+ - .*shippable.*
+ safari:
+ - .*macos.*shippable.*
+ custom-car:
+ - (windows10-64|linux).*shippable.*
+ default: []
+ max-run-time:
+ by-subtest:
+ imgur: 4800
+ default: 4000
+ mozharness:
+ extra-options:
+ by-test-platform:
+ windows10-64.*shippable.*-qr.*:
+ - --chimera
+ - --browsertime-no-ffwindowrecorder
+ - --conditioned-profile=settled
+ - --extra-profiler-run
+ - --extra-summary-methods=geomean
+ default:
+ - --chimera
+ - --conditioned-profile=settled
+ - --extra-profiler-run
+ - --extra-summary-methods=geomean
+ run-on-projects:
+ by-subtest:
+ netflix: # Bug 1756212 - Disable temporarily due to perma on mac
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ (linux|windows10-64).*shippable-qr.*: [trunk, mozilla-beta, mozilla-release]
+ macosx1015.*shippable-qr.*: []
+ default: []
+ default: []
+ default:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+ default: []
+ default: []
+ treeherder-symbol: Btime(tp6)
+ raptor:
+ apps: [firefox, chrome, chromium]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ - buzzfeed
+ - ebay
+ - espn
+ - expedia
+ - [facebook, fb]
+ - [google-docs, gdocs]
+ - [google-mail, gmail]
+ - [google-search, gsearch]
+ - imdb
+ - imgur
+ - linkedin
+ - microsoft
+ - netflix
+ - nytimes
+ - office
+ - outlook
+ - paypal
+ - pinterest
+ - reddit
+ - tumblr
+ - twitch
+ - wikia
+ - youtube
+ tier:
+ by-app:
+ firefox: 2
+ default: 3
+
+browsertime-responsiveness:
+ <<: *tp6-defaults
+ raptor:
+ apps: [firefox, chrome, chromium]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ - cnn-nav
+ - [facebook-nav, fbnav]
+ - [reddit-billgates-ama, rdt-ama]
+ - [reddit-billgates-post-1, rdt-post-1]
+ - [reddit-billgates-post-2, rdt-post-2]
+ mozharness:
+ extra-options:
+ by-test-platform:
+ windows10-.*:
+ - --cold
+ - --browsertime-no-ffwindowrecorder
+ - --conditioned-profile=settled
+ - --extra-profiler-run
+ default:
+ - --cold
+ - --conditioned-profile=settled
+ - --extra-profiler-run
+ run-on-projects:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
+ default: []
+ default: []
+ tier:
+ by-app:
+ firefox: 2
+ default: 3
+
+browsertime-tp6-essential:
+ <<: *tp6-defaults
+ <<: &tp6-essential-defaults
+ mozharness:
+ extra-options:
+ by-test-platform:
+ windows10-64.*shippable.*-qr.*:
+ - --chimera
+ - --browsertime-no-ffwindowrecorder
+ - --conditioned-profile=settled
+ - --collect-perfstats
+ - --extra-profiler-run
+ - --extra-summary-methods=geomean
+ default:
+ - --chimera
+ - --conditioned-profile=settled
+ - --extra-profiler-run
+ - --extra-summary-methods=geomean
+ raptor:
+ apps: [firefox, chrome, chromium, custom-car]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ - amazon
+ - [bing-search, bing]
+ - cnn
+ - fandom
+ - [google-slides, gslides]
+ - instagram
+ - twitter
+ - wikipedia
+ - yahoo-mail
+ run-on-projects:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ macosx1100.*shippable.*: [mozilla-central, mozilla-beta]
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+ default: []
+ default: []
+ tier:
+ by-test-platform:
+ linux1804-64-clang-trunk-qr/opt: 2
+ default:
+ by-app:
+ firefox: 1
+ default: 3
+
+browsertime-tp6-bytecode:
+ <<: *tp6-defaults
+ <<: *tp6-essential-defaults
+ raptor:
+ apps: [firefox]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ - amazon
+ - [bing-search, bing]
+ - cnn
+ - fandom
+ - [google-slides, gslides]
+ - instagram
+ - twitter
+ - wikipedia
+ - yahoo-mail
+ run-on-projects:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ macosx1100.*shippable.*: [mozilla-central, mozilla-beta]
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+ default: []
+ default: []
+ mozharness:
+ extra-options:
+ by-test-platform:
+ windows10-64.*shippable.*-qr.*:
+ - --chimera
+ - --browsertime
+ - --browsertime-no-ffwindowrecorder
+ - --conditioned-profile=settled
+ - --test-bytecode-cache
+ - --extra-profiler-run
+ default:
+ - --chimera
+ - --browsertime
+ - --conditioned-profile=settled
+ - --test-bytecode-cache
+ - --extra-profiler-run
+ tier:
+ by-app:
+ firefox: 2
+ default: 3
+ treeherder-symbol: Btime-cache(tp6)
+
+browsertime-benchmark:
+ description: Raptor (browsertime) Benchmark tests
+ raptor:
+ apps: [firefox, chrome, chromium, safari, custom-car]
+ subtests:
+ by-app:
+ firefox:
+ - ares6
+ - [assorted-dom, dom]
+ - [jetstream2, js2]
+ - [matrix-react-bench, mrb]
+ - [motionmark-animometer, mm-a]
+ - [motionmark-htmlsuite, mm-h]
+ - [speedometer, sp]
+ - [speedometer3, sp3]
+ - [stylebench, sb]
+ - [sunspider, ss]
+ - [twitch-animation, tw-a]
+ - [unity-webgl, ugl]
+ - [webaudio, wa]
+ safari:
+ - ares6
+ - [assorted-dom, dom]
+ - [jetstream2, js2]
+ # - [matrix-react-bench, mrb] (Bug 1802922)
+ - [motionmark-animometer, mm-a]
+ - [motionmark-htmlsuite, mm-h]
+ - [speedometer, sp]
+ - [speedometer3, sp3]
+ - [stylebench, sb]
+ - [sunspider, ss]
+ - [unity-webgl, ugl]
+ # - [webaudio, wa] (Bug 1802922)
+ custom-car:
+ - [speedometer, sp]
+ - [speedometer3, sp3]
+ default:
+ - ares6
+ - [assorted-dom, dom]
+ - [jetstream2, js2]
+ - [matrix-react-bench, mrb]
+ - [motionmark-animometer, mm-a]
+ - [motionmark-htmlsuite, mm-h]
+ - [speedometer, sp]
+ - [speedometer3, sp3]
+ - [stylebench, sb]
+ - [sunspider, ss]
+ - [unity-webgl, ugl]
+ - [webaudio, wa]
+ treeherder-symbol: Btime()
+ tier:
+ by-app:
+ firefox:
+ by-subtest:
+ motionmark-animometer:
+ by-test-platform:
+ linux1804-64-clang-trunk-qr/opt: 2
+ default: 1
+ unity-webgl: 3
+ default:
+ by-test-platform:
+ windows10-64-ccov.*/.*: 3
+ linux64-ccov.*/.*: 3
+ default: 2
+ default:
+ by-subtest:
+ unity-webgl: 3
+ default: 2
+ run-on-projects:
+ by-subtest:
+ speedometer3:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64.*shippable-qr/.*: [trunk, mozilla-release]
+ .*nightlyasrelease.*: [autoland]
+ default: []
+ default: []
+ unity-webgl:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64.*shippable-qr/.*: [trunk, mozilla-release]
+ default: []
+ default: []
+ speedometer:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+ .*nightlyasrelease.*: [autoland]
+ default: []
+ default: []
+ default:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+ default: []
+ default: []
+ max-run-time:
+ by-subtest:
+ ares6: 1500
+ assorted-dom:
+ by-test-platform:
+ .*-qr/.*: 2100
+ default: 1500
+ jetstream2: 8000
+ matrix-react-bench: 1500
+ motionmark-htmlsuite: 1500
+ unity-webgl: 1500
+ default: 900
+ mozharness:
+ extra-options:
+ - --extra-profiler-run
+
+browsertime-benchmark-wasm:
+ description: Raptor (browsertime) WASM benchmarks
+ raptor:
+ apps: [firefox, chrome, chromium] # Safari depends on Bug 1802922
+ subtests:
+ by-app:
+ firefox:
+ - [wasm-godot, godot]
+ - [wasm-godot-baseline, godot-b]
+ - [wasm-godot-optimizing, godot-i]
+ - [wasm-misc, wm]
+ - [wasm-misc-baseline, wm-b]
+ - [wasm-misc-optimizing, wm-i]
+ default:
+ - [wasm-misc, wm]
+ - [wasm-godot, godot]
+ treeherder-symbol: Btime()
+ tier: 2
+ run-on-projects:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release]
+ default: []
+ default: []
+ max-run-time:
+ by-subtest:
+ wasm-godot: 1500
+ wasm-godot-baseline: 1500
+ default: 900
+ mozharness:
+ extra-options:
+ - --extra-profiler-run
+
+browsertime-youtube-playback:
+ description: Browsertime YouTube Playback on Firefox
+ raptor:
+ apps: [firefox]
+ subtests:
+ by-app:
+ firefox:
+ - [youtube-playback-av1-sfr, ytp-av1-sfr]
+ - [youtube-playback-h264-sfr, ytp-h264-sfr]
+ - [youtube-playback-hfr, ytp-hfr]
+ - [youtube-playback-vp9-sfr, ytp-vp9-sfr]
+ - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
+ - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
+ - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
+ treeherder-symbol: Btime()
+ run-on-projects:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release]
+ default: []
+ tier: 2
+ limit-platforms:
+ by-subtest:
+ youtube-playback-av1-sfr:
+ - macosx1015-64-shippable-qr/opt
+ default: []
+ max-run-time:
+ by-subtest:
+ youtube-playback-hfr: 3600
+ youtube-playback-widevine.*: 3600
+ default: 2700
+
+browsertime-youtube-playback-power:
+ description: Browsertime YouTube Playback on Firefox with power usage tracking
+ raptor:
+ apps: [firefox]
+ subtests:
+ by-app:
+ firefox:
+
+ - [youtube-playback-h264-1080p30, ytp-h264-1080p30]
+ - [youtube-playback-h264-1080p60, ytp-h264-1080p60]
+ - [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30]
+ - [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60]
+ - [youtube-playback-v9-1080p30, ytp-v9-1080p30]
+ - [youtube-playback-v9-1080p60, ytp-v9-1080p60]
+ - [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30]
+ - [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60]
+ treeherder-symbol: Btime()
+ run-on-projects:
+ by-test-platform:
+ macosx.*shippable-qr.*: [mozilla-central]
+ default: []
+ tier: 2
+ max-run-time: 2700
+
+browsertime-tp6-live:
+ <<: *tp6-defaults
+ description: Raptor (browsertime) tp6 on live-sites
+ mozharness:
+ extra-options:
+ - --browser-cycles=15
+ - --chimera
+ - --live-sites
+ - --extra-profiler-run
+ raptor:
+ apps: [firefox, chrome, chromium, safari]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ by-app:
+ safari:
+ - amazon
+ - [bing-search, bing]
+ - ebay
+ - [facebook, fb]
+ # - fandom (Bug 1802922)
+ - [google-docs, gdocs]
+ # - [google-mail, gmail] (Bug 1802922)
+ - [google-search, gsearch]
+ - [google-slides, gslides]
+ - imdb
+ # - imgur (Bug 1802922)
+ # - instagram (see bug 1736999)
+ - linkedin
+ - microsoft
+ # - netflix (Bug 1802922)
+ # - office (site loads blank page if not signed in)
+ - outlook
+ - paypal
+ - pinterest
+ - reddit
+ - tumblr
+ - twitch
+ - twitter
+ - wikipedia
+ # - yahoo-mail (Bug 1802922)
+ - youtube
+ chrome:
+ - amazon
+ - [bing-search, bing]
+ - ebay
+ - [facebook, fb]
+ - fandom
+ - [google-docs, gdocs]
+ - [google-mail, gmail]
+ - [google-search, gsearch]
+ - [google-slides, gslides]
+ - imdb
+ # - imgur (see bugs 1825337 & 1817420)
+ # - instagram (see bug 1736999)
+ - linkedin
+ - microsoft
+ - netflix
+ # - office (site loads blank page if not signed in)
+ - outlook
+ - paypal
+ - pinterest
+ - reddit
+ - tumblr
+ - twitch
+ - twitter
+ - wikipedia
+ - yahoo-mail
+ - youtube
+ default:
+ - amazon
+ - [bing-search, bing]
+ - ebay
+ - [facebook, fb]
+ - fandom
+ - [google-docs, gdocs]
+ - [google-mail, gmail]
+ - [google-search, gsearch]
+ - [google-slides, gslides]
+ - imdb
+ - imgur
+ # - instagram (see bug 1736999)
+ - linkedin
+ - microsoft
+ - netflix
+ # - office (site loads blank page if not signed in)
+ - outlook
+ - paypal
+ - pinterest
+ - reddit
+ - tumblr
+ - twitch
+ - twitter
+ - wikipedia
+ - yahoo-mail
+ - youtube
+ run-on-projects: []
+ tier: 3
+ treeherder-symbol: Btime-live(tp6)
+
+browsertime-tp6-live-sheriffed:
+ <<: *tp6-defaults
+ raptor:
+ apps: [firefox]
+ test: tp6
+ run-visual-metrics: true
+ subtests:
+ - cnn
+ description: Raptor (browsertime) tp6 on live-sites
+ mozharness:
+ extra-options:
+ - --browser-cycles=5
+ - --chimera
+ - --live-sites
+ - --extra-profiler-run
+ run-on-projects:
+ by-test-platform:
+ (linux|windows10-64|macos)(?!.*shippable).*: []
+ (linux|windows|macosx1015)(?!.*-qr).*: []
+ windows.*-32.*: []
+ default: []
+ tier: 2
+ treeherder-symbol: Btime-live(tp6)
+
+browsertime-tp6-profiling:
+ <<: *tp6-defaults
+ raptor:
+ apps: [firefox]
+ test: tp6
+ run-visual-metrics: true
+ subtests: [amazon]
+ description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling
+ mozharness:
+ extra-options:
+ - --chimera
+ - --gecko-profile
+ run-on-projects:
+ by-test-platform:
+ (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central]
+ default: []
+ tier: 2
+ treeherder-symbol: Btime-Prof(tp6)
+
+browsertime-custom:
+ raptor:
+ apps: [firefox]
+ run-visual-metrics: true
+ subtests: [[process-switch, ps]]
+ description: Raptor (browsertime) custom pageload tests
+ max-run-time: 2700
+ mozharness:
+ extra-options:
+ - --chimera
+ - --extra-profiler-run
+ run-on-projects:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central]
+ default: []
+ tier: 1
+ treeherder-symbol: Btime(ps)
+
+browsertime-first-install:
+ raptor:
+ apps: [firefox]
+ run-visual-metrics: true
+ subtests: [welcome]
+ description: Raptor (browsertime) custom pageload tests
+ max-run-time: 2700
+ mozharness:
+ extra-options:
+ - --chimera
+ - --extra-profiler-run
+ run-on-projects:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
+ default: []
+ tier: 1
+ treeherder-symbol: Btime(welcome)
+
+browsertime-regression-tests:
+ raptor:
+ apps: [firefox]
+ subtests: [[constant-regression, regress-c]]
+ description: Raptor (browsertime) tests for our regression detection system
+ max-run-time: 700
+ run-on-projects: [autoland]
+ limit-platforms:
+ - .*linux(?!.*shippable)(?!.*clang).*
+ mozharness:
+ extra-options:
+ by-test-platform:
+ windows10-64.*shippable.*-qr.*: []
+ default: []
+ tier: 2
+ treeherder-symbol: Btime(regressor)
+
+browsertime-upload:
+ raptor:
+ apps: [firefox, chrome, chromium, safari]
+ subtests:
+ - [upload, upload-h2]
+ - upload-h3
+ description: Raptor (browsertime) http2, http3 file upload performance test
+ max-run-time: 2700
+ run-on-projects:
+ by-app:
+ firefox:
+ by-test-platform:
+ windows.*-32.*: []
+ .*64.*shippable-qr/.*: [autoland, mozilla-central]
+ default: []
+ default: []
+ tier: 3
+ treeherder-symbol: Btime(upload)
+ fetches:
+ fetch:
+ - upload-speed-test-file