diff options
Diffstat (limited to 'taskcluster/ci/test/browsertime-desktop.yml')
| -rw-r--r-- | taskcluster/ci/test/browsertime-desktop.yml | 721 |
1 files changed, 721 insertions, 0 deletions
diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..b15fe1ae36 --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,721 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: [mozilla-central] + default: [] + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - linux.*shippable.* + - windows10-64.*shippable.* + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + - --conditioned-profile=settled + fetches: + toolchain: + by-app: + custom-car: + by-test-platform: + # this satisfied taskgraph for now, but once mac/win is implemented + # then linux will be the default + linux.*: + - linux64-custom-car + - linux64-minidump-stackwalk + - linux64-fix-stacks + windows10-64.*: + - win64-custom-car + - win64-minidump-stackwalk + - win64-fix-stacks + default: [] + default: + by-test-platform: + windows.*-32.*.: + - win32-minidump-stackwalk + - win32-fix-stacks + windows10-64.*: + - win64-minidump-stackwalk + - win64-fix-stacks + macos.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + +browsertime-tp6: + <<: &tp6-defaults + description: Raptor (browsertime) tp6 page-load tests + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - (windows10-64|linux).*shippable.* + default: [] + max-run-time: + by-subtest: + imgur: 4800 + default: 4000 + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + - --extra-summary-methods=geomean + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + - --extra-summary-methods=geomean + run-on-projects: + by-subtest: + netflix: # Bug 1756212 - Disable temporarily due to perma on mac + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64).*shippable-qr.*: [trunk, mozilla-beta, mozilla-release] + macosx1015.*shippable-qr.*: [] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + treeherder-symbol: Btime(tp6) + raptor: + apps: [firefox, chrome, chromium] + test: tp6 + run-visual-metrics: true + subtests: + - buzzfeed + - ebay + - espn + - expedia + - [facebook, fb] + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - imdb + - imgur + - linkedin + - microsoft + - netflix + - nytimes + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - wikia + - youtube + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-responsiveness: + <<: *tp6-defaults + raptor: + apps: [firefox, chrome, chromium] + test: tp6 + run-visual-metrics: true + subtests: + - cnn-nav + - [facebook-nav, fbnav] + - [reddit-billgates-ama, rdt-ama] + - [reddit-billgates-post-1, rdt-post-1] + - [reddit-billgates-post-2, rdt-post-2] + mozharness: + extra-options: + by-test-platform: + windows10-.*: + - --cold + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --cold + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-tp6-essential: + <<: *tp6-defaults + <<: &tp6-essential-defaults + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --collect-perfstats + - --extra-profiler-run + - --extra-summary-methods=geomean + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + - --extra-summary-methods=geomean + raptor: + apps: [firefox, chrome, chromium, custom-car] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central, mozilla-beta] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + tier: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: + by-app: + firefox: 1 + default: 3 + +browsertime-tp6-bytecode: + <<: *tp6-defaults + <<: *tp6-essential-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central, mozilla-beta] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + default: + - --chimera + - --browsertime + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + tier: + by-app: + firefox: 2 + default: 3 + treeherder-symbol: Btime-cache(tp6) + +browsertime-benchmark: + description: Raptor (browsertime) Benchmark tests + raptor: + apps: [firefox, chrome, chromium, safari, custom-car] + subtests: + by-app: + firefox: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [twitch-animation, tw-a] + - [unity-webgl, ugl] + - [webaudio, wa] + safari: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + # - [matrix-react-bench, mrb] (Bug 1802922) + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + # - [webaudio, wa] (Bug 1802922) + custom-car: + - [speedometer, sp] + - [speedometer3, sp3] + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + treeherder-symbol: Btime() + tier: + by-app: + firefox: + by-subtest: + motionmark-animometer: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: 1 + unity-webgl: 3 + default: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + default: + by-subtest: + unity-webgl: 3 + default: 2 + run-on-projects: + by-subtest: + speedometer3: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-release] + .*nightlyasrelease.*: [autoland] + default: [] + default: [] + unity-webgl: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-release] + default: [] + default: [] + speedometer: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + .*nightlyasrelease.*: [autoland] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + max-run-time: + by-subtest: + ares6: 1500 + assorted-dom: + by-test-platform: + .*-qr/.*: 2100 + default: 1500 + jetstream2: 8000 + matrix-react-bench: 1500 + motionmark-htmlsuite: 1500 + unity-webgl: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-benchmark-wasm: + description: Raptor (browsertime) WASM benchmarks + raptor: + apps: [firefox, chrome, chromium] # Safari depends on Bug 1802922 + subtests: + by-app: + firefox: + - [wasm-godot, godot] + - [wasm-godot-baseline, godot-b] + - [wasm-godot-optimizing, godot-i] + - [wasm-misc, wm] + - [wasm-misc-baseline, wm-b] + - [wasm-misc-optimizing, wm-i] + default: + - [wasm-misc, wm] + - [wasm-godot, godot] + treeherder-symbol: Btime() + tier: 2 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release] + default: [] + default: [] + max-run-time: + by-subtest: + wasm-godot: 1500 + wasm-godot-baseline: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-youtube-playback: + description: Browsertime YouTube Playback on Firefox + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + - [youtube-playback-av1-sfr, ytp-av1-sfr] + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release] + default: [] + tier: 2 + limit-platforms: + by-subtest: + youtube-playback-av1-sfr: + - macosx1015-64-shippable-qr/opt + default: [] + max-run-time: + by-subtest: + youtube-playback-hfr: 3600 + youtube-playback-widevine.*: 3600 + default: 2700 + +browsertime-youtube-playback-power: + description: Browsertime YouTube Playback on Firefox with power usage tracking + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + + - [youtube-playback-h264-1080p30, ytp-h264-1080p30] + - [youtube-playback-h264-1080p60, ytp-h264-1080p60] + - [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30] + - [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60] + - [youtube-playback-v9-1080p30, ytp-v9-1080p30] + - [youtube-playback-v9-1080p60, ytp-v9-1080p60] + - [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30] + - [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + macosx.*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + max-run-time: 2700 + +browsertime-tp6-live: + <<: *tp6-defaults + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=15 + - --chimera + - --live-sites + - --extra-profiler-run + raptor: + apps: [firefox, chrome, chromium, safari] + test: tp6 + run-visual-metrics: true + subtests: + by-app: + safari: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + # - fandom (Bug 1802922) + - [google-docs, gdocs] + # - [google-mail, gmail] (Bug 1802922) + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (Bug 1802922) + # - instagram (see bug 1736999) + - linkedin + - microsoft + # - netflix (Bug 1802922) + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + # - yahoo-mail (Bug 1802922) + - youtube + chrome: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (see bugs 1825337 & 1817420) + # - instagram (see bug 1736999) + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + default: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + # - instagram (see bug 1736999) + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-live-sheriffed: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - cnn + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=5 + - --chimera + - --live-sites + - --extra-profiler-run + run-on-projects: + by-test-platform: + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows|macosx1015)(?!.*-qr).*: [] + windows.*-32.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-profiling: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: [amazon] + description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling + mozharness: + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + treeherder-symbol: Btime-Prof(tp6) + +browsertime-custom: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [[process-switch, ps]] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + tier: 1 + treeherder-symbol: Btime(ps) + +browsertime-first-install: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [welcome] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + tier: 1 + treeherder-symbol: Btime(welcome) + +browsertime-regression-tests: + raptor: + apps: [firefox] + subtests: [[constant-regression, regress-c]] + description: Raptor (browsertime) tests for our regression detection system + max-run-time: 700 + run-on-projects: [autoland] + limit-platforms: + - .*linux(?!.*shippable)(?!.*clang).* + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime(regressor) + +browsertime-upload: + raptor: + apps: [firefox, chrome, chromium, safari] + subtests: + - [upload, upload-h2] + - upload-h3 + description: Raptor (browsertime) http2, http3 file upload performance test + max-run-time: 2700 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime(upload) + fetches: + fetch: + - upload-speed-test-file |