diff options
Diffstat (limited to 'taskcluster/ci/test/browsertime-desktop.yml')
| -rw-r--r-- | taskcluster/ci/test/browsertime-desktop.yml | 918 |
1 files changed, 918 insertions, 0 deletions
diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..b1422e1b2f --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,918 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: [mozilla-central] + default: [] + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + - --conditioned-profile=settled + fetches: + toolchain: + by-app: + custom-car: + by-test-platform: + # this satisfied taskgraph for now, but once mac/win is implemented + # then linux will be the default + linux.*: + - linux64-custom-car + - linux64-minidump-stackwalk + - linux64-fix-stacks + windows10-64.*: + - win64-custom-car + - win64-minidump-stackwalk + - win64-fix-stacks + macosx1015.*: + - macosx64-custom-car + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + macosx1400.*: + - macosx-arm64-custom-car + - macosx64-aarch64-minidump-stackwalk + - macosx64-aarch64-fix-stacks + default: [] + default: + by-test-platform: + windows.*-32.*.: + - win32-minidump-stackwalk + - win32-fix-stacks + windows10-64.*: + - win64-minidump-stackwalk + - win64-fix-stacks + macos.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + +browsertime-tp6: + <<: &tp6-defaults + description: Raptor (browsertime) tp6 page-load tests + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + max-run-time: + by-subtest: + imgur: 4800 + default: 4000 + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-subtest: + netflix: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + (macosx1015|linux|windows10-64).*shippable-qr.*: [trunk] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + treeherder-symbol: Btime(tp6) + raptor: + apps: [firefox, chrome] + test: tp6 + run-visual-metrics: true + subtests: + - buzzfeed + - ebay + - espn + - expedia + - [facebook, fb] + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - imdb + - imgur + - linkedin + - microsoft + - netflix + - nytimes + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - wikia + - youtube + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-responsiveness: + <<: *tp6-defaults + raptor: + apps: [firefox, chrome] + test: tp6 + run-visual-metrics: true + subtests: + - cnn-nav + - [facebook-nav, fbnav] + - [reddit-billgates-ama, rdt-ama] + - [reddit-billgates-post-1, rdt-post-1] + - [reddit-billgates-post-2, rdt-post-2] + mozharness: + extra-options: + by-test-platform: + windows10-.*: + - --cold + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --cold + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-tp6-essential: + <<: *tp6-defaults + <<: &tp6-essential-defaults + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --collect-perfstats + - --extra-profiler-run + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + raptor: + apps: [firefox, chrome, custom-car] + test: tp6 + run-visual-metrics: + by-test-platform: + macosx1400.*shippable.*: false + # Bug 1845014 Resolve osx 1300 vismet permafails + default: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: + by-app: + firefox: 1 + default: 3 + +browsertime-tp6-bytecode: + <<: *tp6-defaults + <<: *tp6-essential-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + default: + - --chimera + - --browsertime + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + tier: + by-app: + firefox: 2 + default: 3 + treeherder-symbol: Btime-cache(tp6) + +browsertime-benchmark: + description: Raptor (browsertime) Benchmark tests + raptor: + apps: [firefox, chrome, safari, custom-car] + subtests: + by-test-platform: + macosx1400.*shippable.*: + - [speedometer, sp] + - [speedometer3, sp3] + default: + by-app: + firefox: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [twitch-animation, tw-a] + - [unity-webgl, ugl] + - [webaudio, wa] + safari: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + # - [matrix-react-bench, mrb] Perma fails on osx 10.15.x machines (Bug 1807754) + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + # - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535) + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + custom-car: + - [speedometer, sp] + - [speedometer3, sp3] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + chrome: + by-test-platform: + macosx1015.*shippable.*: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite, mm-h] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite, mm-h] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + treeherder-symbol: Btime() + tier: + by-app: + firefox: + by-subtest: + motionmark-animometer: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: 1 + default: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + default: + by-subtest: + unity-webgl: 3 + default: 2 + run-on-projects: + by-subtest: + speedometer3: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + .*nightlyasrelease.*: [autoland, mozilla-central] + default: [] + custom-car: + by-test-platform: + (linux|windows10-64).*shippable-qr.*: [mozilla-central] + default: [] + default: [] + unity-webgl: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + # Disabled on autoland due to bug 1720063, and bug 1795826 + .*64.*shippable-qr/.*: [mozilla-central] + default: [] + default: [] + speedometer: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + .*nightlyasrelease.*: [autoland] + default: [] + custom-car: + by-test-platform: + (linux|windows10-64).*shippable-qr.*: [mozilla-central] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + default: [] + default: [] + max-run-time: + by-subtest: + ares6: 1500 + assorted-dom: + by-test-platform: + .*-qr/.*: 2100 + default: 1500 + jetstream2: 8000 + matrix-react-bench: 1500 + motionmark-htmlsuite: 1500 + motionmark-htmlsuite-ramp: 3000 + motionmark-animometer-ramp: 2100 + unity-webgl: 2400 + speedometer3: + by-test-platform: + macosx1400.*shippable.*: 4200 + default: 2100 + speedometer: 2100 + default: 900 + mozharness: + extra-options: + by-test-platform: + macosx1400.*shippable.*: + - --extra-profiler-run + - --page-timeout=900000 + default: + - --extra-profiler-run + +browsertime-benchmark-wasm: + description: Raptor (browsertime) WASM benchmarks + raptor: + apps: [firefox, chrome, safari] + subtests: + by-app: + firefox: + - [wasm-godot, godot] + - [wasm-godot-baseline, godot-b] + - [wasm-godot-optimizing, godot-i] + - [wasm-misc, wm] + - [wasm-misc-baseline, wm-b] + - [wasm-misc-optimizing, wm-i] + safari: + # - [wasm-misc, wm] Perma fails (Bug 1809533) + - [wasm-godot, godot] + default: + - [wasm-misc, wm] + - [wasm-godot, godot] + treeherder-symbol: Btime() + tier: 2 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + default: [] + max-run-time: + by-subtest: + wasm-godot: 1500 + wasm-godot-baseline: 1500 + wasm-misc-baseline: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-youtube-playback: + description: Browsertime YouTube Playback on Firefox + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + - [youtube-playback-av1-sfr, ytp-av1-sfr] + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + tier: 2 + limit-platforms: + by-subtest: + youtube-playback-av1-sfr: + - macosx1015-64-shippable-qr/opt + default: [] + max-run-time: + by-subtest: + youtube-playback-hfr: 3600 + youtube-playback-widevine.*: 3600 + default: 2700 + +browsertime-youtube-playback-power: + description: Browsertime YouTube Playback on Firefox with power usage tracking + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + + - [youtube-playback-h264-1080p30, ytp-h264-1080p30] + - [youtube-playback-h264-1080p60, ytp-h264-1080p60] + - [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30] + - [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60] + - [youtube-playback-v9-1080p30, ytp-v9-1080p30] + - [youtube-playback-v9-1080p60, ytp-v9-1080p60] + - [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30] + - [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + macosx.*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + max-run-time: 2700 + +browsertime-tp6-live: + <<: *tp6-defaults + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=15 + - --chimera + - --live-sites + - --extra-profiler-run + raptor: + apps: [firefox, chrome, safari] + test: tp6 + run-visual-metrics: true + subtests: + by-app: + safari: + - amazon + - [bing-search, bing] + - ebay + # - [facebook, fb] Perma fails on osx 10.15.7 + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + # - instagram # (see bug 1736999) + - linkedin + - microsoft + # - netflix # Perma fails on osx 10.15.7 (Bug 1802922) + # - office (site loads blank page if not signed in) + # - outlook Perma fails on osx 10.15.7 + - paypal + - pinterest + - reddit + - tumblr + - twitch + # - twitter Perma fails on osx 10.15.7 + - wikipedia + - yahoo-mail + - youtube + chrome: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (see bugs 1825337 & 1817420) + - instagram + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + default: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + - instagram + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-live-sheriffed: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - cnn + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=5 + - --chimera + - --live-sites + - --extra-profiler-run + - --browsertime-arg=maxLoadTime=240000 + run-on-projects: + by-test-platform: + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows|macosx1015)(?!.*-qr).*: [] + windows.*-32.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-profiling: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: [amazon] + description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling + mozharness: + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + treeherder-symbol: Btime-Prof(tp6) + +browsertime-custom: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [[process-switch, ps]] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + tier: 1 + treeherder-symbol: Btime(ps) + +browsertime-first-install: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [welcome] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + tier: 1 + treeherder-symbol: Btime(welcome) + +browsertime-regression-tests: + raptor: + apps: [firefox] + subtests: [[constant-regression, regress-c]] + description: Raptor (browsertime) tests for our regression detection system + max-run-time: 700 + run-on-projects: [autoland] + limit-platforms: + - .*linux(?!.*shippable)(?!.*clang).* + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime(regressor) + +browsertime-upload: + raptor: + apps: [firefox, chrome, safari] + subtests: + - [upload, upload-h2] + - upload-h3 + description: Raptor (browsertime) http2, http3 file upload performance test + max-run-time: 2700 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime(upload) + fetches: + fetch: + - upload-speed-test-file + +browsertime-pageload-benchmark: + raptor: + apps: [firefox] + subtests: [tp6-bench] + description: Raptor (browsertime) TP6 pageload benchmark test + max-run-time: 4500 + mozharness: + extra-options: + - --browser-cycles=5 + run-on-projects: [] + tier: 2 + treeherder-symbol: Btime(tp6-bench) + +browsertime-speculative: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [connect] + description: Raptor (browsertime) speculative connect pageload tests + max-run-time: 420 + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland] + default: [] + tier: 2 + treeherder-symbol: Btime(speculative) + +browsertime-throttled: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [throttled] + description: Raptor (browsertime) throttled network pageload tests + max-run-time: 420 + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime(throttled) + +browsertime-tp6-webextensions: + <<: *tp6-defaults + <<: *tp6-essential-defaults + description: Raptor (browsertime) tp6 page-load tests with webextensions installed on Firefox Desktop + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + fetches: + fetch: + - firefox-addons + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk-only] + default: [] + default: [] + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled-webext + default: + - --chimera + - --browsertime + - --conditioned-profile=settled-webext + # TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3. + tier: 3 + treeherder-symbol: Btime-webext(tp6) + +browsertime-indexeddb: + description: Raptor (browsertime) tests for IndexedDB + max-run-time: 2700 + treeherder-symbol: Btime(idb) + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + raptor: + apps: [firefox, chrome, chromium] + run-visual-metrics: false + subtests: + - addMab1 + - addMabN + - addMar1 + - addMarN + - addMbl1 + - addMblN + - addkAB1 + - addkABN + - addkAR1 + - addkARN + - addkBL1 + - addkBLN + - getkeyrng + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + mozharness: + extra-options: + - --extra-profiler-run |