# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- job-defaults: max-run-time: by-test-platform: .*-qr/.*: 2400 default: 1800 suite: raptor run-on-projects: by-test-platform: linux.*shippable[^-qr].*: [mozilla-central] default: [] limit-platforms: by-app: chrome: - .*shippable.* safari: - .*macos.*shippable.* custom-car: - .*64.*shippable.* default: [] test-manifest-loader: null # don't load tests in the taskgraph tier: 3 virtualization: hardware mozharness: script: raptor_script.py config: by-test-platform: macosx.*: - raptor/mac_config.py windows.*: - raptor/windows_config.py default: - raptor/linux_config.py extra-options: - --conditioned-profile=settled fetches: toolchain: by-app: custom-car: by-test-platform: # this satisfied taskgraph for now, but once mac/win is implemented # then linux will be the default linux.*: - linux64-custom-car - linux64-minidump-stackwalk - linux64-fix-stacks windows10-64.*: - win64-custom-car - win64-minidump-stackwalk - win64-fix-stacks macosx1015.*: - macosx64-custom-car - macosx64-minidump-stackwalk - macosx64-fix-stacks macosx1400.*: - macosx-arm64-custom-car - macosx64-aarch64-minidump-stackwalk - macosx64-aarch64-fix-stacks default: [] default: by-test-platform: windows.*-32.*.: - win32-minidump-stackwalk - win32-fix-stacks windows10-64.*: - win64-minidump-stackwalk - win64-fix-stacks macos.*: - macosx64-minidump-stackwalk - macosx64-fix-stacks default: - linux64-minidump-stackwalk - linux64-fix-stacks browsertime-tp6: <<: &tp6-defaults description: Raptor (browsertime) tp6 page-load tests limit-platforms: by-app: chrome: - .*shippable.* safari: - .*macos.*shippable.* custom-car: - .*64.*shippable.* default: [] max-run-time: by-subtest: imgur: 4800 default: 4000 mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: - --chimera - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --extra-profiler-run default: - --chimera - --conditioned-profile=settled - --extra-profiler-run run-on-projects: by-subtest: netflix: by-app: firefox: by-test-platform: windows.*-32.*: [] (macosx1015|linux|windows10-64).*shippable-qr.*: [trunk] default: [] default: [] default: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] default: [] default: [] treeherder-symbol: Btime(tp6) raptor: apps: [firefox, chrome] test: tp6 run-visual-metrics: true subtests: - buzzfeed - ebay - espn - expedia - [facebook, fb] - [google-docs, gdocs] - [google-mail, gmail] - [google-search, gsearch] - imdb - imgur - linkedin - microsoft - netflix - nytimes - office - outlook - paypal - pinterest - reddit - tumblr - twitch - wikia - youtube tier: by-app: firefox: 2 default: 3 browsertime-responsiveness: <<: *tp6-defaults raptor: apps: [firefox, chrome] test: tp6 run-visual-metrics: true subtests: - cnn-nav - [facebook-nav, fbnav] - [reddit-billgates-ama, rdt-ama] - [reddit-billgates-post-1, rdt-post-1] - [reddit-billgates-post-2, rdt-post-2] mozharness: extra-options: by-test-platform: windows10-.*: - --cold - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --extra-profiler-run default: - --cold - --conditioned-profile=settled - --extra-profiler-run run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] default: [] default: [] tier: by-app: firefox: 2 default: 3 browsertime-tp6-essential: <<: *tp6-defaults <<: &tp6-essential-defaults mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: - --chimera - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --collect-perfstats - --extra-profiler-run default: - --chimera - --conditioned-profile=settled - --extra-profiler-run raptor: apps: [firefox, chrome, custom-car] test: tp6 run-visual-metrics: by-test-platform: macosx1400.*shippable.*: false # Bug 1845014 Resolve osx 1300 vismet permafails default: true subtests: - amazon - [bing-search, bing] - cnn - fandom - [google-slides, gslides] - instagram - twitter - wikipedia - yahoo-mail run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] macosx1100.*shippable.*: [mozilla-central] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] default: [] default: [] tier: by-test-platform: linux1804-64-clang-trunk-qr/opt: 2 default: by-app: firefox: 1 default: 3 browsertime-tp6-bytecode: <<: *tp6-defaults <<: *tp6-essential-defaults raptor: apps: [firefox] test: tp6 run-visual-metrics: true subtests: - amazon - [bing-search, bing] - cnn - fandom - [google-slides, gslides] - instagram - twitter - wikipedia - yahoo-mail run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] macosx1100.*shippable.*: [mozilla-central] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] default: [] default: [] mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: - --chimera - --browsertime - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --test-bytecode-cache - --extra-profiler-run default: - --chimera - --browsertime - --conditioned-profile=settled - --test-bytecode-cache - --extra-profiler-run tier: by-app: firefox: 2 default: 3 treeherder-symbol: Btime-cache(tp6) browsertime-tp7: <<: &tp7-defaults description: Raptor (browsertime) tp7 page-load tests limit-platforms: by-app: chrome: - .*shippable.* safari: - .*macos.*shippable.* custom-car: - .*64.*shippable.* default: [] max-run-time: by-subtest: imgur: 4800 default: 4000 mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: - --chimera - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --extra-profiler-run default: - --chimera - --conditioned-profile=settled - --extra-profiler-run run-on-projects: [] treeherder-symbol: Btime(tp7) raptor: apps: [firefox, chrome] test: tp7 run-visual-metrics: true subtests: - tiktok - openai - samsung - weather - docomo tier: by-app: firefox: 3 default: 3 browsertime-benchmark: description: Raptor (browsertime) Benchmark tests raptor: apps: [firefox, chrome, safari, custom-car] subtests: by-test-platform: macosx1400.*shippable.*: - [speedometer, sp] - [speedometer3, sp3] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] default: by-app: firefox: - ares6 - [assorted-dom, dom] - [jetstream2, js2] - [matrix-react-bench, mrb] - [motionmark-animometer, mm-a] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-htmlsuite, mm-h] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] - [sunspider, ss] - [twitch-animation, tw-a] - [unity-webgl, ugl] - [webaudio, wa] safari: - ares6 - [assorted-dom, dom] - [jetstream2, js2] # - [matrix-react-bench, mrb] Perma fails on osx 10.15.x machines (Bug 1807754) - [motionmark-animometer, mm-a] - [motionmark-htmlsuite, mm-h] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] # - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535) - [stylebench, sb] - [sunspider, ss] # - [unity-webgl, ugl] Almost Perma failing on osx 10.15.x machines (Bug 1888398) - [webaudio, wa] custom-car: - [speedometer, sp] - [speedometer3, sp3] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] chrome: by-test-platform: macosx1015.*shippable.*: - ares6 - [assorted-dom, dom] - [jetstream2, js2] - [matrix-react-bench, mrb] - [motionmark-animometer, mm-a] - [motionmark-htmlsuite, mm-h] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] - [sunspider, ss] - [unity-webgl, ugl] - [webaudio, wa] default: - ares6 - [assorted-dom, dom] - [jetstream2, js2] - [matrix-react-bench, mrb] - [motionmark-animometer, mm-a] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite, mm-h] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] - [sunspider, ss] - [unity-webgl, ugl] - [webaudio, wa] default: - ares6 - [assorted-dom, dom] - [jetstream2, js2] - [matrix-react-bench, mrb] - [motionmark-animometer, mm-a] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite, mm-h] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-1-3, mm-1.3] - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] - [sunspider, ss] - [unity-webgl, ugl] - [webaudio, wa] treeherder-symbol: Btime() tier: by-app: firefox: by-subtest: motionmark-animometer: by-test-platform: linux1804-64-clang-trunk-qr/opt: 2 default: 1 default: by-test-platform: windows10-64-ccov.*/.*: 3 linux64-ccov.*/.*: 3 default: 2 default: by-subtest: unity-webgl: 3 default: 2 run-on-projects: by-subtest: speedometer3: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [trunk] .*nightlyasrelease.*: [autoland, mozilla-central] default: [] custom-car: by-test-platform: (linux|windows10-64).*shippable-qr.*: [mozilla-central] default: [] default: [] unity-webgl: by-app: firefox: by-test-platform: windows.*-32.*: [] # Disabled on autoland due to bug 1720063, and bug 1795826 .*64.*shippable-qr/.*: [mozilla-central] default: [] default: [] speedometer: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [trunk] .*nightlyasrelease.*: [autoland] default: [] custom-car: by-test-platform: (linux|windows10-64).*shippable-qr.*: [mozilla-central] default: [] default: [] default: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [trunk] default: [] default: [] max-run-time: by-subtest: ares6: 1500 assorted-dom: by-test-platform: .*-qr/.*: 2100 default: 1500 jetstream2: 8000 matrix-react-bench: 1500 motionmark-htmlsuite: 1500 motionmark-htmlsuite-ramp: 3000 motionmark-animometer-ramp: 2100 motionmark-1-3: 3000 motionmark-htmlsuite-1-3: 3000 unity-webgl: 2400 speedometer3: by-test-platform: macosx1400.*shippable.*: 4200 default: 2100 speedometer: 2100 default: 900 mozharness: extra-options: by-test-platform: macosx1400.*shippable.*: - --extra-profiler-run - --page-timeout=900000 default: - --extra-profiler-run browsertime-benchmark-wasm: description: Raptor (browsertime) WASM benchmarks raptor: apps: [firefox, chrome, safari] subtests: by-app: firefox: - [wasm-godot, godot] - [wasm-godot-baseline, godot-b] - [wasm-godot-optimizing, godot-i] - [wasm-misc, wm] - [wasm-misc-baseline, wm-b] - [wasm-misc-optimizing, wm-i] safari: # - [wasm-misc, wm] Perma fails (Bug 1809533) - [wasm-godot, godot] default: - [wasm-misc, wm] - [wasm-godot, godot] treeherder-symbol: Btime() tier: 2 run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] default: [] default: [] max-run-time: by-subtest: wasm-godot: 1500 wasm-godot-baseline: 1500 wasm-misc-baseline: 1500 default: 900 mozharness: extra-options: - --extra-profiler-run browsertime-youtube-playback: description: Browsertime YouTube Playback on Firefox raptor: apps: [firefox] subtests: by-app: firefox: - [youtube-playback-av1-sfr, ytp-av1-sfr] - [youtube-playback-h264-sfr, ytp-h264-sfr] - [youtube-playback-hfr, ytp-hfr] - [youtube-playback-vp9-sfr, ytp-vp9-sfr] - [youtube-playback-widevine-hfr, ytp-widevine-hfr] - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] treeherder-symbol: Btime() run-on-projects: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] default: [] tier: 2 limit-platforms: by-subtest: youtube-playback-av1-sfr: - macosx1015-64-shippable-qr/opt default: [] max-run-time: by-subtest: youtube-playback-hfr: 3600 youtube-playback-widevine.*: 3600 default: 2700 browsertime-youtube-playback-power: description: Browsertime YouTube Playback on Firefox with power usage tracking raptor: apps: [firefox] subtests: by-app: firefox: - [youtube-playback-h264-1080p30, ytp-h264-1080p30] - [youtube-playback-h264-1080p60, ytp-h264-1080p60] - [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30] - [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60] - [youtube-playback-v9-1080p30, ytp-v9-1080p30] - [youtube-playback-v9-1080p60, ytp-v9-1080p60] - [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30] - [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60] treeherder-symbol: Btime() run-on-projects: by-test-platform: macosx.*shippable-qr.*: [mozilla-central] default: [] tier: 2 max-run-time: 2700 browsertime-tp6-live: <<: *tp6-defaults description: Raptor (browsertime) tp6 on live-sites mozharness: extra-options: - --browser-cycles=15 - --chimera - --live-sites - --extra-profiler-run raptor: apps: [firefox, chrome, safari] test: tp6 run-visual-metrics: true subtests: by-app: safari: - amazon - [bing-search, bing] - ebay # - [facebook, fb] Perma fails on osx 10.15.7 - fandom - [google-docs, gdocs] - [google-mail, gmail] - [google-search, gsearch] - [google-slides, gslides] - imdb - imgur # - instagram # (see bug 1736999) - linkedin - microsoft # - netflix # Perma fails on osx 10.15.7 (Bug 1802922) # - office (site loads blank page if not signed in) # - outlook Perma fails on osx 10.15.7 - paypal - pinterest - reddit - tumblr - twitch # - twitter Perma fails on osx 10.15.7 - wikipedia - yahoo-mail - youtube chrome: - amazon - [bing-search, bing] - ebay - [facebook, fb] - fandom - [google-docs, gdocs] - [google-mail, gmail] - [google-search, gsearch] - [google-slides, gslides] - imdb # - imgur (see bugs 1825337 & 1817420) - instagram - linkedin - microsoft - netflix # - office (site loads blank page if not signed in) - outlook - paypal - pinterest - reddit - tumblr - twitch - twitter - wikipedia - yahoo-mail - youtube default: - amazon - [bing-search, bing] - ebay - [facebook, fb] - fandom - [google-docs, gdocs] - [google-mail, gmail] - [google-search, gsearch] - [google-slides, gslides] - imdb - imgur - instagram - linkedin - microsoft - netflix # - office (site loads blank page if not signed in) - outlook - paypal - pinterest - reddit - tumblr - twitch - twitter - wikipedia - yahoo-mail - youtube run-on-projects: [] tier: 3 treeherder-symbol: Btime-live(tp6) browsertime-tp6-live-sheriffed: <<: *tp6-defaults raptor: apps: [firefox] test: tp6 run-visual-metrics: true subtests: - cnn description: Raptor (browsertime) tp6 on live-sites mozharness: extra-options: - --browser-cycles=5 - --chimera - --live-sites - --extra-profiler-run - --browsertime-arg=maxLoadTime=240000 run-on-projects: by-test-platform: (linux|windows10-64|macos)(?!.*shippable).*: [] (linux|windows|macosx1015)(?!.*-qr).*: [] windows.*-32.*: [] default: [] tier: 2 treeherder-symbol: Btime-live(tp6) browsertime-tp6-profiling: <<: *tp6-defaults raptor: apps: [firefox] test: tp6 run-visual-metrics: true subtests: [amazon] description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling mozharness: extra-options: - --chimera - --gecko-profile run-on-projects: by-test-platform: (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central] default: [] tier: 2 treeherder-symbol: Btime-Prof(tp6) browsertime-custom: raptor: apps: [firefox] run-visual-metrics: true subtests: [[process-switch, ps]] description: Raptor (browsertime) custom pageload tests max-run-time: 2700 mozharness: extra-options: - --chimera - --extra-profiler-run run-on-projects: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] default: [] tier: 1 treeherder-symbol: Btime(ps) browsertime-first-install: raptor: apps: [firefox] run-visual-metrics: true subtests: [welcome] description: Raptor (browsertime) custom pageload tests max-run-time: 2700 mozharness: extra-options: - --chimera - --extra-profiler-run run-on-projects: by-test-platform: windows.*-32.*: [] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] default: [] tier: 1 treeherder-symbol: Btime(welcome) browsertime-regression-tests: raptor: apps: [firefox] subtests: [[constant-regression, regress-c]] description: Raptor (browsertime) tests for our regression detection system max-run-time: 700 run-on-projects: [autoland] limit-platforms: - .*linux(?!.*shippable)(?!.*clang).* mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: [] default: [] tier: 2 treeherder-symbol: Btime(regressor) browsertime-upload: raptor: apps: [firefox, chrome, safari] subtests: - [upload, upload-h2] - upload-h3 description: Raptor (browsertime) http2, http3 file upload performance test max-run-time: 2700 run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [autoland, mozilla-central] default: [] default: [] tier: 3 treeherder-symbol: Btime(upload) fetches: fetch: - upload-speed-test-file browsertime-pageload-benchmark: raptor: apps: [firefox] subtests: [tp6-bench] description: Raptor (browsertime) TP6 pageload benchmark test max-run-time: 4500 mozharness: extra-options: - --browser-cycles=5 run-on-projects: [] tier: 2 treeherder-symbol: Btime(tp6-bench) browsertime-speculative: raptor: apps: [firefox] run-visual-metrics: true subtests: [connect] description: Raptor (browsertime) speculative connect pageload tests max-run-time: 420 run-on-projects: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [autoland] default: [] tier: 2 treeherder-symbol: Btime(speculative) browsertime-throttled: raptor: apps: [firefox] run-visual-metrics: true subtests: [throttled] description: Raptor (browsertime) throttled network pageload tests max-run-time: 420 run-on-projects: [] tier: 3 treeherder-symbol: Btime(throttled) browsertime-tp6-webextensions: <<: *tp6-defaults <<: *tp6-essential-defaults description: Raptor (browsertime) tp6 page-load tests with webextensions installed on Firefox Desktop raptor: apps: [firefox] test: tp6 run-visual-metrics: true subtests: - amazon fetches: fetch: - firefox-addons run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] macosx1100.*shippable.*: [mozilla-central] .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk-only] default: [] default: [] mozharness: extra-options: by-test-platform: windows10-64.*shippable.*-qr.*: - --chimera - --browsertime - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled-webext default: - --chimera - --browsertime - --conditioned-profile=settled-webext # TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3. tier: 3 treeherder-symbol: Btime-webext(tp6) browsertime-indexeddb: description: Raptor (browsertime) tests for IndexedDB max-run-time: 2700 treeherder-symbol: Btime(idb) limit-platforms: by-app: chrome: - .*shippable.* custom-car: - .*64.*shippable.* default: [] raptor: apps: [firefox, chrome] run-visual-metrics: false subtests: - addMab1 - addMabN - addMar1 - addMarN - addMbl1 - addMblN - addkAB1 - addkABN - addkAR1 - addkARN - addkBL1 - addkBLN - getkeyrng run-on-projects: by-app: firefox: by-test-platform: windows.*-32.*: [] .*64.*shippable-qr/.*: [autoland, mozilla-central] default: [] default: [] tier: by-app: firefox: 2 default: 3 mozharness: extra-options: - --extra-profiler-run