diff options
Diffstat (limited to 'taskcluster/ci/test')
| -rw-r--r-- | taskcluster/ci/test/awsy.yml | 75 | ||||
| -rw-r--r-- | taskcluster/ci/test/browsertime-desktop.yml | 591 | ||||
| -rw-r--r-- | taskcluster/ci/test/browsertime-mobile.yml | 371 | ||||
| -rw-r--r-- | taskcluster/ci/test/compiled.yml | 136 | ||||
| -rw-r--r-- | taskcluster/ci/test/firefox-ui.yml | 27 | ||||
| -rw-r--r-- | taskcluster/ci/test/kind.yml | 87 | ||||
| -rw-r--r-- | taskcluster/ci/test/marionette.yml | 55 | ||||
| -rw-r--r-- | taskcluster/ci/test/misc.yml | 273 | ||||
| -rw-r--r-- | taskcluster/ci/test/mochitest.yml | 1032 | ||||
| -rw-r--r-- | taskcluster/ci/test/raptor-chrome.yml | 27 | ||||
| -rw-r--r-- | taskcluster/ci/test/raptor.yml | 90 | ||||
| -rw-r--r-- | taskcluster/ci/test/reftest.yml | 259 | ||||
| -rw-r--r-- | taskcluster/ci/test/talos.yml | 812 | ||||
| -rw-r--r-- | taskcluster/ci/test/test-platforms.yml | 419 | ||||
| -rw-r--r-- | taskcluster/ci/test/test-sets.yml | 431 | ||||
| -rw-r--r-- | taskcluster/ci/test/variants.yml | 316 | ||||
| -rw-r--r-- | taskcluster/ci/test/web-platform.yml | 436 | ||||
| -rw-r--r-- | taskcluster/ci/test/xpcshell.yml | 193 |
18 files changed, 5630 insertions, 0 deletions
diff --git a/taskcluster/ci/test/awsy.yml b/taskcluster/ci/test/awsy.yml new file mode 100644 index 0000000000..c9925dd458 --- /dev/null +++ b/taskcluster/ci/test/awsy.yml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: awsy + max-run-time: 7200 + instance-size: xlarge + virtualization: + by-test-platform: + windows10-64-2004(?:-shippable)?-qr/.*: virtual-with-gpu + default: virtual + allow-software-gl-layers: false + mozharness: + script: awsy_script.py + config: + by-test-platform: + windows.*: + - awsy/taskcluster_windows_config.py + macosx.*/opt: + - awsy/macosx_config.py + default: + - awsy/linux_config.py + tier: default + test-manifest-loader: null # don't load tests in the taskgraph + + +awsy: + description: Are we slim yet - tp5 pageset + treeherder-symbol: SY(sy) + run-on-projects: [] + +awsy-tp6: + description: Are we slim yet - tp6 pageset + treeherder-symbol: SY(sy-tp6) + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition + windows.*-32.*/opt: [] + (linux|windows10-64-2004|mac)(?!.*shippable).*/opt: [] + default: [integration, mozilla-central, mozilla-beta, mozilla-release] + mozharness: + extra-options: + - --tp6 + +awsy-dmd: + description: Are we slim yet - dmd enabled + treeherder-symbol: SY(sy-d) + run-on-projects: [] + mozharness: + extra-options: + - --dmd + +awsy-base: + description: Are we slim yet - about:blank base case + treeherder-symbol: SY(ab) + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition. + windows.*-32.*/opt: [] + (linux|windows10-64-2004|mac)(?!.*shippable).*/opt: [] + default: [integration, mozilla-central, mozilla-beta, mozilla-release] + tier: default + mozharness: + extra-options: + - --base + +awsy-base-dmd: + description: Are we slim yet - about:blank base case - dmd enabled + treeherder-symbol: SY(ab-d) + run-on-projects: [] + mozharness: + extra-options: + - --base + - --dmd diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..2e6415fcde --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,591 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: [mozilla-central] + default: [] + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + - --conditioned-profile=settled + +browsertime-tp6: + <<: &tp6-defaults + description: Raptor (browsertime) tp6 page-load tests + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + default: [] + max-run-time: + by-subtest: + imgur: 4800 + default: 4000 + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-subtest: + netflix: # Bug 1756212 - Disable temporarily due to perma on mac + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64).*shippable-qr.*: [trunk, mozilla-beta, mozilla-release] + macosx1015.*shippable-qr.*: [] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + treeherder-symbol: Btime(tp6) + raptor: + apps: [firefox, chrome, chromium] + test: tp6 + run-visual-metrics: true + subtests: + - buzzfeed + - ebay + - espn + - expedia + - [facebook, fb] + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - imdb + - imgur + - linkedin + - microsoft + - netflix + - nytimes + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - wikia + - youtube + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-responsiveness: + <<: *tp6-defaults + raptor: + apps: [firefox, chrome, chromium] + test: tp6 + run-visual-metrics: true + subtests: + - cnn-nav + - [facebook-nav, fbnav] + - [reddit-billgates-ama, rdt-ama] + - [reddit-billgates-post-1, rdt-post-1] + - [reddit-billgates-post-2, rdt-post-2] + mozharness: + extra-options: + by-test-platform: + windows10-.*: + - --cold + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --cold + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-tp6-essential: + <<: *tp6-defaults + <<: &tp6-essential-defaults + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --collect-perfstats + - --extra-profiler-run + - --extra-summary-methods=mean + - --extra-summary-methods=geomean + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + - --extra-summary-methods=mean + - --extra-summary-methods=geomean + raptor: + apps: [firefox, chrome, chromium] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central, mozilla-beta] + .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + tier: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: + by-app: + firefox: 1 + default: 3 + +browsertime-tp6-bytecode: + <<: *tp6-defaults + <<: *tp6-essential-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + default: + - --chimera + - --browsertime + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + tier: + by-app: + firefox: 2 + default: 3 + treeherder-symbol: Btime-cache(tp6) + +browsertime-benchmark: + description: Raptor (browsertime) Benchmark tests + raptor: + apps: [firefox, chrome, chromium, safari] + subtests: + by-app: + firefox: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [twitch-animation, tw-a] + - [unity-webgl, ugl] + - [webaudio, wa] + safari: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + # - [matrix-react-bench, mrb] (Bug 1802922) + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + # - [webaudio, wa] (Bug 1802922) + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + treeherder-symbol: Btime() + tier: + by-app: + firefox: + by-subtest: + motionmark-animometer: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: 1 + unity-webgl: 3 + default: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + default: + by-subtest: + unity-webgl: 3 + default: 2 + run-on-projects: + by-subtest: + speedometer3: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-release] + default: [] + default: [] + unity-webgl: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-release] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release] + default: [] + default: [] + max-run-time: + by-subtest: + ares6: 1500 + assorted-dom: + by-test-platform: + .*-qr/.*: 2100 + default: 1500 + jetstream2: 8000 + matrix-react-bench: 1500 + motionmark-htmlsuite: 1500 + unity-webgl: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-benchmark-wasm: + description: Raptor (browsertime) WASM benchmarks + raptor: + apps: [firefox, chrome, chromium] # Safari depends on Bug 1802922 + subtests: + by-app: + firefox: + - [wasm-godot, godot] + - [wasm-godot-baseline, godot-b] + - [wasm-godot-optimizing, godot-i] + - [wasm-misc, wm] + - [wasm-misc-baseline, wm-b] + - [wasm-misc-optimizing, wm-i] + default: + - [wasm-misc, wm] + - [wasm-godot, godot] + treeherder-symbol: Btime() + tier: 2 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [mozilla-central, mozilla-release] + default: [] + default: [] + max-run-time: + by-subtest: + wasm-godot: 1500 + wasm-godot-baseline: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-youtube-playback: + description: Browsertime YouTube Playback on Firefox + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + - [youtube-playback-av1-sfr, ytp-av1-sfr] + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [mozilla-central, mozilla-release] + default: [] + tier: 2 + limit-platforms: + by-subtest: + youtube-playback-av1-sfr: + - macosx1015-64-shippable-qr/opt + default: [] + max-run-time: + by-subtest: + youtube-playback-hfr: 3600 + youtube-playback-widevine.*: 3600 + default: 2700 + +browsertime-tp6-live: + <<: *tp6-defaults + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=15 + - --chimera + - --live-sites + - --extra-profiler-run + raptor: + apps: [firefox, chrome, chromium, safari] + test: tp6 + run-visual-metrics: true + subtests: + by-app: + safari: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + # - fandom (Bug 1802922) + - [google-docs, gdocs] + # - [google-mail, gmail] (Bug 1802922) + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (Bug 1802922) + # - instagram (see bug 1736999) + - linkedin + - microsoft + # - netflix (Bug 1802922) + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + # - yahoo-mail (Bug 1802922) + - youtube + default: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + # - instagram (see bug 1736999) + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-live-sheriffed: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - cnn + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=5 + - --chimera + - --live-sites + - --extra-profiler-run + run-on-projects: + by-test-platform: + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows|macosx1015)(?!.*-qr).*: [] + windows.*-32.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-profiling: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: [amazon] + description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling + mozharness: + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + treeherder-symbol: Btime-Prof(tp6) + +browsertime-custom: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [[process-switch, ps]] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [mozilla-central] + default: [] + tier: 2 + treeherder-symbol: Btime(ps) + +browsertime-first-install: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [welcome] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + default: [] + tier: 2 + treeherder-symbol: Btime(welcome) + +browsertime-grandprix: + raptor: + apps: [firefox, chrome, chromium, safari] + subtests: [grandprix] + description: Raptor (browsertime) grandprix benchmark test + max-run-time: 2700 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime(grandprix) diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml new file mode 100644 index 0000000000..9dea0cbdee --- /dev/null +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -0,0 +1,371 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: raptor + variants: + - geckoview-no-fission + run-on-projects: [] + tier: 3 + test-manifest-loader: null # don't load tests in the taskgraph + target: + by-app: + fenix: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.fenix.nightly.latest.arm64-v8a + name: build/arm64-v8a/target.apk + default: + index: mobile.v2.fenix.nightly.latest.armeabi-v7a + name: build/armeabi-v7a/target.apk + refbrow: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.reference-browser.raptor.latest.arm64-v8a + name: target.arm64-v8a.apk + default: + index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a + name: target.armeabi-v7a.apk + default: geckoview_example.apk + virtualization: hardware + raptor: + activity: + by-app: + fenix: org.mozilla.fenix.IntentReceiverActivity + geckoview: org.mozilla.geckoview_example.GeckoViewActivity + refbrow: org.mozilla.reference.browser.BrowserTestActivity + default: None + binary-path: + by-app: + fenix: org.mozilla.fenix + geckoview: org.mozilla.geckoview_example + refbrow: org.mozilla.reference.browser.raptor + default: None + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + # Bug 1635749, disable window recorder temporarily + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + optimization: + skip-unless-backstop: null + limit-platforms: + by-app: + chrome-m: + - .*shippable-qr.* + default: [] + +browsertime-tp6m: + <<: &tp6m-defaults + description: Raptor (browsertime) tp6 page-load tests on android + mozharness: + extra-options: + - --chimera + tier: + by-app: + geckoview: 1 + refbrow: 2 + default: 3 + treeherder-symbol: Btime(tp6m) + raptor: + apps: [geckoview, fenix, chrome-m, refbrow] + run-visual-metrics: true + test: tp6m + subtests: + - amazon + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - cnn + - [cnn-ampstories, cnn-amp] + - dailymail + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - [facebook-cristiano, fb-cris] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - reddit + - sina + - [stackoverflow, stacko] + - web-de + - wikipedia + - youtube + run-on-projects: + by-app: + geckoview: + by-variant: + geckoview-no-fission: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [trunk] + default: [] + default: + by-subtest: + wikipedia: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [trunk] + default: [] + google-search-restaurants: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [trunk] + default: [] + default: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [mozilla-central] + default: [] + default: [] + +browsertime-tp6m-essential: + <<: *tp6m-defaults + raptor: + apps: [geckoview, fenix, chrome-m, refbrow] + run-visual-metrics: true + test: tp6m + subtests: + - allrecipes + - [amazon-search, amazon-s] + - espn + - facebook + - google + - [microsoft-support, micros-sup] + - [youtube-watch, youtube-w] + run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-(g5|a51).*shippable.*: [trunk] + default: [] + default: [] + tier: + by-app: + geckoview: 1 + refbrow: 2 + default: 3 + +browsertime-youtube-playback-mobile: + description: Browsertime YouTube Playback on Android + treeherder-symbol: Btime(ytp) + run-without-variant: false + raptor: + apps: [fenix, geckoview, refbrow] + subtests: + by-test-platform: + android-hw-(g5|a51).*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + # Bug 1699469 - Disabled until a crash is fixed. + # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + # - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + android-hw-p2.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + # Bug 1699469 - Disabled until a crash is fixed. + # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + # - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + # - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + test-url-param: + by-subtest: + youtube-playback-h264-sfr: # remove high resolution tests + exclude=1,2, + 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38 + youtube-playback-hfr: + by-test-platform: + android-hw-(g5|a51).*: # remove VP9(1-34) and AV1(51-74) tests + exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74 + android-hw-p2.*: # remove AV1(51-74) tests + exclude=1,2, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74 + default: "" + youtube-playback-widevine-hfr: + by-test-platform: + android-hw-(g5|a51).*: # remove VP9(1-34) tests + exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34 + default: "" + default: "" + run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-(g5|a51).*shippable.*: [mozilla-central] + default: [] + default: [] + max-run-time: 3600 + +browsertime-tp6m-live: + <<: *tp6m-defaults + run-without-variant: false + raptor: + apps: [chrome-m, fenix, geckoview] + run-visual-metrics: true + test: tp6m + subtests: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - booking-sf + - cnn + - [cnn-ampstories, cnn-amp] + - dailymail + - discord + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - expedia + - facebook + - [facebook-cristiano, fb-cris] + - [fashionbeans, fashbea] + - google + - [google-accounts, gacc] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - [imdb-firefox, imdb-f] + - [medium-article, medium] + - [microsoft-support, micros-sup] + - nytimes + - [people-article, people] + - reddit + - [reddit-thread, reddit-thr] + - rumble-fox + - [stackoverflow, stacko] + - [stackoverflow-question, stack-q] + - [urbandictionary-define, urbict-def] + - web-de + - [wikia-marvel, wikia-m] + - wikipedia + - youtube + - [youtube-watch, youtube-w] + mozharness: + extra-options: + - --live-sites + - --chimera + run-on-projects: + by-subtest: + cnn-ampstories: + by-app: + geckoview: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [autoland] + default: [] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime-live(tp6m) + +browsertime-tp6m-profiling: + description: Raptor (browsertime) tp6 page-load tests on android with Gecko profiling + run-without-variant: false + raptor: + apps: [geckoview] + test: tp6m + subtests: [amazon, youtube] + treeherder-symbol: Btime-Prof(tp6m) + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + android-hw-(a51).*aarch64-shippable-qr/opt: [mozilla-central] + default: [] + +browsertime-speedometer-mobile: + description: Browsertime Speedometer on Android + run-without-variant: false + raptor: + apps: [geckoview, fenix, chrome-m] + tier: + by-app: + fenix: 1 + default: 3 + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + android-hw-(g5|a51).*shippable-qr.*: [trunk] + default: [] + default: [] + treeherder-symbol: Btime(sp) + mozharness: + extra-options: + - --test=speedometer + +browsertime-speedometer3-mobile: + description: Browsertime Speedometer 3 on Android + raptor: + apps: [geckoview, fenix, chrome-m] + tier: + by-app: + fenix: 2 + geckoview: 2 + default: 3 + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + android-hw-a51.*shippable-qr.*: [trunk] + default: [] + default: [] + treeherder-symbol: Btime(sp3) + mozharness: + extra-options: + - --test=speedometer3 + +browsertime-unity-webgl-mobile: + description: Browsertime Unity WebGL on Android + run-without-variant: false + raptor: + apps: [geckoview, refbrow, fenix, chrome-m] + treeherder-symbol: Btime(ugl) + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + android-hw-a51-.*aarch64-shippable-qr.*: [trunk] + default: [] + default: + by-test-platform: + android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central] + default: [] + max-run-time: + by-test-platform: + anrdroid-hw-a51.*: 1500 + default: 900 + mozharness: + extra-options: + - --test=unity-webgl + +browsertime-power: + description: Browsertime Power Usage Tests on Android + run-without-variant: false + raptor: + apps: [geckoview, fenix, refbrow] + subtests: + - idle + - idle-bg + - [speedometer, sp] + tier: 2 + run-on-projects: [] + treeherder-symbol: Btime-P(power) + mozharness: + extra-options: + - --power-test diff --git a/taskcluster/ci/test/compiled.yml b/taskcluster/ci/test/compiled.yml new file mode 100644 index 0000000000..72657c128b --- /dev/null +++ b/taskcluster/ci/test/compiled.yml @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + run-without-variant: false + variants: + - 1proc + virtualization: virtual + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + supports-artifact-builds: false + +cppunit: + description: "CPP Unit Tests" + suite: cppunittest + treeherder-symbol: cppunit + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + tier: default + run-on-projects: + by-variant: + 1proc: built-projects + default: [] + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: GTest + instance-size: xlarge + run-on-projects: + by-variant: + 1proc: + by-test-platform: + .*android.*: built-projects + .*-devedition.*/.*: [] # don't run on devedition + .*tsan.*: [] + linux1804-32-qr/opt: [] # try only, shippable on m-c + linux1804-32-qr/debug: ['mozilla-central'] # try and m-c + linux1804-32-shippable-qr/opt: ['mozilla-central'] # try and m-c + default: built-projects + default: [] + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + tier: default + +jittest: + description: "JIT Test run" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-variant: + 1proc: + by-test-platform: + android-hw-.*-aarch64-qr/debug: ['mozilla-central', 'release'] + android-hw-.*-arm7-qr/debug: ['mozilla-central', 'release'] + default: built-projects + default: [] + chunks: + by-test-platform: + windows.*: 1 + windows10-64-2004-ccov-qr/opt: 8 + macosx.*/opt: 1 + macosx.*/debug: 3 + android.*: 10 + default: 6 + max-run-time: + by-test-platform: + windows10-64-2004-ccov.*/.*: 7200 + macosx.*-ccov.*/.*: 7200 + android-hw.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + windows.*: false + macosx.*: false + default: true + target: + by-test-platform: + android-.*: geckoview-test_runner.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default + +jittest-all: + description: "JIT Test run with all flags for android arm7 opt" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-test-platform: + android-hw-.*-arm7(?:-shippable)?-qr/.*: ['mozilla-central', 'release'] + default: [] + chunks: 10 + max-run-time: 3600 + mozharness: + chunked: true + extra-options: + by-test-platform: + android.*: + - --jittest-flags=all + default: None + target: + by-test-platform: + android-.*: geckoview-test_runner.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default diff --git a/taskcluster/ci/test/firefox-ui.yml b/taskcluster/ci/test/firefox-ui.yml new file mode 100644 index 0000000000..c45bb3d138 --- /dev/null +++ b/taskcluster/ci/test/firefox-ui.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: firefox-ui + max-run-time: 5400 + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: built-projects + mozharness: + script: firefox_ui_tests.py + config: + by-test-platform: + windows.*: + - firefox_ui_tests/taskcluster_windows.py + macosx.*: + - firefox_ui_tests/taskcluster.py + - firefox_ui_tests/taskcluster_mac.py + linux.*: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + +firefox-ui-functional: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn(en-US) + tier: 2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml new file mode 100644 index 0000000000..17730d8881 --- /dev/null +++ b/taskcluster/ci/test/kind.yml @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.test:loader + +kind-dependencies: + - build + - build-signing + - repackage-signing-shippable-l10n-msix + - repackage-signing-msix + - repackage + - fetch + - toolchain + +transforms: + - gecko_taskgraph.transforms.test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +# Each stanza in a file pointed to by 'jobs-from' describes a particular test +# suite or sub-suite. These are processed through the transformations described +# above to produce a bunch of tasks. See the schema in +# `taskcluster/gecko_taskgraph/transforms/test/__init__.py` for a description of the fields +# used in these files. + +# Adding a new test type or running tests on a new platform? Be sure to review +# https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy + +jobs-from: + - awsy.yml + - browsertime-desktop.yml + - browsertime-mobile.yml + - compiled.yml + - firefox-ui.yml + - marionette.yml + - misc.yml + - mochitest.yml + - raptor.yml + - raptor-chrome.yml + - reftest.yml + - talos.yml + - web-platform.yml + - xpcshell.yml + +job-defaults: + attributes: + retrigger: true + require-signed-extensions: + by-release-type: + release|esr.*: true + beta: + by-test-platform: + .*-devedition/.*: false + default: true + default: false + mozharness: + requires-signed-builds: + by-test-platform: + macosx1100.*: true + windows10-64-2004-shippable-qr/opt: + by-variant: + msix: true + default: false + windows10-64-2004-qr/opt: + by-variant: + msix: true + default: false + default: false + fetches: + toolchain: + by-test-platform: + win.*: + - win32-minidump-stackwalk + - win32-fix-stacks + macosx.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks diff --git a/taskcluster/ci/test/marionette.yml b/taskcluster/ci/test/marionette.yml new file mode 100644 index 0000000000..a8041b2fb6 --- /dev/null +++ b/taskcluster/ci/test/marionette.yml @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: marionette + mozharness: + script: marionette.py + config: + by-test-platform: + macosx.*: + - marionette/prod_config.py + - marionette/mac_taskcluster_config.py + windows.*: + - marionette/windows_taskcluster_config.py + default: + - marionette/prod_config.py + - remove_executables.py + extra-options: + - --setpref=toolkit.asyncshutdown.log=true + test-manifest-loader: null # don't load tests in the taskgraph + tier: + by-variant: + headless: 2 + default: default + variants: + - webrender-sw + - headless + run-on-projects: + by-variant: + headless: + by-test-platform: + (linux.*64)(-shippable)?-qr/opt: built-projects + default: [] + webrender-sw: + by-test-platform: + .*/debug: built-projects + default: [] + default: + by-test-platform: + .*tsan.*: [] + default: built-projects + virtualization: virtual + +marionette: + description: "Marionette unittest run" + treeherder-symbol: Mn + max-run-time: + by-test-platform: + .*asan.*: 5400 + .*ccov.*: 3600 + .*debug.*: 3600 + default: 2700 + instance-size: default + chunks: 1 diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml new file mode 100644 index 0000000000..c6f6d6d04a --- /dev/null +++ b/taskcluster/ci/test/misc.yml @@ -0,0 +1,273 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + +geckoview-junit: + description: "Geckoview junit run" + suite: geckoview-junit + variants: ["geckoview-e10s-single", "geckoview-fission", "fission-no-isolate", "no-fission"] + treeherder-symbol: gv-junit + loopback-video: true + target: geckoview-androidTest.apk + max-run-time: 3600 + run-on-projects: + by-variant: + geckoview-e10s-single: ['trunk'] + geckoview-fission: ['trunk'] + fission-no-isolate: ['trunk'] + no-fission: built-projects + default: [] + tier: default + mozharness: + script: android_emulator_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + extra-options: + - --test-suite=geckoview-junit + +telemetry-tests-client: + description: "Telemetry tests client run" + suite: telemetry-tests-client + treeherder-symbol: tt(c) + run-on-projects: built-projects + max-run-time: 1200 + tier: default + mozharness: + script: telemetry/telemetry_client.py + config: + by-test-platform: + linux.*: + - remove_executables.py + windows.*: [] + macosx.*: [] + +test-verify: + description: "Extra verification of tests modified on this push" + suite: test-verify + treeherder-symbol: TV + loopback-video: true + max-run-time: 10800 + allow-software-gl-layers: false + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: + by-test-platform: + # do not run on ccov or asan or shippable + .*(-ccov|-asan|-shippable).*: [] + # we prefer -qr platforms with fis, no need to run on both + (?!.*(-ccov|-asan|-shippable))(?!.*-qr).*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + android.*: ['integration'] + default: [] + default: + by-test-platform: + # do not run on ccov or asan or shippable + .*(-ccov|-asan|-shippable).*: [] + # we prefer -qr platforms with fis, no need to run on both + (?!.*(-ccov|-asan|-shippable))(?!.*-qr).*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + android.*: [] + default: ['integration'] + built-projects-only: true + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + optimization: + test-verify: ["test-verify"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-verify-gpu: + description: "Extra verification of tests modified on this push on gpu instances" + suite: + category: test-verify + name: test-verify-gpu + treeherder-symbol: TVg + loopback-video: true + virtualization: virtual-with-gpu + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + linux.*: ['integration'] + default: [] + built-projects-only: true + optimization: + test-verify: ["test-verify-gpu"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + - --gpu-required + +test-coverage: + description: "Per-test coverage" + suite: test-coverage + treeherder-symbol: TC + loopback-video: true + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-coverage-gpu: + description: "Per-test coverage for tests that require gpu instances" + suite: test-coverage + treeherder-symbol: TCg + loopback-video: true + virtualization: virtual-with-gpu + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # only run on mozilla-central and try. + .*-ccov.*/.*: ['mozilla-central'] + default: [] + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + - --gpu-required diff --git a/taskcluster/ci/test/mochitest.yml b/taskcluster/ci/test/mochitest.yml new file mode 100644 index 0000000000..8dbc12960f --- /dev/null +++ b/taskcluster/ci/test/mochitest.yml @@ -0,0 +1,1032 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: mochitest + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + android-hw.*: geckoview-test_runner.apk + windows10-64-2004-shippable-qr/opt: + by-variant: + msix: target.installer.msix + default: null + windows10-64-2004-qr/opt: + by-variant: + msix: target.installer.msix + default: null + default: null + tier: + by-variant: + xorigin: + by-test-platform: + linux.*64.*-qr/debug: 1 + default: 2 + default: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +mochitest-plain: + description: "Mochitest plain run" + treeherder-symbol: M() + schedules-component: mochitest-plain + loopback-video: true + virtualization: virtual + variants: + - aab+no-fission + - conditioned_profile + - xorigin + - no-fission + - socketprocess_networking + - wayland + - webrender-sw+no-fission + - webrender-sw + - headless + - headless+socketprocess_networking + run-on-projects: + by-variant: + conditioned_profile: + by-test-platform: + linux1804-64(-shippable)?-qr/opt: ['trunk'] + windows10-64-2004(-shippable)?-qr/opt: ['trunk'] + default: [] + xorigin: + by-test-platform: + (linux.*64|macosx.*64|windows10.*2004)(-shippable)?-qr/(opt|debug|asan|tsan): ['trunk'] + windows.*64-ccov-qr/opt: ['mozilla-central'] + linux.*64-ccov-qr/opt: ['mozilla-central'] + default: [] + socketprocess_networking: + by-test-platform: + (linux.*64|macosx.*64|windows10-64-2004)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + wayland: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*asan.*: ['trunk'] + linux.*64(-shippable)?-qr/(opt|debug): ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + aab+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + headless: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['mozilla-central'] + default: [] + headless+socketprocess_networking: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['mozilla-central'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + chunks: + by-test-platform: + android-em-7.*: 4 + linux.*/debug: 16 + linux.*64-asan/opt: 10 + linux.*64-tsan-qr/opt: 20 + linux.*64-.*cov/opt: 10 + windows10-64-2004-ccov.*/.*: 10 + macosx.*64-ccov.*/.*: 10 + default: 5 + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB + default: default + max-run-time: 5400 + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android-em.*: + - --test-suite=mochitest-plain + default: [] + chunked: + by-test-platform: + android-em.*: false + default: true + +mochitest-a11y: + description: "Mochitest a11y run" + treeherder-symbol: M(a11y) + schedules-component: mochitest-a11y + test-manifest-loader: default # ensure we don't run with manifest-scheduling + loopback-video: true + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - wayland+1proc + - webrender-sw+1proc + - a11y-no-cache+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: + by-test-platform: + linux.*64-qr/debug: ['autoland', 'mozilla-central'] + default: [] + wayland+1proc: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + a11y-no-cache+1proc: ['trunk'] + 1proc: built-projects + mozharness: + mochitest-flavor: a11y + +mochitest-browser-chrome: + description: "Mochitest browser-chrome run" + suite: + name: mochitest-browser-chrome + treeherder-symbol: M(bc) + schedules-component: mochitest-browser-chrome + loopback-video: true + variants: + - webrender-sw + - webrender-sw+a11y-checks + - webrender-sw+wayland + - socketprocess_networking + - msix + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + linux.*64(-asan)?-qr/(opt|debug): ['all'] + linux.*64-tsan-qr/opt: built-projects + linux.*64-shippable-qr/opt: ['release'] + linux.*64(-shippable)?-qr/opt: built-projects + linux.*64-devedition-qr/opt: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + webrender-sw+a11y-checks: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + (linux.*64|macosx.*64|windows10-64-2004)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + webrender-sw+wayland: [] + msix: + by-test-platform: + windows10-64-2004-shippable-qr/opt: built-projects + windows10-64-2004-qr/opt: built-projects + default: [] + default: + by-test-platform: + android.*: [] + linux.*: [] # linux runs M-swr(bc), not M(bc) + default: built-projects + tier: + by-variant: + webrender-sw: + by-test-platform: + linux.*64(-asan|-shippable)?-qr/(opt|debug): 1 + default: default + default: default + chunks: + by-test-platform: + linux.*-qr/debug: 16 + linux.*64-asan-qr/opt: 16 + linux.*64-tsan-qr/opt: 32 + macosx.*64/debug: 16 + windows10-64-2004-ccov.*/.*: 14 + windows10.*-asan-qr/opt: 12 + windows10-32-qr/debug: 12 + windows7-32.*: 20 + default: 7 + max-run-time: + by-variant: + msix: 5400 + default: + by-test-platform: + linux.*64-ccov.*/.*: 9000 + windows10-64-2004-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 10800 + linux.*64-tsan-qr/opt: 10800 + linux.*-qr/opt: 5400 + .*/debug: 5400 + default: 3600 + mozharness: + mochitest-flavor: browser + chunked: true + # Bug 1281241: migrating to m3.large instances + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB + default: default + allow-software-gl-layers: false + +mochitest-browser-a11y: + description: "Mochitest browser-a11y run" + suite: + name: mochitest-browser-a11y + treeherder-symbol: M(ba) + schedules-component: mochitest-browser-a11y + loopback-video: true + variants: + - a11y-no-cache + run-on-projects: + by-variant: + a11y-no-cache: + by-test-platform: + android.*: [] + default: ["trunk"] + default: + by-test-platform: + android.*: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + .*tsan.*: 2700 + default: 1800 + mozharness: + mochitest-flavor: browser + instance-size: default + allow-software-gl-layers: false + virtualization: virtual + +mochitest-browser-media: + description: "Mochitest browser media" + suite: + name: mochitest-browser-media + treeherder-symbol: M(mda-bc) + schedules-component: mochitest-browser-media + loopback-video: true + run-on-projects: + by-test-platform: + android.*: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + .*tsan.*: 1800 + default: 1200 + mozharness: + mochitest-flavor: browser + instance-size: default + allow-software-gl-layers: false + virtualization: virtual + +browser-screenshots: + description: "Browser Screenshots" + suite: + name: mochitest-browser-chrome-screenshots + treeherder-symbol: M(ss) + loopback-video: true + test-manifest-loader: null # don't load tests in the taskgraph + variants: ["no-fission"] + run-on-projects: + by-test-platform: + linux1804-64.*/opt: ['trunk'] + default: [] + max-run-time: 3600 + mozharness: + mochitest-flavor: browser + allow-software-gl-layers: false + +mochitest-chrome: + description: "Mochitest chrome run" + treeherder-symbol: M(c) + schedules-component: mochitest-chrome + loopback-video: true + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - wayland+1proc + - webrender-sw+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: + by-test-platform: + (linux.*64|macosx.*64|windows10-64-2004)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + wayland+1proc: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + 1proc: built-projects + chunks: + by-test-platform: + .*tsan.*/opt: 6 + default: 3 + max-run-time: 3600 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory otherwise using 8GB + default: default + +mochitest-devtools-chrome: + description: "Mochitest devtools-chrome run" + suite: + name: mochitest-devtools-chrome + schedules-component: mochitest-browser-chrome + treeherder-symbol: M(dt) + loopback-video: true + variants: + - a11y-checks + - socketprocess_networking + - wayland + - webrender-sw + - devtools-no-eft+no-fission + run-on-projects: + by-variant: + a11y-checks: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + devtools-no-eft+no-fission: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + (linux.*64|macosx.*64|windows10-64-2004)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + wayland: + by-test-platform: + linux.*64-ccov-qr/opt: [] + linux.*64(-shippable)?-qr/(opt|debug): built-projects + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + default: + by-test-platform: + android.*: [] + linux.*64-ccov-qr/opt: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + windows10-64-2004-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 9000 + linux.*64-ccov.*/.*: 7200 + linux.*64-tsan-qr/opt: 7200 + default: 5400 + chunks: + by-test-platform: + .*-ccov.*/.*: 12 + linux.*64-qr/debug: 12 + macosx.*64-qr/debug: 8 + .*-asan(-qr)?/opt: 8 + .*-tsan-qr/opt: 16 + default: 5 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-[at]san(-qr)?/opt: xlarge # runs out of memory otherwise using 8GB + default: default + # Bug 1296086: high number of intermittents observed with software GL and large instances + allow-software-gl-layers: false + +mochitest-plain-gpu: + description: "Mochitest plain GPU run" + suite: + name: mochitest-plain-gpu + treeherder-symbol: M(gpu) + schedules-component: mochitest-plain + loopback-video: true + variants: + - no-fission + - wayland + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + wayland: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*asan.*: ['trunk'] + linux.*64(-shippable)?-qr/(opt|debug): ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux.*64-ccov-qr/opt: [] + linux1804-32.*: [] + .*-tsan-qr/opt: [] + linux.*: ['trunk'] # linux opt/debug/asan + android.*: built-projects + default: [] + default: + by-test-platform: + android.*/.*: [] + default: built-projects + virtualization: virtual-with-gpu + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-plain-gpu + default: + - --mochitest-suite=mochitest-plain-gpu + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory otherwise using 8GB + default: default + +mochitest-chrome-gpu: + description: "Mochitest chrome GPU run" + suite: + name: mochitest-chrome-gpu + treeherder-symbol: M(gpu-c) + loopback-video: true + virtualization: virtual-with-gpu + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - wayland+1proc + - webrender-sw+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: [] + wayland+1proc: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + 1proc: + by-test-platform: + android.*/.*: [] + default: built-projects + tier: default + mozharness: + mochitest-flavor: chrome + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-chrome-gpu + default: + - --mochitest-suite=mochitest-chrome-gpu + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB of memory + default: default + +mochitest-media: + description: "Mochitest media run" + treeherder-symbol: M(mda) + schedules-component: mochitest-plain + max-run-time: + by-test-platform: + windows10-64-2004-ccov.*/.*: 7200 + macosx.*64-ccov.*/.*: 7200 + linux.*64-tsan-qr/opt: 7200 + default: 5400 + variants: + - no-fission + - socketprocess+no-fission + - socketprocess + - wayland + - webgl-ipc + - webrender-sw+no-fission + - webrender-sw + - wmf-media-engine + - msix + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + android.*: [] + linux.*64(-shippable|-asan|-tsan)?-qr/opt: ['autoland', 'mozilla-central'] + macosx.*64(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + windows10-(32|64)-2004(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + windows10-64-2004-asan-qr/opt: ['autoland', 'mozilla-central'] + default: [] + socketprocess+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: ['autoland', 'mozilla-central'] + android-hw-.*-aarch64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + socketprocess: + by-test-platform: + android.*: [] + linux.*64(-asan|-devedition|-shippable)?-qr/(opt|debug): built-projects + linux.*64-tsan-qr/opt: ['autoland', 'mozilla-central'] + macosx.*-64(-devedition|-shippable)?-qr/(opt|debug): built-projects + windows10-(32|64)-2004(-devedition|-shippable)?-qr/(opt|debug): built-projects + windows10-64-2004-asan-qr/opt: ['autoland', 'mozilla-central'] + default: [] + wayland: + by-test-platform: + linux.*64-qr/debug: ['autoland', 'mozilla-central'] + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: ['autoland', 'mozilla-central'] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + android.*: [] + linux1804-64-qr/debug: ['autoland', 'mozilla-central'] + default: [] + no-fission: + by-test-platform: + android-hw-.*/opt: [] + android-hw-.*-arm7.*/(?:debug)?: ['trunk', 'mozilla-beta', 'mozilla-release'] + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + wmf-media-engine: + by-test-platform: + windows10-64-2004.*: ['autoland', 'mozilla-central'] + default: [] + msix: + by-test-platform: + windows10-64-2004-shippable-qr/opt: built-projects + windows10-64-2004-qr/opt: ['autoland'] + default: [] + default: + by-test-platform: + android.*: [] + linux.*(asan|ccov)/opt: [] + .*(ccov/|asan/|-32).*: built-projects + (android|linux|windows10-64-2004|macos)(?!.*-qr).*: [] + default: built-projects + loopback-video: true + chunks: + by-variant: + wmf-media-engine: 1 + default: + by-test-platform: + (android|macos).*/opt: 1 + ^(?!.*asan).*windows10.*/opt: 1 + linux.*/debug: 3 + linux.*64-tsan-qr/opt: 5 + default: 2 + mozharness: + mochitest-flavor: plain + chunked: + by-test-platform: + android.*: false + macosx.*64.*: false + windows10-64-2004.*: false + default: true + extra-options: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + tier: + by-test-platform: + .*ccov.*: 2 + android-em.*: 1 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + android-hw.*: 1 + default: default + +mochitest-valgrind: + description: "Mochitest plain Valgrind run" + suite: + name: mochitest-valgrind-plain + treeherder-symbol: M-V() + run-on-projects: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + loopback-video: true + chunks: 40 + max-run-time: 14400 + run-without-variant: false + variants: ["1proc"] + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + +mochitest-webgl1-core: + description: "Mochitest webgl1-core run" + treeherder-symbol: M(gl1c) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: + by-test-platform: + macosx.*64-ccov.*/.*: 7200 + default: 1800 + variants: + - no-fission + - wayland + - webgl-ipc + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + .*-ccov.*: [] + .*-tsan.*: [] + linux.*-64.*: ["trunk"] + mac.*: ["trunk"] + win.*: ["trunk"] + default: [] + wayland: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable-qr/opt: ['mozilla-central'] + android-hw.*aarch.*-qr/debug: ['mozilla-central'] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*-tsan.*: [] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + +mochitest-webgl1-ext: + description: "Mochitest webgl1-ext run" + treeherder-symbol: M(gl1e) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + chunks: + by-test-platform: + android.*: 2 + default: 1 + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: 2700 + variants: + - no-fission + - webgl-ipc + - wayland + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + .*-ccov.*: [] + .*-tsan.*: [] + linux.*-64.*: built-projects + mac.*: ["trunk"] + windows10-64-2004.*: built-projects + default: [] + wayland: + by-test-platform: + linux.*64(-shippable)?-qr/(opt|debug): built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable-qr/opt: ['mozilla-central'] + android-hw.*aarch.*-qr/debug: ['mozilla-central'] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*-tsan.*: [] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + +mochitest-webgl2-core: + description: "Mochitest webgl2-core run" + treeherder-symbol: M(gl2c) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + chunks: + by-test-platform: + android.*: 2 + default: 1 + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: 1800 + variants: + - no-fission + - webgl-ipc + - wayland + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + .*-ccov.*/.*: [] + .*-tsan.*: [] + android.*: [] + linux.*-64.*: ["trunk"] + mac.*: ["trunk"] + win.*: ["trunk"] + default: [] + wayland: + by-test-platform: + linux.*64(-shippable|-asan)?-qr/(opt|debug): built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable-qr/opt: ['mozilla-central'] + android-hw.*aarch.*-qr/debug: ['mozilla-central'] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*-tsan.*: [] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + +mochitest-webgl2-ext: + description: "Mochitest webgl2-ext run" + treeherder-symbol: M(gl2e) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + - no-fission + - webgl-ipc + - webrender-sw+no-fission + - webrender-sw + - wayland + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + .*ccov.*/.*: [] + .*-tsan.*: [] + linux.*-64.*: ["trunk"] + mac.*: ["trunk"] + win.*: ["trunk"] + default: [] + wayland: + by-test-platform: + linux.*64(-shippable|-asan)?-qr/(opt|debug): built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ["trunk"] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + .*-tsan.*: [] + default: built-projects + chunks: 4 + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: 2700 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + +mochitest-webgl2-deqp: + description: "Mochitest webgl2-deqp run" + treeherder-symbol: M(gl2d) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + - wayland + - webgl-ipc + - webrender-sw + run-on-projects: [] # Don't run this for now. + chunks: 4 + loopback-video: true + max-run-time: 1800 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + +mochitest-webgpu: + description: "Mochitest webgpu run" + treeherder-symbol: M(webgpu) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + loopback-video: true + variants: + - no-fission + - wayland + - webrender-sw+no-fission + run-on-projects: + by-variant: + wayland: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: ["trunk"] + default: [] + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*mingw.*: ["release"] + .*shippable.*: ["mozilla-central"] + .*ccov.*: ['mozilla-central'] + .*tsan.*: ['trunk'] + linux1804-64-qr/debug: ['trunk'] + .*(!-shippable)-qr.*: ["trunk"] + default: [] + max-run-time: + by-test-platform: + macosx.*64-ccov.*/.*: 7200 + default: 1800 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + +mochitest-remote: + description: "Mochitest for the remote agent (/remote folder)" + suite: + name: mochitest-remote + treeherder-symbol: M(remote) + loopback-video: true + variants: + - no-fission + - socketprocess_networking + - wayland + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + socketprocess_networking: + by-test-platform: + (linux.*64|macosx.*64|windows10-64-2004)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: ['all'] + default: [] + wayland: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + tier: default + max-run-time: 5400 + mozharness: + mochitest-flavor: browser + extra-options: + - --setpref=remote.log.level=Trace + virtualization: + by-test-platform: + windows10-64.*: virtual + default: virtual diff --git a/taskcluster/ci/test/raptor-chrome.yml b/taskcluster/ci/test/raptor-chrome.yml new file mode 100644 index 0000000000..892e079159 --- /dev/null +++ b/taskcluster/ci/test/raptor-chrome.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + .*-ux/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + - --webext diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml new file mode 100644 index 0000000000..93614acd50 --- /dev/null +++ b/taskcluster/ci/test/raptor.yml @@ -0,0 +1,90 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-app: + firefox: + by-test-platform: + .*-ref-hw-2017-qr/.*: 3600 + .*-qr/.*: 2400 + default: 1800 + default: + by-test-platform: + .*-qr/.*: 2400 + .*-ux/.*: 2400 + default: 1800 + suite: raptor + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: [] + limit-platforms: + by-app: + chrome: + - .*shippable(?!-qr).* + chromium: + - .*shippable(?!-qr).* + default: [] + tier: 2 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + # Bug 1626522: Temporarily enable extension logging for + # an easier investigation of intermittents on mobile devices + - --setpref=extensions.logging.enabled=true + - --webext + fetches: + fetch: + by-app: + chromium: + by-test-platform: + win.*64.*: + - win64-chromium + win.*32.*: + - win32-chromium + macosx.*: + - mac-chromium + default: + - linux64-chromium + default: [] + +raptor-youtube-playback-v9-power-firefox: + description: Raptor V9 YouTube Playback on Firefox with Power Testing + try-name: raptor-youtube-playback-v9-power-firefox + treeherder-symbol: Rap(ytp-v9-p) + run-on-projects: + by-test-platform: + (linux|windows.*64|macos)(?!.*shippable).*: [] + (linux|windows.*64|macos)(?!.*-qr).*: [] + default: [mozilla-central] + max-run-time: 2700 + tier: 2 + mozharness: + extra-options: + - --test=raptor-youtube-playback-v9 + - --power-test + +raptor-youtube-playback-h264-power-firefox: + description: Raptor H264 YouTube Playback on Firefox with Power Testing + try-name: raptor-youtube-playback-h264-power-firefox + treeherder-symbol: Rap(ytp-h264-p) + run-on-projects: + by-test-platform: + (linux|windows.*64|macos)(?!.*shippable).*: [] + (linux|windows.*64|macos)(?!.*-qr).*: [] + default: [mozilla-central] + max-run-time: 2700 + tier: 2 + mozharness: + extra-options: + - --test=raptor-youtube-playback-h264 + - --power-test diff --git a/taskcluster/ci/test/reftest.yml b/taskcluster/ci/test/reftest.yml new file mode 100644 index 0000000000..213d2ea42d --- /dev/null +++ b/taskcluster/ci/test/reftest.yml @@ -0,0 +1,259 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: reftest + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + android-hw-.*: geckoview-test_runner.apk + default: null + tier: default + test-manifest-loader: null # don't load tests in the taskgraph + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +crashtest: + description: "Crashtest run" + schedules-component: crashtest + treeherder-symbol: R(C) + chunks: + by-test-platform: + android-em-7.*: 1 + linux.*64-tsan-qr/opt: 32 + default: 1 + variants: + - no-fission + - wayland + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + wayland: + by-test-platform: + linux1804-64-qr/debug: built-projects + linux1804-64(-tsan|-asan)-qr/opt: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + macosx101.*64-qr/debug: built-projects + linux.*-64-qr/debug: built-projects + linux.*-64(-tsan|-asan)-qr/opt: built-projects + windows10-64-2004-asan-qr/opt: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + max-run-time: 3600 + tier: + by-test-platform: + android-hw-.*-qr/.*: 2 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + +crashtest-qr: + description: "Crashtest Webrender run" + schedules-component: crashtest + treeherder-symbol: R(C) + chunks: 1 + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: built-projects + default: [] + max-run-time: 1200 + tier: 2 + virtualization: virtual + +jsreftest: + description: "JS Reftest run" + schedules-component: jsreftest + treeherder-symbol: R(J) + instance-size: default + chunks: + by-test-platform: + android-hw.*\/debug: 12 + android-hw.*/(opt|pgo)?: 6 + windows.*\/opt: 2 + windows10-64-2004-asan-qr/opt: 3 + windows10-64-2004-ccov-qr/opt: 5 + linux.*64-ccov.*/.*: 5 + linux1804-64-qr/opt: 4 + linux1804-64-qr/debug: 5 + linux1804-64/debug: 5 + .*-tsan-qr/opt: 8 + macosx.*-64-ccov.*/.*: 5 + macosx.*-64(-shippable)?/opt: 2 + default: 3 + max-run-time: + by-test-platform: + .*ccov.*: 7200 + default: 3600 + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android-hw-.*-aarch64(?:-shippable)?-qr/.*: ['mozilla-central', 'release'] + android-hw-.*-arm7(?:-shippable)?-qr/.*: ['mozilla-central', 'release'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*/.*: [] + default: built-projects + virtualization: virtual + +reftest: + description: "Reftest run" + treeherder-symbol: R(R) + virtualization: virtual-with-gpu + chunks: + by-test-platform: + android-em-7.*: 6 + linux1804-64(-shippable|-devedition|-qr)?/opt: 5 + linux.*64-tsan-qr/opt: 32 + macosx101.*-64-shippable(-qr)?/opt: 3 + macosx101.*-64/debug: 4 + macosx101.*-64(-ccov.*|-qr)/debug: 6 + windows.*/opt: 2 + windows10-64-2004.*/debug: 4 + windows10-32.*/debug: 6 + windows10-64-2004-asan-qr/opt: 3 + windows10-64-2004-ccov-qr/opt: 9 + default: 8 + variants: + - no-fission + - wayland + - webrender-dcomp-disabled + - webrender-dcomp-present + - webrender-dcomp-overlays + - webrender-dcomp-compositing # Should match default + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + wayland: + by-test-platform: + linux1804-64-qr/debug: built-projects + linux1804-64(-asan|-tsan)-qr/opt: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux1804-64-qr/debug: built-projects + linux1804-64(-asan|-tsan)-qr/opt: built-projects + macosx101.*64-qr/debug: built-projects + windows10-64-2004(-asan)?-qr/opt: built-projects + windows7-32(?:-shippable)?-qr/opt: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*/.*: [] + linux.*asan/opt: [] + default: built-projects + max-run-time: + by-test-platform: + windows10-32-mingwclang-qr/debug: 5400 + windows10-64-2004-ccov.*/.*: 5400 + windows10-64-2004-asan-qr/opt: 5400 + macosx.*64-ccov.*/.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + android-em.*: false + macosx.*64/opt: false + windows10-64-2004.*/opt: false + default: true + tier: + by-test-platform: + android-hw-.*-qr/.*: 2 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + +reftest-qr: + description: "Reftest webrender run" + treeherder-symbol: R(R) + chunks: 2 + variants: + - no-fission + - webrender-sw+no-fission + run-without-variant: false + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-hw-p5-13-0-android-aarch64-qr/debug: built-projects + default: [] + no-fission: built-projects + max-run-time: 2400 + mozharness: + chunked: true + tier: 2 + virtualization: virtual + +reftest-snapshot: + description: "Reftest snapshot" + treeherder-symbol: R(Rs) + virtualization: virtual + run-on-projects: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + default: [] + tier: default + chunks: 8 + mozharness: + chunked: true diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml new file mode 100644 index 0000000000..9987bc5b56 --- /dev/null +++ b/taskcluster/ci/test/talos.yml @@ -0,0 +1,812 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: talos + tier: + by-test-platform: + macosx1100.*: 2 + default: default + virtualization: hardware + mozharness: + script: talos_script.py + config: + by-test-platform: + macosx.*: + - talos/mac_config.py + windows.*: + - talos/windows_config.py + default: + - talos/linux_config.py + test-manifest-loader: null # don't load tests in the taskgraph + +talos-bcv: + description: Talos basic compositor video + try-name: bcv + treeherder-symbol: T(bcv) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|macosx1015|windows10)-64-shippable-qr/opt: [mozilla-beta, trunk] + (windows10-64|macosx1015-64).*-qr.*: [trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=bcv + +talos-bcv-profiling: + description: Talos basic compositor video with gecko profiling + try-name: bcv-prof + treeherder-symbol: T-Prof(bcv) + tier: 2 + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=bcv + - --gecko-profile + +talos-chrome: + description: Talos chrome + try-name: chromez + treeherder-symbol: T(c) + variants: + - webrender-sw + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=chromez + +talos-chrome-profiling: + description: Talos chrome with gecko profiling + try-name: chromez-prof + treeherder-symbol: T-Prof(c) + tier: 2 + max-run-time: 1200 + run-on-projects: [] + mozharness: + extra-options: + - --suite=chromez + - --gecko-profile + +talos-damp-inspector: + description: Talos devtools inspector (damp-inspector) + try-name: damp-inspector + treeherder-symbol: T(damp-inspector) + max-run-time: 5400 + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32-shippable.*/.*: [] + (linux|windows|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + mozharness: + extra-options: + - --suite=damp-inspector + +talos-damp-webconsole: + description: Talos devtools webconsole (damp-webconsole) + try-name: damp-webconsole + treeherder-symbol: T(damp-webconsole) + max-run-time: 5400 + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32-shippable.*/.*: [] + (linux|windows|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + mozharness: + extra-options: + - --suite=damp-webconsole + +talos-damp-other: + description: Talos devtools other (damp-other) + try-name: damp-other + treeherder-symbol: T(damp-other) + max-run-time: 5400 + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32-shippable.*/.*: [] + (linux|windows|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + mozharness: + extra-options: + - --suite=damp-other + +talos-damp-inspector-profiling: + description: Talos devtools inspector (damp-inspector) with gecko profiling + try-name: damp-inspector-prof + treeherder-symbol: T-Prof(damp-inspector) + tier: 2 + max-run-time: 5400 + run-on-projects: [] + mozharness: + extra-options: + - --suite=damp-inspector + - --gecko-profile + +talos-damp-webconsole-profiling: + description: Talos devtools webconsole (damp-webconsole) with gecko profiling + try-name: damp-webconsole-prof + treeherder-symbol: T-Prof(damp-webconsole) + tier: 2 + max-run-time: 5400 + run-on-projects: [] + mozharness: + extra-options: + - --suite=damp-webconsole + - --gecko-profile + +talos-damp-other-profiling: + description: Talos devtools other (damp-other) with gecko profiling + try-name: damp-other-prof + treeherder-symbol: T-Prof(damp-other) + tier: 2 + max-run-time: 5400 + run-on-projects: [] + mozharness: + extra-options: + - --suite=damp-other + - --gecko-profile + +talos-dromaeojs: + description: Talos dromaeojs + try-name: dromaeojs + treeherder-symbol: T(d) + max-run-time: 2100 + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + mozharness: + extra-options: + - --suite=dromaeojs + +talos-dromaeojs-profiling: + description: Talos dromaeojs with gecko profiling + try-name: dromaeojs-prof + treeherder-symbol: T-Prof(d) + tier: 2 + max-run-time: 2100 + run-on-projects: [] + mozharness: + extra-options: + - --suite=dromaeojs + - --gecko-profile + +talos-flex: + description: Talos XUL flexbox emulation enabled + try-name: flex + treeherder-symbol: T(f) + tier: 2 + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-central] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=flex + +talos-flex-profiling: + description: Talos XUL flexbox emulation enabled with gecko profiling + try-name: flex-prof + treeherder-symbol: T-Prof(f) + tier: 3 + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + default: [mozilla-central] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=flex + - --gecko-profile + +talos-g1: + description: Talos g1 + try-name: g1 + treeherder-symbol: T(g1) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: + by-test-platform: + linux1804-64.*: 3600 + default: 7200 + mozharness: + extra-options: + - --suite=g1 + +talos-g1-profiling: + description: Talos g1 with gecko profiling + try-name: g1-prof + treeherder-symbol: T-Prof(g1) + tier: 2 + run-on-projects: [] + max-run-time: + by-test-platform: + linux1804-64.*: 3600 + default: 7200 + mozharness: + extra-options: + - --suite=g1 + - --gecko-profile + +talos-g3: + description: Talos g3 + try-name: g3 + treeherder-symbol: T(g3) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 900 + mozharness: + extra-options: + - --suite=g3 + +talos-g3-profiling: + description: Talos g3 with gecko profiling + try-name: g3-prof + treeherder-symbol: T-Prof(g3) + tier: 2 + max-run-time: 900 + run-on-projects: [] + mozharness: + extra-options: + - --suite=g3 + - --gecko-profile + +talos-g4: + description: Talos g4 + try-name: g4 + treeherder-symbol: T(g4) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx1015)-64-shippable-qr/opt: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: + by-test-platform: + linux1804-64.*: 1500 + default: 1800 + mozharness: + extra-options: + - --suite=g4 + +talos-g4-profiling: + description: Talos g4 with gecko profiling + try-name: g4-prof + treeherder-symbol: T-Prof(g4) + tier: 2 + max-run-time: + by-test-platform: + linux1804-64.*: 1500 + default: 1800 + run-on-projects: [] + mozharness: + extra-options: + - --suite=g4 + - --gecko-profile + +talos-g5: + description: Talos g5 + try-name: g5 + treeherder-symbol: T(g5) + mozharness: + extra-options: + - --suite=g5 + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: + by-test-platform: + linux1804-64.*: 1200 + macosx.*: 2400 + default: 1800 + +talos-g5-profiling: + description: Talos g5 with gecko profiling + try-name: g5-prof + treeherder-symbol: T-Prof(g5) + tier: 2 + run-on-projects: [] + mozharness: + extra-options: + - --suite=g5 + - --gecko-profile + max-run-time: + by-test-platform: + linux1804-64.*: 1200 + default: 1800 + +talos-h1: + description: Talos h1 + try-name: h1 + treeherder-symbol: T(h1) + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + mozharness: + extra-options: + - --suite=h1 + max-run-time: + by-test-platform: + linux1804-64.*: 900 + default: 1800 + +talos-h2: + description: Talos h2 + try-name: h2 + treeherder-symbol: T(h2) + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: + by-test-platform: + linux1804-64.*: 900 + default: 7200 + mozharness: + extra-options: + - --suite=h2 + +talos-motionmark: + description: Talos motionmark + try-name: motionmark + treeherder-symbol: T(mm) + run-on-projects: + by-test-platform: + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-central] + max-run-time: 3600 + tier: 2 + mozharness: + extra-options: + - --suite=motionmark + +talos-motionmark-profiling: + description: Talos motionmark with gecko profiling + try-name: motionmark-prof + treeherder-symbol: T-Prof(mm) + run-on-projects: [] + max-run-time: 3600 + tier: 2 + mozharness: + extra-options: + - --suite=motionmark + - --gecko-profile + +talos-other: + description: Talos other + try-name: other + treeherder-symbol: T(o) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=other + +talos-other-profiling: + description: Talos other with gecko profiling + try-name: other-prof + treeherder-symbol: T-Prof(o) + run-on-projects: [] + tier: 2 + max-run-time: 1500 + mozharness: + extra-options: + - --suite=other + - --gecko-profile + +talos-sessionrestore-many-windows: + description: Talos sessionrestore-many-windows + try-name: sessionrestore-many-windows + treeherder-symbol: T(smw) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-central, autoland] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-central, autoland] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=sessionrestore-many-windows + +talos-sessionrestore-many-windows-profiling: + description: Talos sessionrestore-many-windows with gecko profiling + try-name: sessionrestore-many-windows-prof + treeherder-symbol: T-Prof(smw) + tier: 2 + run-on-projects: [] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=sessionrestore-many-windows + - --gecko-profile + +talos-perf-reftest: + description: Talos perf-reftest + try-name: perf-reftest + treeherder-symbol: T(p) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1200 + mozharness: + extra-options: + - --suite=perf-reftest + +talos-perf-reftest-profiling: + description: Talos perf-reftest with gecko profiling + try-name: perf-reftest-prof + treeherder-symbol: T-Prof(p) + tier: 2 + run-on-projects: [] + max-run-time: 1200 + mozharness: + extra-options: + - --suite=perf-reftest + - --gecko-profile + +talos-perf-reftest-singletons: + description: Talos perf-reftest singletons + try-name: perf-reftest-singletons + treeherder-symbol: T(ps) + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=perf-reftest-singletons + +talos-perf-reftest-singletons-profiling: + description: Talos perf-reftest singletons with gecko profiling + try-name: perf-reftest-singletons-prof + treeherder-symbol: T-Prof(ps) + tier: 2 + run-on-projects: [] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=perf-reftest-singletons + - --gecko-profile + +talos-svgr: + description: Talos svgr + try-name: svgr + treeherder-symbol: T(s) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=svgr + +talos-svgr-profiling: + description: Talos svgr with gecko profiling + try-name: svgr-prof + treeherder-symbol: T-Prof(s) + tier: 2 + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=svgr + - --gecko-profile + +talos-tp5o: + description: Talos tp5o + try-name: tp5o + treeherder-symbol: T(tp) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=tp5o + +talos-tp5o-profiling: + description: Talos tp5o with gecko profiling + try-name: tp5o-prof + treeherder-symbol: T-Prof(tp) + tier: 2 + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=tp5o + - --gecko-profile + +talos-tabswitch: + description: Talos page scroll (tabswitch) + try-name: tabswitch + treeherder-symbol: T(tabswitch) + variants: + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx.*)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=tabswitch + +talos-tabswitch-profiling: + description: Talos page scroll (tabswitch) with gecko profiling + try-name: tabswitch-prof + treeherder-symbol: T-Prof(tabswitch) + tier: 2 + run-on-projects: [] + max-run-time: 1200 + mozharness: + extra-options: + - --suite=tabswitch + - --gecko-profile + +talos-webgl: + description: Talos WebGL + try-name: webgl + treeherder-symbol: T(webgl) + variants: + - webrender-sw + - webgl-ipc + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + (linux1804|windows10|macosx1015)-64.*: [trunk] + default: [] + webrender-sw: + by-test-platform: + (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] + default: [] + default: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macos)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 2100 + mozharness: + extra-options: + by-test-platform: + windows10-64-ref-hw.*: + - --suite=webgl + - --skip-preflight + default: + - --suite=webgl + +talos-webgl-profiling: + description: Talos WebGL with gecko profiling + try-name: webgl + treeherder-symbol: T-Prof(webgl) + tier: 2 + variants: + - webgl-ipc + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=webgl + - --gecko-profile + +talos-xperf: + description: Talos xperf + try-name: xperf + treeherder-symbol: T(x) + virtualization: virtual + os-groups: [Administrators] + run-as-administrator: + by-test-platform: + windows.*32.*: false + windows10-64.*: true + windows11-64.*: true + variants: + - webrender-sw + run-on-projects: + by-test-platform: + windows.*-32-shippable(?:-qr)?/.*: [] + windows10-64.*-shippable-qr/.*: [mozilla-beta, trunk] + windows11-64.*-shippable-qr/.*: [mozilla-beta, trunk] + default: [] + mozharness: + extra-options: + - --suite=xperf + config: + by-test-platform: + windows.*: + - talos/windows_vm_config.py + +talos-realworld-webextensions: + description: Talos tests with realworld WebExtensions + try-name: realworld-webextensions + treeherder-symbol: T(realwebext) + run-on-projects: + by-test-platform: + windows.*-32.*: [] + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows10-64|macosx1015)(?!.*-qr).*: [] + default: [mozilla-beta, trunk] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=realworld-webextensions + +talos-realworld-webextensions-profiling: + description: Talos tests with realworld WebExtensions with gecko profiling + try-name: realworld-webextensions-prof + treeherder-symbol: T-Prof(realwebext) + tier: 2 + run-on-projects: [] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=realworld-webextensions + - --gecko-profile diff --git a/taskcluster/ci/test/test-platforms.yml b/taskcluster/ci/test/test-platforms.yml new file mode 100644 index 0000000000..c460449ad3 --- /dev/null +++ b/taskcluster/ci/test/test-platforms.yml @@ -0,0 +1,419 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file maps build platforms to test platforms. In some cases, a +# single build may be tested on multiple test platforms, but a single test +# platform can only link to one build platform. Both build and test platforms +# are represented as <platform>/<type>, where <type> is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + + +## +# Linux platforms (matching /linux.*/) +# Linux1804-64 uses Ubuntu 18.04 and is run on docker + +linux1804-32-qr/debug: + build-platform: linux/debug + test-sets: + - linux32-gtest + +linux1804-32-qr/opt: + build-platform: linux/opt + test-sets: + - linux32-tests + - linux32-gtest + +linux1804-32-shippable-qr/opt: + build-platform: linux-shippable/opt + test-sets: + - linux32-tests + - linux32-gtest + +linux1804-64-qr/opt: + build-platform: linux64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - linux-talos-flex + - talos + - browsertime + +linux1804-64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - linux-talos-flex + - talos + - talos-profiling + - browsertime + - browsertime-profiling + +linux1804-64-qr/debug: + build-platform: linux64/debug + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-asan-qr/opt: + build-platform: linux64-asan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-tsan-qr/opt: + build-platform: linux64-tsan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-ccov-qr/opt: + build-platform: linux64-ccov/opt + test-sets: + - ccov-code-coverage-tests + - linux1804-tests + - linux1804-ccov-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-devedition-qr/opt: + build-platform: linux64-devedition/opt + test-sets: + - desktop-screenshot-capture + - linux1804-tests + - web-platform-tests + +linux1804-64-clang-trunk-qr/opt: + build-platform: linux64-plain-clang-trunk/opt + test-sets: + - awsy + - browsertime + +## +# Windows platforms (matching /windows.*/) + +# win32 +windows7-32-qr/opt: + build-platform: win32/opt + build-signing-label: build-signing-win32/opt + test-sets: + - windows-7-tests + +windows7-32-shippable-qr/opt: + build-platform: win32-shippable/opt + build-signing-label: build-signing-win32-shippable/opt + test-sets: + - windows-7-tests + +windows10-32-shippable-qr/opt: + build-platform: win32-shippable/opt + build-signing-label: build-signing-win32-shippable/opt + test-sets: + - browsertime + - windows-talos + +windows10-32-qr/opt: + build-platform: win32/opt + test-sets: + - browsertime + - windows-talos + +windows10-32-2004-shippable-qr/opt: + build-platform: win32-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - windows32-2004-tests + + +windows10-32-2004-qr/opt: + build-platform: win32/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - windows32-2004-tests + +windows10-32-2004-qr/debug: + build-platform: win32/debug + build-signing-label: build-signing-win32/debug + test-sets: + - web-platform-tests + - windows-7-tests + - windows32-2004-tests + +windows10-32-2004-mingwclang-qr/debug: + build-platform: win32-mingwclang/debug + test-sets: + - mingwclang-win32-tests-debug + +windows10-32-2004-mingwclang-qr/opt: + build-platform: win32-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +# win64 +windows10-64-qr/opt: + build-platform: win64/opt + test-sets: + - windows-talos + - browsertime + +windows10-64-ref-hw-2017-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - talos-ref-hw + +windows10-64-shippable-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - windows-talos + - browsertime + +# win10-64 2004 +windows10-64-2004-ccov-qr/opt: + build-platform: win64-ccov/opt + build-signing-label: build-signing-win64-ccov/opt + test-sets: + - desktop-screenshot-capture + - ccov-code-coverage-tests + - windows10-64-2004-ccov-tests + - windows10-2004-tests + +windows10-64-2004-qr/debug: + build-platform: win64/debug + build-signing-label: build-signing-win64/debug + test-sets: + - web-platform-tests + - windows10-2004-tests + +windows10-64-2004-qr/opt: + build-platform: win64/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - web-platform-tests-backlog + - windows10-2004-tests + - talos-xperf + +windows10-64-2004-shippable-qr/opt: + build-platform: win64-shippable/opt + build-signing-label: + by-variant: + msix: repackage-signing-shippable-l10n-msix-win64-shippable/opt + default: build-signing-win64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - web-platform-tests-backlog + - windows10-2004-tests + - talos-xperf + +windows10-64-2004-devedition-qr/opt: + build-platform: win64-devedition/opt + build-signing-label: build-signing-win64-devedition/opt + test-sets: + - desktop-screenshot-capture + - web-platform-tests + - windows10-2004-tests + +windows10-64-2004-asan-qr/opt: + build-platform: win64-asan/opt + test-sets: + - windows10-2004-tests + +windows10-64-2004-mingwclang-qr/debug: + build-platform: win64-mingwclang/debug + test-sets: + - mingwclang-tests-debug + +windows10-64-2004-mingwclang-qr/opt: + build-platform: win64-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +## +# MacOS X platforms (matching /macosx.*/) + +macosx1100-64-qr/debug: + build-platform: macosx64/debug + build-signing-label: repackage-macosx64/debug + test-sets: + - macosx1100-64-jittests + +macosx1100-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + build-signing-label: repackage-macosx64-shippable/opt + test-sets: + - macosx1100-64-jittests + - macosx1100-64-qr-tests + # - macosx64-talos + # - browsertime-high-priority + # - raptor-firefox + +macosx1015-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - awsy + - browsertime + - browsertime-profiling + - desktop-screenshot-capture + - macosx1015-64-qr-tests + - macosx64-talos + - raptor-firefox-power + - talos-profiling + - web-platform-tests + - web-platform-tests-backlog + +macosx1015-64-devedition-qr/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1015-64-qr-tests + - web-platform-tests + +macosx1015-64-qr/opt: + build-platform: macosx64/opt + test-sets: + - desktop-screenshot-capture + - macosx1015-64-qr-tests + - web-platform-tests + - web-platform-tests-backlog + +macosx1015-64-qr/debug: + build-platform: macosx64/debug + test-sets: + - macosx1015-64-qr-tests + - web-platform-tests + +macosx1015-64-ccov-qr/opt: + build-platform: macosx64-ccov/opt + test-sets: + - macosx1015-64-qr-tests + +## +# Android platforms (matching /android-em.*/) +# +# android-em test platforms execute on android emulators. + +android-em-7.0-x86_64-qr/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-lite-qr/opt: + build-platform: android-x86_64-lite/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable-qr/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable-lite-qr/opt: + build-platform: android-x86_64-shippable-lite/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-qr/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-tests + +android-em-7.0-x86_64-qr/debug-isolated-process: + build-platform: android-x86_64/debug-isolated-process + test-sets: + - android-x86_64-tests + +# android-hw test platforms execute on real devices attached to Autophone hosts. + +# android-hw-a51-11-0 Samsung Galaxy A51 Android 11.0 + +android-hw-a51-11-0-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-browsertime +# - android-hw-browsertime-power + +android-hw-a51-11-0-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-browsertime +# - android-hw-browsertime-power + +# android-hw-p2-8-0 Google Pixel 2 Android 8.0 + +android-hw-p2-8-0-arm7-qr/opt: + build-platform: android-arm/opt + test-sets: + - android-hw-browsertime + +android-hw-p2-8-0-arm7-shippable-qr/opt: + build-platform: android-arm-shippable/opt + test-sets: + - android-hw-browsertime + +# android-hw-p5-13-0 Google Pixel 5 Android 13.0 + +android-hw-p5-13-0-arm7-qr/opt: + build-platform: android-arm/opt + test-sets: + - android-hw-arm7-pgo-unittests + +android-hw-p5-13-0-arm7-shippable-qr/opt: + build-platform: android-arm-shippable/opt + test-sets: + - android-hw-arm7-pgo-unittests + +android-hw-p5-13-0-arm7-qr/debug: + build-platform: android-arm/debug + test-sets: + - android-hw-arm7-debug-unittests + +android-hw-p5-13-0-android-aarch64-qr/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-debug-unittests + +android-hw-p5-13-0-android-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-opt-unittests + +android-hw-p5-13-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-pgo-unittests diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml new file mode 100644 index 0000000000..d59d00391f --- /dev/null +++ b/taskcluster/ci/test/test-sets.yml @@ -0,0 +1,431 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# +# Test names given here reference tests.yml. + +### Feature-specific test groups + +web-platform-tests: + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + +web-platform-tests-backlog: + - web-platform-tests-backlog + - web-platform-tests-reftest-backlog + +mochitest-devtools-chrome: + - mochitest-devtools-chrome + +mochitest-webgpu: + - mochitest-webgpu + +talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g3 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions + +talos-profiling: + - talos-bcv-profiling + - talos-chrome-profiling + # - talos-damp-profiling - Disabled b/c of Bug 1597734 + - talos-dromaeojs-profiling + # - talos-flex-profiling - Disabled b/c of Bug 1597736 + - talos-g1-profiling + - talos-g3-profiling + - talos-g4-profiling + - talos-g5-profiling + - talos-motionmark-profiling + - talos-other-profiling + - talos-sessionrestore-many-windows-profiling + - talos-perf-reftest-profiling + - talos-perf-reftest-singletons-profiling + - talos-svgr-profiling + - talos-tp5o-profiling + - talos-tabswitch-profiling + - talos-realworld-webextensions-profiling + - talos-webgl-profiling + +talos-ref-hw: + - talos-webgl + +browsertime: + - browsertime-tp6 + - browsertime-tp6-essential + - browsertime-tp6-bytecode + - browsertime-tp6-live + - browsertime-tp6-live-sheriffed + - browsertime-benchmark + - browsertime-benchmark-wasm + - browsertime-youtube-playback + - browsertime-custom + - browsertime-first-install + - browsertime-grandprix + - browsertime-responsiveness + +browsertime-profiling: + - browsertime-tp6-profiling + +browsertime-high-priority: + - browsertime-benchmark + - browsertime-tp6 + +# Raptor desktop power tests +# (requires Intel Power Gadget to be pre-installed) +raptor-firefox-power: + - raptor-youtube-playback-v9-power-firefox + - raptor-youtube-playback-h264-power-firefox + +awsy: + - awsy + - awsy-base + - awsy-dmd + - awsy-base-dmd + - awsy-tp6 + +ccov-code-coverage-tests: + - test-coverage + - test-coverage-wpt + +mochitest-valgrind: + - mochitest-valgrind + +desktop-screenshot-capture: + - browser-screenshots + +### Specific platform test sets + +# Linux +linux32-gtest: + - gtest + +linux32-tests: + - web-platform-tests + +linux1804-tests: + - cppunit + - crashtest + - firefox-ui-functional + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - reftest-snapshot + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +linux1804-ccov-tests: + # jittest run on regular linux builds is redundant with SM(p), so we omit it + # from the linux1804-tests and linux1804-qr-tests sets. But, we still want + # to run it on ccov builds, so we have this set that has it and which runs + # on ccov builds. + - jittest + +linux-talos-flex: + - talos-flex + +# Windows + +windows-7-tests: + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-webgpu + - xpcshell + +windows32-2004-tests: + - cppunit + - crashtest + - gtest + - firefox-ui-functional + - jsreftest + - marionette + - mochitest-a11y + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + +windows10-2004-tests: + - cppunit + - crashtest + - firefox-ui-functional + - gtest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +windows10-64-2004-ccov-tests: + - jittest + - test-coverage + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + +mingwclang-tests-debug: + - cppunit + - firefox-ui-functional + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + +# The following is the same as the above block, minus reftests. +# Reftests were disabled due to frequent hangs; see bug 1642719 +# for re-enabling. +mingwclang-win32-tests-debug: + - cppunit + - firefox-ui-functional + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + # - reftest - disabled, see bug 1642719 + - telemetry-tests-client + +mingwclang-tests-opt: + - cppunit + - mochitest-chrome-gpu + - mochitest-plain-gpu + +windows-talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-svgr + - talos-tp5o + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + +talos-xperf: + - talos-xperf + +# macOS +macosx1100-64-jittests: + - jittest + +macosx1100-64-qr-tests: + - crashtest + - jsreftest + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome-gpu + - mochitest-media + - mochitest-webgpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - xpcshell + +macosx1015-64-qr-tests: + - cppunit + - crashtest + - firefox-ui-functional + - gtest + - jittest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +macosx64-talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-realworld-webextensions + - talos-webgl + # - talos-tabswitch # Bug 1453007 times out + # - talos-h1 # too long to unpack profile- Bug 1442893 + +# Android + +android-x86_64-tests: + - cppunit + - crashtest + - geckoview-junit + - gtest + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + # - mochitest-webgl1-core # Harness issues? + # - mochitest-webgl1-ext + # - mochitest-webgl2-core # Emulator not creating ES3? + # - mochitest-webgl2-ext + - reftest + - test-verify + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - web-platform-tests-wdspec + - xpcshell + +android-hw-aarch64-opt-unittests: + - jittest + +android-hw-arm7-pgo-unittests: + - jittest-all + +android-hw-aarch64-pgo-unittests: + - jittest + +android-hw-arm7-debug-unittests: + - jittest + +android-hw-aarch64-debug-unittests: + - jittest + +android-hw-aarch64: + - crashtest-qr + - reftest-qr + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + +android-hw-browsertime: + - browsertime-tp6m + - browsertime-tp6m-essential + - browsertime-tp6m-live + - browsertime-speedometer-mobile + - browsertime-speedometer3-mobile + - browsertime-youtube-playback-mobile + - browsertime-unity-webgl-mobile + +android-hw-browsertime-profiling: + - browsertime-tp6m-profiling diff --git a/taskcluster/ci/test/variants.yml b/taskcluster/ci/test/variants.yml new file mode 100644 index 0000000000..0844619dcb --- /dev/null +++ b/taskcluster/ci/test/variants.yml @@ -0,0 +1,316 @@ +--- +1proc: + description: "{description} with e10s disabled" + component: "Testing::General" + expiration: "never" + suffix: "1proc" + merge: + mozharness: + extra-options: + - "--disable-e10s" + - "--disable-fission" + +headless: + description: "{description} without a window" + component: "Firefox::Headless" + expiration: "never" + suffix: "headless" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--headless" + +a11y-checks: + description: "{description} with accessibility checks enabled" + component: "Core::Disability Access APIs" + expiration: "2023-03-01" + suffix: "a11y-checks" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--enable-a11y-checks" + +aab: + description: "{description} with aab test_runner" + component: "GeckoView::General" + expiration: "2023-03-01" + when: &gv_e10s_filter + $eval: '"android" in task["test-platform"]' + suffix: "aab" + # Need to add jdk but there isn't a good way to do that + # so we just replace the toolchain list + merge: + fetches: + toolchain: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-jdk + replace: + target: "geckoview-test_runner.aab" + +geckoview-e10s-single: + description: "{description} with single-process e10s" + component: "Core::DOM: Content Processes" + expiration: "2023-03-01" + when: + <<: *gv_e10s_filter + suffix: "e10s-single" + merge: + mozharness: + extra-options: + - "--setpref=dom.ipc.processCount=1" + - "--disable-fission" + +geckoview-no-fission: + description: "{description} without fission enabled" + component: "Core::DOM: Content Processes" + expiration: "2023-03-01" + when: + <<: *gv_e10s_filter + suffix: "nofis" + merge: + mozharness: + extra-options: + - "--disable-fission" + +geckoview-fission: + description: "{description} with fission enabled" + component: "Core::DOM: Content Processes" + expiration: "2023-03-01" + when: + <<: *gv_e10s_filter + suffix: "fis" + +fission: + description: "{description} with fission enabled" + component: "Testing::General" + expiration: "2023-03-01" + suffix: "fis" + +fission-no-isolate: + description: "{description} with fission and webContentIsolationStrategy=0" + component: "Core::DOM: Content Processes" + expiration: "2023-03-01" + suffix: "fis-0" + merge: + mozharness: + extra-options: + - "--web-content-isolation-strategy=0" + +no-fission: + description: "{description} without fission enabled" + component: "Testing::General" + expiration: "2023-03-01" + suffix: "nofis" + merge: + mozharness: + extra-options: + - "--disable-fission" + +xorigin: + description: "{description} with cross-origin" + component: "Core::DOM: Content Processes" + expiration: "2023-03-01" + suffix: "xorig" + replace: + e10s: true + merge: + mozharness: + extra-options: + - "--enable-xorigin-tests" + - "--setpref=privacy.partition.always_partition_third_party_non_cookie_storage=false" + +conditioned_profile: + description: "{description} with conditioned profile" + component: "Testing::General" + expiration: "2023-03-01" + suffix: "condprof" + merge: + mozharness: + extra-options: + - "--conditioned-profile" + replace: + chunks: 8 + +socketprocess: + description: "{description} with socket process enabled" + component: "Core::Networking" + expiration: "2023-03-01" + suffix: "spi" + merge: + mozharness: + extra-options: + - "--setpref=media.peerconnection.mtransport_process=true" + - "--setpref=network.process.enabled=true" + +socketprocess_networking: + description: "{description} with networking on socket process enabled" + component: "Core::Networking" + expiration: "2023-03-01" + suffix: "spi-nw" + merge: + mozharness: + extra-options: + - "--setpref=network.process.enabled=true" + - "--setpref=network.http.network_access_on_socket_process.enabled=true" + +wayland: + description: "{description} with Wayland backend enabled" + component: "Core::Widget: Gtk" + expiration: "2023-03-01" + when: + $eval: '"linux" in task["test-platform"]' + suffix: "wayland" + replace: + run-on-projects: [] + tier: 3 + merge: + mozharness: + extra-options: + - "--setpref=widget.wayland.test-workarounds.enabled=true" + +# - +# Webrender DComp variants for Win10 + +webrender-dcomp-disabled: + description: "{description} with dcomp disabled" + component: "Core::Graphics: WebRender" + expiration: "2023-03-01" + suffix: "wr-dc0" + when: + $eval: '"windows10" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=false" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-present: # Default variant ends up like this. + description: "{description} with dcomp present but not overlays" + component: "Core::Graphics: WebRender" + expiration: "2023-03-01" + suffix: "wr-dc1-p" + when: + $eval: '"windows10" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=false" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-overlays: + description: "{description} with dcomp overlays but not compositing" + component: "Core::Graphics: WebRender" + expiration: "2023-03-01" + suffix: "wr-dc2-o" + when: + $eval: '"windows10" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=true" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-compositing: + description: "{description} with dcomp compositing" + component: "Core::Graphics: WebRender" + expiration: "2023-03-01" + suffix: "wr-dc3-c" + when: + $eval: '"windows10" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=true" + - "--setpref=gfx.webrender.compositor=true" + - "--setpref=gfx.webrender.compositor.force-enabled=true" + +# - + +webrender-sw: + description: "{description} with software webrender enabled" + component: "Core::Graphics: WebRender" + expiration: "2023-03-01" + suffix: "swr" + merge: + webrender: true + mozharness: + extra-options: + - "--setpref=gfx.webrender.software=true" + +webgl-ipc: + description: "{description} with WebGL IPC process enabled" + component: "Core::Canvas: WebGL" + expiration: "2023-03-01" + suffix: "gli" + merge: + mozharness: + extra-options: + - "--setpref=webgl.out-of-process=true" + - "--setpref=security.sandbox.content.headless=true" + +devtools-no-eft: + description: "{description} with DevTools EFT disabled" + component: "DevTools::Framework" + expiration: "2023-03-01" + suffix: "dt-no-eft" + merge: + mozharness: + extra-options: + - "--setpref=devtools.every-frame-target.enabled=false" + +wmf-media-engine: + description: "{description} with Windows Media Foundation media engine enabled" + component: "Core::Audio/Video:Playback" + expiration: "2023-04-10" + when: + $eval: '"windows10" in task["test-platform"]' + suffix: "wmfme" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--tag=media-engine-compatible" + - "--setpref=media.wmf.media-engine.enabled=true" + - "--setpref=media.wmf.media-engine.channel-decoder.enabled=true" + +a11y-no-cache: + description: "{description} with accessibility cache disabled" + component: "Core::Disability Access APIs" + expiration: "2023-05-01" + suffix: "a11y-no-cache" + merge: + mozharness: + extra-options: + - "--setpref=accessibility.cache.enabled=false" + +msix: + description: "{description} from msix packages" + component: "Firefox:Installer" + expiration: "never" + suffix: "msix" + when: + $eval: '"windows10-64" in task["test-platform"] && "opt" in task["test-platform"] && !("asan" in task["test-platform"]) && !("ccov" in task["test-platform"])' + merge: + mozharness: + extra-options: + - "--installer-path installer.msix" diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml new file mode 100644 index 0000000000..3e6d05f8d6 --- /dev/null +++ b/taskcluster/ci/test/web-platform.yml @@ -0,0 +1,436 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: web-platform-tests + instance-size: xlarge + max-run-time: + by-test-platform: + android-em-7.0-x86_64/debug: 7200 + default: 5400 + mozharness: + script: web_platform_tests.py + config: + by-test-platform: + windows.*: + - web_platform_tests/prod_config_windows_taskcluster.py + macosx.*: + - web_platform_tests/prod_config_mac.py + android-em.*: + - android/android-x86_64.py + - web_platform_tests/prod_config_android.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + target: + by-test-platform: + android-em-7.0-x86_64-shippable(-lite)?-qr/opt: geckoview-test_runner.apk + android-em-7.0-x86_64(-lite)?-qr/opt: geckoview-test_runner.apk + android-em-7.0-x86_64(-lite)?-qr/debug(-isolated-process)?: geckoview-test_runner.apk + default: null + +web-platform-tests: + description: "Web platform test run" + suite: web-platform-tests + treeherder-symbol: W(wpt) + virtualization: virtual + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + chunks: + by-test-platform: + android.*/debug: 32 + android.*: 24 + linux.*64-tsan-qr/opt: 32 + linux.*64-asan(-qr)?/opt: 22 + linux.*64-ccov.*/opt: 20 + linux.*/debug: 16 + macosx.*/debug: 18 + windows.*32.*/opt: 12 + windows.*/debug: 14 + windows10-64-2004-ccov.*/opt: 14 + default: 10 + max-run-time: + by-test-platform: + .*-ccov.*/.*: 10800 + linux.*64(-tsan|-asan)-qr/opt: 14400 + linux.*64(-qr)?/debug: 10800 + macosx.*/debug: 10800 + android-em.*/.*: 10800 + default: 7200 + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10)-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows10-64-2004-qr/debug: built-projects + android-em-7.0-x86_64-qr/debug: [] + default: [] + webrender-sw+no-fission: + by-test-platform: + (linux1804|windows10)-32.*/.*: [] + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-backlog: + description: "Web platform test (backlog) run" + treeherder-symbol: W-b(wpt) + variants: + - no-fission + chunks: + by-test-platform: + android.*: 3 + default: 1 + max-run-time: + by-test-platform: + android.*/opt: 7200 + default: 3600 + run-on-projects: + by-variant: + no-fission: + by-test-platform: + android.*: ['mozilla-central'] + default: [] + default: + by-test-platform: + android.*: [] + default: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-reftest: + description: "Web platform reftest run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W(Wr) + virtualization: + by-test-platform: + windows10-64-2004.*-qr/.*: virtual-with-gpu + default: virtual + chunks: + by-test-platform: + .*-ccov.*/.*: 8 + linux.*64-tsan-qr/opt: 16 + linux1804-64(-asan)?-qr/.*: 6 + macosx10.*-64/debug: 6 + macosx10.*-64/opt: 3 + windows.*-(32|64)(-qr)?/debug: 5 + android.*: 6 + default: 4 + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + (linux1804|windows10)-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-reftest-backlog: + description: "Web platform reftest (backlog) run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W-b(Wr) + virtualization: virtual + chunks: 1 + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: + by-test-platform: + android.*: ['mozilla-central'] + default: [] + default: + by-test-platform: + android.*: [] + default: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-wdspec: + description: "Web platform webdriver-spec run" + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(Wd) + chunks: + by-test-platform: + .*-ccov.*/.*: 4 + .*-tsan.*/.*: 8 + default: 3 + mozharness: + extra-options: + - --test-type=wdspec + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + - headless + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + android-em-7.0-x86_64-qr/debug: [] + (linux1804|windows10)-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + headless: + by-test-platform: + android.*: [] + .*-tsan.*: [] + default: built-projects + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + tier: default + fetches: + toolchain: + by-test-platform: + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + +web-platform-tests-crashtest: + description: "Web platform crashtests run" + schedules-component: web-platform-tests-crashtest + treeherder-symbol: W(Wc) + chunks: 1 + mozharness: + extra-options: + - --test-type=crashtest + variants: + - no-fission + - webrender-sw + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + (linux1804|windows10)-32.*/.*: [] + linux.*64-qr/debug: built-projects + macosx101.*64-qr/debug: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + android-em-7.0-x86_64-qr/debug: built-projects + default: built-projects + tier: default + +web-platform-tests-print-reftest: + description: "Web platform print-reftest run" + schedules-component: web-platform-tests-print-reftest + treeherder-symbol: W(Wp) + chunks: 1 + test-manifest-loader: null # don't load tests in the taskgraph + tier: default + variants: + - no-fission + - webrender-sw + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + android.*: [] + (linux1804|windows10)-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows10-64-2004-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + mozharness: + extra-options: + - --test-type=print-reftest + +test-verify-wpt: + description: "Extra verification of web-platform tests modified on this push" + suite: + category: test-verify + name: test-verify-wpt + treeherder-symbol: TVw + max-run-time: 10800 + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan.*/.*: [] + .*shippable.*: [] + # do not run on beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + test-manifest-loader: null # don't load tests in the taskgraph + optimization: + test-verify: ["test-verify-wpt"] + tier: 2 + mozharness: + extra-options: + - --verify + +test-coverage-wpt: + description: "Per web-platform test coverage" + suite: + category: test-coverage + name: test-coverage-wpt + schedules-component: test-coverage-wpt + treeherder-symbol: TCw + max-run-time: 10800 + variants: + - no-fission + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + default: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --per-test-coverage diff --git a/taskcluster/ci/test/xpcshell.yml b/taskcluster/ci/test/xpcshell.yml new file mode 100644 index 0000000000..c28d276327 --- /dev/null +++ b/taskcluster/ci/test/xpcshell.yml @@ -0,0 +1,193 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: xpcshell + name: + by-variant: + msix: xpcshell-msix + default: xpcshell + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + windows10-64-2004-shippable-qr/opt: + by-variant: + msix: target.installer.msix + default: null + windows10-64-2004-qr/opt: + by-variant: + msix: target.installer.msix + default: null + default: null + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + requires-signed-builds: + by-test-platform: + macosx1100.*: true + windows10-64-2004-asan-qr/opt: false # No XPCShell on ASAN yet + windows10-32-mingwclang-qr/opt: false + windows10-32-mingwclang-qr/debug: false + windows10-64-2004-mingwclang-qr/opt: false + windows10-64-2004-mingwclang-qr/debug: false + windows.*: true + default: false + +xpcshell: + description: "xpcshell test run" + treeherder-symbol: X(X) + variants: + - conditioned_profile + - socketprocess_networking+no-fission + - socketprocess_networking + - no-fission + - msix + run-on-projects: + by-variant: + conditioned_profile: + by-test-platform: + (linux1804-64|windows10-64-2004)(-shippable)?-qr/opt: ['trunk'] + default: [] + socketprocess_networking+no-fission: + by-test-platform: + linux1804-64.*: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + (linux1804-64|macosx).*: ['trunk'] # All Linux & Mac variants + ^(?!.*asan).*windows.*: ['trunk'] # All Windows except ASAN + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64.*: ['trunk'] + default: [] + msix: + by-test-platform: + windows10-64-2004-shippable-qr/opt: built-projects + windows10-64-2004-qr/opt: built-projects + default: [] + default: + by-test-platform: + android.*: [] + windows10-64-2004-asan-qr/opt: [] # No XPCShell on ASAN yet + default: built-projects + chunks: + by-test-platform: + .*-asan-qr/opt: 4 + .*-ccov.*/.*: 6 + .*-tsan-qr/opt: 8 + (android-em-7.*|macosx.*|linux1804-64.*/opt): 2 + default: 4 + max-run-time: + by-variant: + msix: 7200 + default: + by-test-platform: + .*64-ccov(-qr)?/.*: 7200 + .*64-2004-ccov(-qr)?/.*: 7200 # Windows 10 x64 2004 CCov specific, the line above does not catch this case. + .*-tsan-qr/opt: 7200 + default: 5400 + allow-software-gl-layers: false + tier: + by-test-platform: + windows10-64-2004-asan.*: 3 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + extra-options: + by-test-platform: + windows10.*: + - --threads=2 + default: [] + fetches: + toolchain: + by-test-platform: + # xpcshell-tests are limited to *-node-12 as node 14+ dropped + # support for Windows 7. Once support for Windows 7 is dropped + # (bug 1594270) we should be able to revert these to the latest + # node version (*-node). + linux.*: + - linux64-node-12 + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node-12 + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node-12 + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node-12 + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-node-12 + - linux64-minidump-stackwalk + +xpcshell-failures: + description: "xpcshell specific failures test run" + treeherder-symbol: X-f(X) + run-on-projects: ["mozilla-central"] + chunks: 2 + max-run-time: 2700 + allow-software-gl-layers: false + tier: 3 + mozharness: + script: + by-test-platform: + macos.*: mac_unittest.py + windows10.*: desktop_unittest.py + config: + by-test-platform: + macos.*: + - unittests/mac_unittest.py + windows10.*: + - unittests/win_unittest.py + extra-options: + by-test-platform: + macosx1100.*: + - --run-failures=apple_silicon + - --timeout-as-pass + - --crash-as-pass + windows10-32-2004.*: + - --run-failures=win10_2004 + - --timeout-as-pass + - --crash-as-pass + fetches: + toolchain: + by-test-platform: + # xpcshell-tests are limited to *-node-12 as node 14+ dropped + # support for Windows 7. Once support for Windows 7 is dropped + # (bug 1594270) we should be able to revert these to the latest + # node version (*-node). + macosx.*: + - macosx64-node-12 + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*32.*: + - win32-node-12 + - win32-minidump-stackwalk + - win32-fix-stacks |