diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
commit | 2aa4a82499d4becd2284cdb482213d541b8804dd (patch) | |
tree | b80bf8bf13c3766139fbacc530efd0dd9d54394c /taskcluster/ci/test | |
parent | Initial commit. (diff) | |
download | firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip |
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/ci/test')
-rw-r--r-- | taskcluster/ci/test/awsy.yml | 86 | ||||
-rw-r--r-- | taskcluster/ci/test/browsertime-desktop.yml | 151 | ||||
-rw-r--r-- | taskcluster/ci/test/browsertime-mobile.yml | 426 | ||||
-rw-r--r-- | taskcluster/ci/test/compiled.yml | 105 | ||||
-rw-r--r-- | taskcluster/ci/test/firefox-ui.yml | 38 | ||||
-rw-r--r-- | taskcluster/ci/test/kind.yml | 72 | ||||
-rw-r--r-- | taskcluster/ci/test/marionette.yml | 69 | ||||
-rw-r--r-- | taskcluster/ci/test/misc.yml | 254 | ||||
-rw-r--r-- | taskcluster/ci/test/mochitest.yml | 607 | ||||
-rw-r--r-- | taskcluster/ci/test/raptor-chrome.yml | 247 | ||||
-rw-r--r-- | taskcluster/ci/test/raptor-chromium.yml | 219 | ||||
-rw-r--r-- | taskcluster/ci/test/raptor-gve.yml | 120 | ||||
-rw-r--r-- | taskcluster/ci/test/raptor-refbrow.yml | 86 | ||||
-rw-r--r-- | taskcluster/ci/test/raptor.yml | 784 | ||||
-rw-r--r-- | taskcluster/ci/test/reftest.yml | 235 | ||||
-rw-r--r-- | taskcluster/ci/test/talos.yml | 646 | ||||
-rw-r--r-- | taskcluster/ci/test/test-platforms.yml | 537 | ||||
-rw-r--r-- | taskcluster/ci/test/test-sets.yml | 560 | ||||
-rw-r--r-- | taskcluster/ci/test/web-platform.yml | 342 | ||||
-rw-r--r-- | taskcluster/ci/test/xpcshell.yml | 99 |
20 files changed, 5683 insertions, 0 deletions
diff --git a/taskcluster/ci/test/awsy.yml b/taskcluster/ci/test/awsy.yml new file mode 100644 index 0000000000..ea50703abb --- /dev/null +++ b/taskcluster/ci/test/awsy.yml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: awsy + max-run-time: 7200 + instance-size: xlarge + virtualization: + by-test-platform: + windows10-64(?:-shippable)?-qr/.*: virtual-with-gpu + windows10-64/.*: virtual-with-gpu + default: virtual + allow-software-gl-layers: false + mozharness: + script: awsy_script.py + config: + by-test-platform: + windows.*: + - awsy/taskcluster_windows_config.py + macosx.*/opt: + - awsy/macosx_config.py + default: + - awsy/linux_config.py + tier: default + fission-tier: + by-test-platform: + linux1804-64(?!-shippable).*: 3 + windows10-64(-qr)?/opt: 3 + default: 2 + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition + windows7-32.*/opt: [] + (linux|windows10|mac)(?!.*shippable).*/opt: [] + default: ['integration', 'mozilla-central', 'mozilla-beta', 'mozilla-release'] + test-manifest-loader: null # don't load tests in the taskgraph + fission-run-on-projects: + by-test-platform: + linux1804-64-shippable-qr/opt: ['mozilla-central'] + windows10-64-shippable-qr/opt: ['mozilla-central'] + default: [] + + +awsy: + description: "Are we slim yet" + treeherder-symbol: SY(sy) + +awsy-tp6: + description: "Are we slim yet - tp6 pageset" + variants: ["fission"] + treeherder-symbol: SY(sy-tp6) + run-on-projects: + by-test-platform: + .*-ccov.*/.*: [] # don't run on coverage + .*-devedition/.*: [] # don't run on devedition + windows7-32.*/opt: [] + (linux|windows10|mac)(?!.*shippable)(?!.*ccov).*/opt: [] + default: ['integration', 'mozilla-central', 'mozilla-beta', 'mozilla-release'] + mozharness: + extra-options: + - --tp6 + +awsy-dmd: + description: "Are we slim yet - dmd enabled" + treeherder-symbol: SY(sy-d) + run-on-projects: [] + mozharness: + extra-options: + - --dmd + +awsy-base: + description: "Are we slim yet - about:blank base case" + treeherder-symbol: SY(ab) + mozharness: + extra-options: + - --base + +awsy-base-dmd: + description: "Are we slim yet - about:blank base case - dmd enabled" + treeherder-symbol: SY(ab-d) + run-on-projects: [] + mozharness: + extra-options: + - --base + - --dmd diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..1111f85168 --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + .*-ref-hw-2017/.*: 3600 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: ['mozilla-central'] + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + fission-tier: 3 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + windows10-64-ccov.*/.*: + - raptor/windows_vm_config.py + linux64-ccov.*/.*: + - raptor/linux64_config_taskcluster.py + default: + - raptor/linux_config.py + extra-options: + - --browsertime + - --no-conditioned-profile + fission-run-on-projects: [] + +browsertime-tp6: + description: "Raptor (browsertime) tp6 page-load tests" + raptor-test: tp6 + raptor-subtests: + - amazon + - apple + - [bing-search, bing] + - ebay + - [facebook, fb] + - [facebook-redesign, fbr] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-sheets, gsheets] + - [google-slides, gslides] + - imdb + - imgur + - instagram + - linkedin + - microsoft + - netflix + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - yahoo-news + - yandex + - youtube + tier: + by-app: + firefox: + by-subtest: + amazon: 1 + google-mail: 1 + google-slides: 1 + imgur: 1 + tumblr: 1 + twitch: 1 + twitter: 1 + default: 2 + default: 3 + apps: ['firefox', 'chrome', 'chromium'] + variants: + by-app: + firefox: ['fission'] + default: [] + pageload: cold + fission-tier: 2 + fission-run-on-projects: ["mozilla-central"] + run-on-projects: + by-app: + firefox: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + linux.*shippable.*: ["trunk", "mozilla-beta"] + macos.*shippable.*: ["trunk", "mozilla-beta"] + default: [] + default: [] + limit-platforms: + by-app: + chrome: + - linux.*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + chromium: + # Run on non-QR shippable, but test-platforms.yml doesn't have + # any more non-QR macosx shippable things. So for macosx we want + # to run it on shippable-qr, otherwise there's nothing running + # this on macosx at all. + - .*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + default: [] + treeherder-symbol: Btime(tp6) + max-run-time: 4000 + run-visual-metrics: true + mozharness: + extra-options: + - --chimera + +browsertime-speedometer: + description: "Raptor (browsertime) Speedometer on Firefox" + raptor-test: speedometer + variants: ["fission"] + apps: ["firefox"] + treeherder-symbol: Btime(sp) + +browsertime-ares6: + description: "Raptor (browsertime) ares6 on Firefox" + raptor-test: raptor-ares6 + variants: ["fission"] + apps: ["firefox"] + treeherder-symbol: Btime(ares6) + +browsertime-tp6-profiling: + description: "Raptor (browsertime) tp6 page-load tests with Gecko Profiling" + raptor-test: tp6 + raptor-subtests: ['amazon'] + apps: ['firefox'] + pageload: 'both' + treeherder-symbol: Btime-Prof(tp6) + max-run-time: 4000 + mozharness: + extra-options: + - --gecko-profile diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml new file mode 100644 index 0000000000..3a1440d810 --- /dev/null +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -0,0 +1,426 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: raptor + run-on-projects: + by-app: + refbrow: [] + fennec: [] + default: + by-test-platform: + android-hw-p2-.*aarch64.*/pgo: ["mozilla-central"] + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central"] + default: [] + webrender-run-on-projects: [] + tier: 3 + test-manifest-loader: null # don't load tests in the taskgraph + target: + by-app: + fenix: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.fenix.nightly.latest.arm64-v8a + name: build/arm64-v8a/target.apk + default: + index: mobile.v2.fenix.nightly.latest.armeabi-v7a + name: build/armeabi-v7a/target.apk + fennec: + by-test-platform: + android-hw-p2-8-0-android-aarch64.*: + index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-aarch64-release-opt + name: build/target.apk + default: + index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt + name: build/target.apk + refbrow: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.reference-browser.raptor.latest.arm64-v8a + name: target.arm64-v8a.apk + default: + index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a + name: target.armeabi-v7a.apk + default: geckoview_example.apk + e10s: true + virtualization: hardware + activity: + by-app: + fenix: org.mozilla.fenix.IntentReceiverActivity + geckoview: org.mozilla.geckoview_example.GeckoViewActivity + fennec: org.mozilla.gecko.BrowserApp + refbrow: org.mozilla.reference.browser.BrowserTestActivity + default: None + binary-path: + by-app: + fenix: org.mozilla.fenix + geckoview: org.mozilla.geckoview_example + fennec: org.mozilla.firefox + refbrow: org.mozilla.reference.browser.raptor + default: None + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --browsertime + # Bug 1635749, disable window recorder temporarily + - --browsertime-no-ffwindowrecorder + - --no-conditioned-profile + optimization: + skip-unless-backstop: null + variants: + by-app: + chrome-m: [] + fennec: [] + default: ["webrender"] + +browsertime-tp6m: + description: "Raptor (browsertime) tp6 page-load tests on android" + raptor-test: tp6m + raptor-subtests: + by-app: + chrome-m: + - amazon + - youtube + default: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bbc + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - cnn + - [cnn-ampstories, cnn-amp] + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - facebook + - [facebook-cristiano, fb-cris] + - google + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - jianshu + - [microsoft-support, micros-sup] + - reddit + - [stackoverflow, stacko] + - web-de + - wikipedia + - youtube + - [youtube-watch, youtube-w] + apps: ["geckoview", "fenix", "fennec", "chrome-m", "refbrow"] + tier: + by-raptor-test: + amazon: + by-app: + fenix: 1 + geckoview: 1 + refbrow: 2 + fennec: 2 + default: 3 + default: + by-app: + geckoview: 1 + refbrow: 2 + fennec: 2 + default: 3 + webrender-run-on-projects: + by-raptor-test: + amazon: + by-app: + fenix: [] + chrome-m: [] + fennec: [] + geckoview: + by-pageload: + cold: + by-test-platform: + &android-cold-settings + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + warm: + by-test-platform: + &android-warm-settings + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['mozilla-central'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['mozilla-central'] + default: [] + default: + by-test-platform: + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central"] + default: [] + youtube: + by-app: + fenix: [] + chrome-m: [] + refbrow: [] + fennec: [] + geckoview: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + default: + by-test-platform: + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central", "try"] + default: [] + default: + by-pageload: + cold: + by-app: + geckoview: + by-test-platform: *android-cold-settings + default: [] + warm: + by-app: + geckoview: + by-test-platform: *android-warm-settings + default: [] + run-on-projects: + by-app: + geckoview: + by-subtest: + allrecipes: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + amazon-search: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + espn: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + facebook: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + google: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + microsoft-support: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + youtube-watch: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + default: [] + default: [] + pageload: cold + run-visual-metrics: true + treeherder-symbol: Btime(tp6m) + mozharness: + extra-options: + - --chimera + +browsertime-youtube-playback-mobile: + description: "Browsertime YouTube Playback on Android" + treeherder-symbol: Btime(ytp) + raptor-subtests: + by-test-platform: + android-hw-g5.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + android-hw-p2.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + apps: ["fenix", "geckoview", "refbrow"] + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ["mozilla-central"] + android-hw-p2-8-0-android-aarch64-shippable/opt: ["mozilla-central"] + default: [] + default: [] + test-url-param: + by-subtest: + youtube-playback-h264-sfr: # remove high resolution tests + "exclude=1,2, + 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38" + youtube-playback-hfr: + by-test-platform: + android-hw-g5.*: # remove VP9(1-34) and AV1(51-74) tests + "exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74" + android-hw-p2.*: # remove AV1(51-74) tests + "exclude=1,2, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74" + default: "" + youtube-playback-widevine-hfr: + by-test-platform: + android-hw-g5.*: # remove VP9(1-34) tests + "exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34" + default: "" + default: "" + max-run-time: 3600 + +browsertime-tp6m-live: + description: "Raptor (browsertime) tp6 live site page-load tests on android" + raptor-test: tp6m + raptor-subtests: + by-app: + geckoview: + - [cnn-ampstories, cnn-amp] + default: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bbc + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - booking-sf + - cnn + - [cnn-ampstories, cnn-amp] + - discord + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - expedia + - facebook + - [facebook-cristiano, fb-cris] + - [fashionbeans, fashbea] + - google + - [google-accounts, gacc] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - [imdb-firefox, imdb-f] + - jianshu + - [medium-article, medium] + - [microsoft-support, micros-sup] + - nytimes + - [people-article, people] + - reddit + - [reddit-thread, reddit-thr] + - rumble-fox + - [stackoverflow, stacko] + - [stackoverflow-question, stack-q] + - [urbandictionary-define, urbict-def] + - web-de + - [wikia-marvel, wikia-m] + - wikipedia + - youtube + - [youtube-watch, youtube-w] + apps: ["fenix", "chrome-m", "geckoview"] + run-on-projects: [] + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ["autoland"] + android-hw-p2-8-0-android-aarch64-shippable/opt: ["autoland"] + default: [] + default: [] + pageload: cold + run-visual-metrics: true + treeherder-symbol: Btime-live(tp6m) + mozharness: + extra-options: + - --live-sites + - --chimera + +browsertime-tp6m-profiling: + description: "Raptor (browsertime) tp6 page-load tests on android with Gecko profiling" + raptor-test: tp6m + raptor-subtests: ["amazon", "youtube"] + apps: ["geckoview"] + pageload: cold + treeherder-symbol: Btime-Prof(tp6m) + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --browsertime + - --gecko-profile + +browsertime-speedometer-mobile: + description: "Browsertime Speedometer on Android" + apps: ["geckoview", "fenix", "fennec", "chrome-m"] + tier: + by-app: + fenix: 1 + default: 3 + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + default: [] + treeherder-symbol: Btime(sp) + mozharness: + extra-options: + - --browsertime + - --test=speedometer + +browsertime-unity-webgl-mobile: + description: "Browsertime Unity WebGL on Android" + apps: ["geckoview", "refbrow", "fenix", "chrome-m"] + treeherder-symbol: Btime(ugl) + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + default: [] + max-run-time: 900 + mozharness: + extra-options: + - --test=unity-webgl + fetches: + fetch: + - unity-webgl + +browsertime-power: + description: "Browsertime Power Usage Tests on Android" + apps: ["geckoview", "fenix", "refbrow"] + tier: 2 + raptor-subtests: + - idle + - idle-bg + - [speedometer, sp] + variants: ["webrender"] + run-on-projects: [] + treeherder-symbol: Btime-P(power) + mozharness: + extra-options: + - --power-test diff --git a/taskcluster/ci/test/compiled.yml b/taskcluster/ci/test/compiled.yml new file mode 100644 index 0000000000..aa4e803747 --- /dev/null +++ b/taskcluster/ci/test/compiled.yml @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + e10s: false + test-manifest-loader: null # don't load tests in the taskgraph + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + python-3: true + +cppunit: + description: "CPP Unit Tests" + suite: cppunittest + treeherder-symbol: cppunit + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + tier: default + run-on-projects: built-projects + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: GTest + instance-size: xlarge + run-on-projects: + by-test-platform: + windows.*-shippable/.*: [] # permafails on shippable + windows.*-nightly/.*: [] # permafails on nightly too + .*-devedition/.*: [] # don't run on devedition + default: built-projects + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + tier: + by-test-platform: + windows7-32-shippable.*: 3 + windows10-64-shippable.*: 3 + windows10-64-asan.*: 3 + linux.*64-ccov.*/opt: 3 + default: default + +jittest: + description: "JIT Test run" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-test-platform: + android-hw-.*-aarch64/debug: ['mozilla-central', 'release'] + android-hw-.*-api-16(?:-shippable)?/.*: ['mozilla-central', 'release'] + default: built-projects + chunks: + by-test-platform: + windows.*: 1 + windows10-64-ccov-qr/opt: 8 + macosx.*/opt: 1 + macosx.*/debug: 3 + android.*: 10 + default: 6 + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 7200 + macosx.*-ccov.*/.*: 7200 + android-hw.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + windows.*: false + macosx.*: false + default: true + target: + by-test-platform: + android-.*: geckoview-androidTest.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default diff --git a/taskcluster/ci/test/firefox-ui.yml b/taskcluster/ci/test/firefox-ui.yml new file mode 100644 index 0000000000..4d911fa0a2 --- /dev/null +++ b/taskcluster/ci/test/firefox-ui.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: firefox-ui + max-run-time: 5400 + test-manifest-loader: null # don't load tests in the taskgraph + mozharness: + script: firefox_ui_tests/functional.py + config: + by-test-platform: + windows.*: + - firefox_ui_tests/taskcluster_windows.py + macosx.*: + - firefox_ui_tests/taskcluster.py + - firefox_ui_tests/taskcluster_mac.py + linux.*: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + +firefox-ui-functional-local: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn-l(en-US) + mozharness: + extra-options: + - "--tag" + - "local" + +firefox-ui-functional-remote: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn-r(en-US) + mozharness: + extra-options: + - "--tag" + - "remote" + tier: 2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml new file mode 100644 index 0000000000..507baf080f --- /dev/null +++ b/taskcluster/ci/test/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.test:loader + +kind-dependencies: + - build + - build-signing + - fetch + - toolchain + +transforms: + - taskgraph.transforms.tests:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +# Each stanza in a file pointed to by 'jobs-from' describes a particular test +# suite or sub-suite. These are processed through the transformations described +# above to produce a bunch of tasks. See the schema in +# `taskcluster/taskgraph/transforms/tests.py` for a description of the fields +# used in these files. + +# Adding a new test type or running tests on a new platform? Be sure to review +# https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy + +jobs-from: + - awsy.yml + - browsertime-desktop.yml + - browsertime-mobile.yml + - compiled.yml + - firefox-ui.yml + - marionette.yml + - misc.yml + - mochitest.yml + - raptor.yml + - raptor-chrome.yml + - raptor-chromium.yml + - raptor-gve.yml + - raptor-refbrow.yml + - reftest.yml + - talos.yml + - web-platform.yml + - xpcshell.yml + +job-defaults: + attributes: + retrigger: true + require-signed-extensions: + by-release-type: + release|esr.*: true + beta: + by-test-platform: + .*-devedition/.*: false + default: true + default: false + fetches: + toolchain: + by-test-platform: + win.*: + - win32-minidump-stackwalk + - win32-fix-stacks + macosx.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + android-em-7.*: + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks diff --git a/taskcluster/ci/test/marionette.yml b/taskcluster/ci/test/marionette.yml new file mode 100644 index 0000000000..65fcbcc0df --- /dev/null +++ b/taskcluster/ci/test/marionette.yml @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: marionette + mozharness: + script: marionette.py + config: + by-test-platform: + macosx.*: + - marionette/prod_config.py + - marionette/mac_taskcluster_config.py + windows.*: + - marionette/windows_taskcluster_config.py + default: + - marionette/prod_config.py + - remove_executables.py + extra-options: + - --setpref=toolkit.asyncshutdown.log=true + + run-on-projects: built-projects + test-manifest-loader: null # don't load tests in the taskgraph + tier: default + fission-run-on-projects: + by-test-platform: + linux.*64/.*: ['autoland'] + linux.*64-shippable/.*: ['mozilla-central'] + windows10-64/.*: ['autoland'] + windows10-64-shippable/.*: ['mozilla-central'] + default: [] + fission-tier: 2 + variants: ["fission"] + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + +marionette: + description: "Marionette unittest run" + treeherder-symbol: Mn + max-run-time: 5400 + instance-size: default + chunks: 1 + +marionette-framescript: + description: "Marionette unittest run with its JSWindowActors disabled" + treeherder-symbol: MnFr + max-run-time: 5400 + instance-size: default + tier: 2 + run-on-projects: + # Platforms for which we release geckodriver + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: built-projects + default: [] + mozharness: + extra-options: + - --disable-actors + +marionette-headless: + description: "Marionette headless unittest run" + treeherder-symbol: MnH + max-run-time: 5400 + instance-size: default + tier: 2 + mozharness: + extra-options: + - --headless diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml new file mode 100644 index 0000000000..8c11fbe247 --- /dev/null +++ b/taskcluster/ci/test/misc.yml @@ -0,0 +1,254 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + +geckoview-junit: + description: "Geckoview junit run" + suite: geckoview-junit + variants: ["geckoview-e10s-single", "geckoview-fission"] + treeherder-symbol: gv-junit + loopback-video: true + e10s: true + target: geckoview-androidTest.apk + max-run-time: 3600 + tier: default + fission-run-on-projects: ['trunk'] + mozharness: + script: android_emulator_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + extra-options: + - --test-suite=geckoview-junit + +telemetry-tests-client: + description: "Telemetry tests client run" + suite: telemetry-tests-client + treeherder-symbol: tt(c) + max-run-time: 1200 + tier: default + mozharness: + script: telemetry/telemetry_client.py + config: + by-test-platform: + linux.*: + - remove_executables.py + windows.*: [] + macosx.*: [] + +test-verify: + description: "Extra verification of tests modified on this push" + suite: test-verify + treeherder-symbol: TV + loopback-video: true + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + optimization: + test-verify: ["test-verify"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-verify-gpu: + description: "Extra verification of tests modified on this push on gpu instances" + suite: + category: test-verify + name: test-verify-gpu + treeherder-symbol: TVg + loopback-video: true + virtualization: virtual-with-gpu + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + optimization: + test-verify: ["test-verify-gpu"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + - --gpu-required + +test-coverage: + description: "Per-test coverage" + suite: test-coverage + treeherder-symbol: TC + loopback-video: true + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-coverage-gpu: + description: "Per-test coverage for tests that require gpu instances" + suite: test-coverage + treeherder-symbol: TCg + loopback-video: true + virtualization: virtual-with-gpu + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # only run on mozilla-central and try. + .*-ccov.*/.*: ['mozilla-central'] + default: [] + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + - --gpu-required diff --git a/taskcluster/ci/test/mochitest.yml b/taskcluster/ci/test/mochitest.yml new file mode 100644 index 0000000000..7906d8b626 --- /dev/null +++ b/taskcluster/ci/test/mochitest.yml @@ -0,0 +1,607 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: mochitest + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + android-hw.*: geckoview-androidTest.apk + default: null + variants: + by-test-platform: + linux.*64/debug: ['fission'] + default: ['fission'] + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + default: built-projects + fission-run-on-projects: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + linux.*64-shippable-qr/opt: ['mozilla-central'] + linux.*64-shippable/.*: ['mozilla-central'] + linux.*64/debug: ['mozilla-central'] + windows10-64-shippable(-qr)?/opt: ['mozilla-central'] + default: [] + fission-tier: + by-test-platform: + linux.*64.*-qr/debug: 1 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +mochitest-plain: + description: "Mochitest plain run" + treeherder-symbol: M() + schedules-component: mochitest-plain + loopback-video: true + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + variants: + by-test-platform: + linux1804-64/opt: ['fission-xorigin'] + linux1804-64/debug: ['fission-xorigin'] + default: ['fission'] + chunks: + by-test-platform: + android-em-7.*: 4 + linux.*/debug: 16 + linux.*64-asan/opt: 10 + linux.*64-tsan/opt: 20 + linux.*64-.*cov/opt: 10 + windows10-64-ccov.*/.*: 10 + macosx.*64-ccov.*/.*: 10 + default: 5 + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + e10s: true + max-run-time: 5400 + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android-em.*: + - --test-suite=mochitest-plain + default: [] + chunked: + by-test-platform: + android-em.*: false + default: true + +mochitest-a11y: + description: "Mochitest a11y run" + treeherder-symbol: M(a11y) + schedules-component: mochitest-a11y + test-manifest-loader: default # ensure we don't run with manifest-scheduling + loopback-video: true + tier: default + e10s: false + run-on-projects: built-projects + mozharness: + mochitest-flavor: a11y + +mochitest-browser-chrome: + description: "Mochitest browser-chrome run" + suite: + name: mochitest-browser-chrome + treeherder-symbol: M(bc) + schedules-component: mochitest-browser-chrome + loopback-video: true + variants: + by-test-platform: + linux.*64(-shippable)?/opt: ['fission', 'a11y-checks'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64(-qr)?/debug: ['trunk'] + linux.*64-shippable(-qr)?/opt: ['mozilla-central'] + windows10-64-shippable(-qr)?/opt: ['mozilla-central'] + default: [] + tier: default + fission-tier: + by-test-platform: + linux.*64/debug: 1 + default: 2 + chunks: + by-test-platform: + linux.*/debug: 16 + linux.*64-asan/opt: 16 + linux.*64-tsan/opt: 32 + macosx.*64/debug: 16 + windows10-64-ccov.*/.*: 14 + windows10.*-asan/opt: 9 + default: 7 + max-run-time: + by-test-platform: + linux.*64-ccov.*/.*: 9000 + windows7-32/debug: 5400 + windows10-64/debug: 5400 + macosx.*64(-qr)?/debug: 5400 + windows10-64-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 10800 + linux.*/debug: 5400 + windows10-64-qr/debug: 5400 + linux.*64-tsan/opt: 9000 + default: 3600 + mozharness: + mochitest-flavor: browser + chunked: true + # Bug 1281241: migrating to m3.large instances + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + allow-software-gl-layers: false + +browser-screenshots: + description: "Browser Screenshots" + suite: + name: mochitest-browser-chrome-screenshots + treeherder-symbol: M(ss) + loopback-video: true + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: + by-test-platform: + windows7-32(?:-shippable)(?:-qr)?/opt: ['mozilla-central'] + windows10-64(?:-shippable)(?:-qr)?/opt: ['mozilla-central'] + (?:windows10-64|windows7-32|linux1804-64|macosx1014-64)(?:-qr)?/opt: ['integration'] + linux1804-64-(?:shippable)(?:-qr)?/opt: ['mozilla-central'] + macosx.*64-shippable/opt: ['mozilla-central'] + default: [] + fission-run-on-projects: [] + fission-tier: 2 + max-run-time: 3600 + mozharness: + mochitest-flavor: browser + allow-software-gl-layers: false + +mochitest-chrome: + description: "Mochitest chrome run" + treeherder-symbol: M(c) + schedules-component: mochitest-chrome + loopback-video: true + tier: default + chunks: + by-test-platform: + .*(-ccov|-qr).*/.*: 3 + windows10-64-ccov-qr/opt: 3 + .*-asan/opt: 3 + .*-tsan/opt: 6 + (linux.*64|windows.*-..|macosx10..)/debug: 3 + default: 2 + max-run-time: 3600 + e10s: false + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-devtools-chrome: + description: "Mochitest devtools-chrome run" + suite: + name: mochitest-devtools-chrome + schedules-component: mochitest-browser-chrome + treeherder-symbol: M(dt) + loopback-video: true + variants: + by-test-platform: + linux.*64(-shippable)?/opt: ['fission', 'a11y-checks'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64(-qr)?/debug: ['trunk'] + linux.*64-shippable(-qr)?/opt: ['trunk'] + windows10-64-shippable(-qr)?/opt: ['trunk'] + default: [] + tier: default + fission-tier: + by-test-platform: + linux.*64(-qr)?/debug: 1 + linux.*64-shippable(-qr)?/opt: 1 + windows10-64-shippable(-qr)?/opt: 1 + default: 2 + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 9000 + linux.*64-ccov.*/.*: 7200 + linux.*64-tsan/opt: 7200 + default: 5400 + chunks: + by-test-platform: + .*-ccov.*/.*: 16 + linux.*64/debug: 12 + macosx.*64/debug: 8 + .*-asan/opt: 8 + .*-tsan/opt: 16 + default: 5 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-[at]san/opt: xlarge # runs out of memory on default/m3.large + default: default + # Bug 1296086: high number of intermittents observed with software GL and large instances + allow-software-gl-layers: false + +mochitest-plain-gpu: + description: "Mochitest plain GPU run" + suite: + name: mochitest-plain-gpu + treeherder-symbol: M(gpu) + schedules-component: mochitest-plain + loopback-video: true + fission-run-on-projects: [] + fission-tier: 2 + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + e10s: true + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-plain-gpu + default: + - --mochitest-suite=mochitest-plain-gpu + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-chrome-gpu: + description: "Mochitest chrome GPU run" + suite: + name: mochitest-chrome-gpu + treeherder-symbol: M(gpu-c) + loopback-video: true + fission-run-on-projects: [] + fission-tier: 2 + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + e10s: true + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android.*/.*: [] + default: built-projects + mozharness: + mochitest-flavor: chrome + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-chrome-gpu + default: + - --mochitest-suite=mochitest-chrome-gpu + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-media: + description: "Mochitest media run" + treeherder-symbol: M(mda) + schedules-component: mochitest-plain + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 7200 + macosx.*64-ccov.*/.*: 7200 + linux.*64-tsan/opt: 7200 + default: 5400 + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android-hw-.*(?<!-shippable)(-qr)?/opt: ['autoland'] + android-hw-.*-api-16(-qr)?/(?:debug)?: ['trunk', 'mozilla-beta', 'mozilla-release'] + android-hw-.*-api-16-shippable(-qr)?/opt: ['trunk', 'mozilla-beta', 'mozilla-release'] + windows10-aarch64/.*: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + default: built-projects + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: ['socketprocess'] + default: ['fission', 'socketprocess', 'webgl-ipc'] + loopback-video: true + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: large + chunks: + by-test-platform: + android-em-7.*: 1 + windows10-64.*: 1 + macosx.*64.*/.*: 2 + windows10-aarch64/.*: 1 + windows7-32-shippable/.*: 2 + linux1804-64(-shippable|-devedition|-.*qr)/opt: 2 + linux.*64-tsan/opt: 4 + default: 3 + mozharness: + mochitest-flavor: plain + chunked: + by-test-platform: + android.*: false + macosx.*64.*: false + windows10-64.*: false + default: true + extra-options: + - --setpref=webgl.out-of-process=false + tier: + by-test-platform: + android-em.*: 1 + windows10-aarch64.*: 2 + android-hw.*: 1 + default: default + +mochitest-plain-headless: + description: "Mochitest plain headless run" + suite: + name: mochitest-plain + treeherder-symbol: M(h) + loopback-video: true + chunks: + by-test-platform: + linux.*/debug: 16 + default: 5 + e10s: true + max-run-time: 5400 + allow-software-gl-layers: false + tier: 2 + run-on-projects: ['mozilla-central'] + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --headless + +mochitest-valgrind: + description: "Mochitest plain Valgrind run" + suite: + name: mochitest-valgrind-plain + treeherder-symbol: M-V() + run-on-projects: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + loopback-video: true + chunks: 40 + max-run-time: 14400 + # We could re-enable e10s later. + # There's no intrinsic reason not to use it. + e10s: false + variants: [] + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + +mochitest-webgl1-core: + description: "Mochitest webgl1-core run" + treeherder-symbol: M(gl1c) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + default: ['webgl-ipc'] + e10s: true + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + tier: default + max-run-time: + by-test-platform: + macosx.*64-ccov.*/.*: 7200 + default: 1800 + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable/opt: ['mozilla-central'] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + - --setpref=webgl.out-of-process=false + +mochitest-webgl1-ext: + description: "Mochitest webgl1-ext run" + treeherder-symbol: M(gl1e) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + default: ['webgl-ipc'] + chunks: + by-test-platform: + android.*: 2 + default: 1 + e10s: true + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + tier: default + max-run-time: 2700 + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable/opt: ['mozilla-central'] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + +mochitest-webgl2-core: + description: "Mochitest webgl2-core run" + treeherder-symbol: M(gl2c) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + default: ['webgl-ipc'] + chunks: + by-test-platform: + android.*: 2 + default: 1 + e10s: true + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + tier: default + max-run-time: 1800 + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android-hw-.*(?<!-shippable)(-qr)?/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + android-hw.*aarch.*-shippable/opt: ['mozilla-central'] + default: built-projects + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + +mochitest-webgl2-ext: + description: "Mochitest webgl2-ext run" + treeherder-symbol: M(gl2e) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + default: ['webgl-ipc'] + chunks: 4 + e10s: true + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + tier: default + max-run-time: 2700 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + +mochitest-webgl2-deqp: + description: "Mochitest webgl2-deqp run" + treeherder-symbol: M(gl2d) + schedules-component: mochitest-plain + run-on-projects: [] # Don't run this for now. + virtualization: virtual-with-gpu + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + default: ['webgl-ipc'] + chunks: 4 + e10s: true + loopback-video: true + tier: default + max-run-time: 1800 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + extra-options: + - --setpref=webgl.out-of-process=false + +mochitest-webgpu: + description: "Mochitest webgpu run" + treeherder-symbol: M(webgpu) + schedules-component: mochitest-plain + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + e10s: true + loopback-video: true + run-on-projects: + by-test-platform: + .*mingw.*: ["release"] + .*shippable.*: ["mozilla-central"] + (!-shippable).*-qr.*: ["trunk"] + default: [] + tier: default + max-run-time: + by-test-platform: + macosx.*64-ccov.*/.*: 7200 + default: 1800 + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + +mochitest-remote: + description: "Mochitest for the remote agent (/remote folder)" + suite: + name: mochitest-remote + treeherder-symbol: M(remote) + loopback-video: true + run-on-projects: + by-test-platform: + .*shippable.*: ["mozilla-central"] + (linux1804-64|windows7-32|windows10-64|macosx1014-64)(?:-qr)?/opt: ['integration'] + default: ["trunk"] + fission-run-on-projects: [] + fission-tier: 2 + max-run-time: 5400 + mozharness: + mochitest-flavor: browser + extra-options: + - --setpref=remote.log.level=Trace + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual diff --git a/taskcluster/ci/test/raptor-chrome.yml b/taskcluster/ci/test/raptor-chrome.yml new file mode 100644 index 0000000000..cb2503842a --- /dev/null +++ b/taskcluster/ci/test/raptor-chrome.yml @@ -0,0 +1,247 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + .*-ux/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + windows10-64-ccov.*/.*: + - raptor/windows_vm_config.py + linux64-ccov.*/.*: + - raptor/linux64_config_taskcluster.py + default: + - raptor/linux_config.py + +raptor-speedometer-chrome: + description: "Raptor Speedometer on Chrome" + try-name: raptor-speedometer-chrome + treeherder-symbol: Rap-ChR(sp) + mozharness: + extra-options: + - --test=raptor-speedometer + - --app=chrome + +raptor-stylebench-chrome: + description: "Raptor StyleBench on Chrome" + try-name: raptor-stylebench-chrome + treeherder-symbol: Rap-ChR(sb) + mozharness: + extra-options: + - --test=raptor-stylebench + - --app=chrome + +raptor-motionmark-htmlsuite-chrome: + description: "Raptor MotionMark HtmlSuite on Chrome" + try-name: raptor-motionmark-htmlsuite-chrome + treeherder-symbol: Rap-ChR(mm-h) + mozharness: + extra-options: + - --test=raptor-motionmark-htmlsuite + - --app=chrome + +raptor-motionmark-animometer-chrome: + description: "Raptor MotionMark Animometer on Chrome" + try-name: raptor-motionmark-animometer-chrome + treeherder-symbol: Rap-ChR(mm-a) + mozharness: + extra-options: + - --test=raptor-motionmark-animometer + - --app=chrome + +raptor-webaudio-chrome: + description: "Raptor WebAudio on Chrome" + try-name: raptor-webaudio-chrome + treeherder-symbol: Rap-ChR(wa) + mozharness: + extra-options: + - --test=raptor-webaudio + - --app=chrome + +raptor-sunspider-chrome: + description: "Raptor SunSpider on Chrome" + try-name: raptor-sunspider-chrome + treeherder-symbol: Rap-ChR(ss) + mozharness: + extra-options: + - --test=raptor-sunspider + - --app=chrome + +raptor-unity-webgl-chrome: + description: "Raptor Unity WebGL on Chrome" + try-name: raptor-unity-webgl-chrome + treeherder-symbol: Rap-ChR(ugl) + tier: 3 + mozharness: + extra-options: + - --test=raptor-unity-webgl + - --app=chrome + fetches: + fetch: + - unity-webgl + +raptor-wasm-misc-chrome: + description: "Raptor WASM Misc on Chrome" + try-name: raptor-wasm-misc-chrome + treeherder-symbol: Rap-ChR(wm) + mozharness: + extra-options: + - --test=raptor-wasm-misc + - --app=chrome + fetches: + fetch: + - wasm-misc + +raptor-assorted-dom-chrome: + description: "Raptor Assorted-Dom on Chrome" + try-name: raptor-assorted-dom-chrome + treeherder-symbol: Rap-ChR(dom) + max-run-time: 1500 + mozharness: + extra-options: + - --test=raptor-assorted-dom + - --app=chrome + fetches: + fetch: + - assorted-dom + +raptor-wasm-godot-chrome: + description: "Raptor Wasm Godot on Chrome" + try-name: raptor-wasm-godot-chrome + treeherder-symbol: Rap-ChR(godot) + max-run-time: 1500 + mozharness: + extra-options: + - --test=raptor-wasm-godot + - --app=chrome + +raptor-ares6-chrome: + description: "Raptor Ares6 on Chrome" + try-name: raptor-ares6-chrome + treeherder-symbol: Rap-ChR(ares6) + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + - --app=chrome + +raptor-jetstream2-chrome: + description: "Raptor Jetstream2 on Chrome" + try-name: raptor-jetstream2-chrome + treeherder-symbol: Rap-ChR(js2) + max-run-time: 8000 + tier: 3 + mozharness: + extra-options: + - --test=raptor-jetstream2 + - --app=chrome + fetches: + fetch: + - jetstream2 + +raptor-youtube-playback-av1-sfr-chrome: + description: "Raptor YouTube Playback AV1 SFR on Chrome" + try-name: raptor-youtube-playback-av1-sfr-chrome + treeherder-symbol: Rap-ChR(ytp-av1-sfr) + max-run-time: 3600 + mozharness: + extra-options: + - --test=raptor-youtube-playback-av1-sfr + - --app=chrome + +raptor-youtube-playback-h264-sfr-chrome: + description: "Raptor YouTube Playback H264 SFR on Chrome" + try-name: raptor-youtube-playback-h264-sfr-chrome + treeherder-symbol: Rap-ChR(ytp-h264-sfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-h264-sfr + - --app=chrome + +raptor-youtube-playback-hfr-chrome: + description: "Raptor YouTube Playback HFR on Chrome" + try-name: raptor-youtube-playback-hfr-chrome + treeherder-symbol: Rap-ChR(ytp-hfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-hfr + - --app=chrome + +raptor-youtube-playback-vp9-sfr-chrome: + description: "Raptor YouTube Playback VP9 SFR on Chrome" + try-name: raptor-youtube-playback-vp9-sfr-chrome + treeherder-symbol: Rap-ChR(ytp-vp9-sfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-vp9-sfr + - --app=chrome + +raptor-youtube-playback-widevine-hfr-chrome: + description: "Raptor YouTube Playback Widevine HFR on Chrome" + try-name: raptor-youtube-playback-widevine-hfr-chrome + treeherder-symbol: Rap-ChR(ytp-widevine-hfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-hfr + - --app=chrome + +raptor-youtube-playback-widevine-h264-sfr-chrome: + description: "Raptor YouTube Playback Widevine H264 SFR on Chrome" + try-name: raptor-youtube-playback-widevine-h264-sfr-chrome + treeherder-symbol: Rap-ChR(ytp-widevine-h264-sfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-h264-sfr + - --app=chrome + +raptor-youtube-playback-widevine-vp9-sfr-chrome: + description: "Raptor YouTube Playback Widevine VP9 SFR on Chrome" + try-name: raptor-youtube-playback-widevine-vp9-sfr-chrome + treeherder-symbol: Rap-ChR(ytp-widevine-vp9-sfr) + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-vp9-sfr + - --app=chrome diff --git a/taskcluster/ci/test/raptor-chromium.yml b/taskcluster/ci/test/raptor-chromium.yml new file mode 100644 index 0000000000..4fd37aab93 --- /dev/null +++ b/taskcluster/ci/test/raptor-chromium.yml @@ -0,0 +1,219 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + .*-ux/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: [] + tier: 2 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + test-manifest-loader: null # don't load tests in the taskgraph + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + windows10-64-ccov.*/.*: + - raptor/windows_vm_config.py + linux64-ccov.*/.*: + - raptor/linux64_config_taskcluster.py + default: + - raptor/linux_config.py + fetches: + fetch: + by-test-platform: + win.*64.*: + - win64-chromium + win.*32.*: + - win32-chromium + macosx.*: + - mac-chromium + default: + - linux64-chromium + +raptor-speedometer-chromium: + description: "Raptor Speedometer on Chromium" + try-name: raptor-speedometer-chromium + treeherder-symbol: Rap-Cr(sp) + max-run-time: 1500 + mozharness: + extra-options: + - --test=raptor-speedometer + - --app=chromium + +raptor-stylebench-chromium: + description: "Raptor StyleBench on Chromium" + try-name: raptor-stylebench-chromium + treeherder-symbol: Rap-Cr(sb) + mozharness: + extra-options: + - --test=raptor-stylebench + - --app=chromium + +raptor-motionmark-htmlsuite-chromium: + description: "Raptor MotionMark HtmlSuite on Chromium" + try-name: raptor-motionmark-htmlsuite-chromium + treeherder-symbol: Rap-Cr(mm-h) + mozharness: + extra-options: + - --test=raptor-motionmark-htmlsuite + - --app=chromium + +raptor-motionmark-animometer-chromium: + description: "Raptor MotionMark Animometer on Chromium" + try-name: raptor-motionmark-animometer-chromium + treeherder-symbol: Rap-Cr(mm-a) + mozharness: + extra-options: + - --test=raptor-motionmark-animometer + - --app=chromium + +raptor-webaudio-chromium: + description: "Raptor WebAudio on Chromium" + try-name: raptor-webaudio-chromium + treeherder-symbol: Rap-Cr(wa) + mozharness: + extra-options: + - --test=raptor-webaudio + - --app=chromium + +raptor-sunspider-chromium: + description: "Raptor SunSpider on Chromium" + try-name: raptor-sunspider-chromium + treeherder-symbol: Rap-Cr(ss) + mozharness: + extra-options: + - --test=raptor-sunspider + - --app=chromium + +raptor-unity-webgl-chromium: + description: "Raptor Unity WebGL on Chromium" + try-name: raptor-unity-webgl-chromium + treeherder-symbol: Rap-Cr(ugl) + tier: 3 + mozharness: + extra-options: + - --test=raptor-unity-webgl + - --app=chromium + fetches: + fetch: + by-test-platform: + win.*64.*: + - win64-chromium + - unity-webgl + win.*32.*: + - win32-chromium + - unity-webgl + macosx.*: + - mac-chromium + - unity-webgl + default: + - linux64-chromium + - unity-webgl + +raptor-wasm-misc-chromium: + description: "Raptor WASM Misc on Chromium" + try-name: raptor-wasm-misc-chromium + treeherder-symbol: Rap-Cr(wm) + mozharness: + extra-options: + - --test=raptor-wasm-misc + - --app=chromium + fetches: + fetch: + by-test-platform: + win.*64.*: + - win64-chromium + - wasm-misc + win.*32.*: + - win32-chromium + - wasm-misc + macosx.*: + - mac-chromium + - wasm-misc + default: + - linux64-chromium + - wasm-misc + +raptor-assorted-dom-chromium: + description: "Raptor Assorted-Dom on Chromium" + try-name: raptor-assorted-dom-chromium + treeherder-symbol: Rap-Cr(dom) + max-run-time: 1500 + mozharness: + extra-options: + - --test=raptor-assorted-dom + - --app=chromium + fetches: + fetch: + by-test-platform: + win.*64.*: + - win64-chromium + - assorted-dom + win.*32.*: + - win32-chromium + - assorted-dom + macosx.*: + - mac-chromium + - assorted-dom + default: + - linux64-chromium + - assorted-dom + +raptor-wasm-godot-chromium: + description: "Raptor Wasm Godot on Chromium" + try-name: raptor-wasm-godot-chromium + treeherder-symbol: Rap-Cr(godot) + max-run-time: 1500 + mozharness: + extra-options: + - --test=raptor-wasm-godot + - --app=chromium + +raptor-ares6-chromium: + description: "Raptor Ares6 on Chromium" + try-name: raptor-ares6-chromium + treeherder-symbol: Rap-Cr(ares6) + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + - --app=chromium + +raptor-jetstream2-chromium: + description: "Raptor Jetstream2 on Chromium" + try-name: raptor-jetstream2-chromium + treeherder-symbol: Rap-Cr(js2) + tier: 3 + max-run-time: 8000 + mozharness: + extra-options: + - --test=raptor-jetstream2 + - --app=chromium + fetches: + fetch: + by-test-platform: + win.*64.*: + - win64-chromium + - jetstream2 + win.*32.*: + - win32-chromium + - jetstream2 + macosx.*: + - mac-chromium + - jetstream2 + default: + - linux64-chromium + - jetstream2 diff --git a/taskcluster/ci/test/raptor-gve.yml b/taskcluster/ci/test/raptor-gve.yml new file mode 100644 index 0000000000..76cfaf8c66 --- /dev/null +++ b/taskcluster/ci/test/raptor-gve.yml @@ -0,0 +1,120 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 1800 + suite: raptor + webrender-run-on-projects: [] + variants: ["webrender"] + run-on-projects: + by-test-platform: + android-hw-p2-.*-api-16/(?!opt).*: [] + android-hw-p2-.*-api-16-shippable/opt: [] + android-hw(?!-p2).*(?<!-api-16-shippable)/opt.*: [] + default: ['mozilla-central'] + target: geckoview_example.apk + test-manifest-loader: null # don't load tests in the taskgraph + tier: 1 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --no-conditioned-profile + optimization: + skip-unless-backstop: null + +raptor-speedometer-geckoview-cpu-memory: + description: "Raptor Speedometer cpu/memory on GeckoView" + try-name: raptor-speedometer-geckoview-cpu-memory + treeherder-symbol: Rap(sp-cm) + target: geckoview_example.apk + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-speedometer + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --page-cycles 5 + - --activity=org.mozilla.geckoview_example.GeckoViewActivity + +raptor-speedometer-geckoview-cpu-memory-power: + description: "Raptor Speedometer cpu/memory/power on GeckoView" + try-name: raptor-speedometer-geckoview-cpu-memory-power + treeherder-symbol: Rap(sp-cmp) + target: geckoview_example.apk + run-on-projects: [] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-speedometer + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --power-test + - --page-cycles 5 + - --activity=org.mozilla.geckoview_example.GeckoViewActivity + +raptor-scn-cpu-memory-idle-geckoview: + description: "Raptor idle-browser cpu/memory on GeckoView" + try-name: raptor-scn-cpu-memory-idle-geckoview + treeherder-symbol: Rap(idl-cm) + run-on-projects: [] + mozharness: + extra-options: + - --test=raptor-scn-power-idle + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --activity=org.mozilla.geckoview_example.GeckoViewActivity + +raptor-scn-cpu-memory-power-idle-geckoview: + description: "Raptor idle-browser cpu/memory/power on GeckoView" + try-name: raptor-scn-cpu-memory-power-idle-geckoview + treeherder-symbol: Rap(idl-cmp) + run-on-projects: [] + mozharness: + extra-options: + - --test=raptor-scn-power-idle + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --power-test + - --activity=org.mozilla.geckoview_example.GeckoViewActivity + +raptor-scn-cpu-memory-idle-bg-geckoview: + description: "Raptor idle-browser (backgrounded) cpu/memory on GeckoView" + try-name: raptor-scn-cpu-memory-idle-bg-geckoview + treeherder-symbol: Rap(idlbg-cm) + run-on-projects: [] + mozharness: + extra-options: + - --test=raptor-scn-power-idle-bg + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --activity=org.mozilla.geckoview_example.GeckoViewActivity + +raptor-scn-cpu-memory-power-idle-bg-geckoview: + description: "Raptor idle-browser (backgrounded) cpu/memory/power on GeckoView" + try-name: raptor-scn-cpu-memory-power-idle-bg-geckoview + treeherder-symbol: Rap(idlbg-cmp) + run-on-projects: [] + mozharness: + extra-options: + - --test=raptor-scn-power-idle-bg + - --app=geckoview + - --binary=org.mozilla.geckoview_example + - --cpu-test + - --memory-test + - --power-test + - --activity=org.mozilla.geckoview_example.GeckoViewActivity diff --git a/taskcluster/ci/test/raptor-refbrow.yml b/taskcluster/ci/test/raptor-refbrow.yml new file mode 100644 index 0000000000..fa8edd1a8d --- /dev/null +++ b/taskcluster/ci/test/raptor-refbrow.yml @@ -0,0 +1,86 @@ +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 1800 + suite: raptor + webrender-run-on-projects: [] + variants: ["webrender"] + run-on-projects: [] + tier: 2 + test-manifest-loader: null # don't load tests in the taskgraph + target: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.reference-browser.raptor.latest.arm64-v8a + name: target.arm64-v8a.apk + default: + index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a + name: target.armeabi-v7a.apk + virtualization: hardware + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + +raptor-scn-cpu-memory-idle-refbrow: + description: "Raptor idle-browser cpu/memory on Reference Browser" + try-name: raptor-scn-cpu-memory-idle-refbrow + treeherder-symbol: Rap-refbrow(idl-cm) + tier: 3 + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-scn-power-idle + - --app=refbrow + - --binary-path=org.mozilla.reference.browser.raptor + - --cpu-test + - --memory-test + - --activity=org.mozilla.reference.browser.BrowserTestActivity + +raptor-scn-cpu-memory-power-idle-refbrow: + description: "Raptor idle-browser cpu/memory/power on Reference Browser" + try-name: raptor-scn-cpu-memory-power-idle-refbrow + treeherder-symbol: Rap-refbrow(idl-cmp) + tier: 3 + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-scn-power-idle + - --app=refbrow + - --binary-path=org.mozilla.reference.browser.raptor + - --cpu-test + - --memory-test + - --power-test + - --activity=org.mozilla.reference.browser.BrowserTestActivity + +raptor-scn-cpu-memory-idle-bg-refbrow: + description: "Raptor idle-browser (backgrounded) cpu/memory on Reference Browser" + try-name: raptor-scn-cpu-memory-idle-bg-refbrow + treeherder-symbol: Rap-refbrow(idlbg-cm) + tier: 3 + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-scn-power-idle-bg + - --app=refbrow + - --binary-path=org.mozilla.reference.browser.raptor + - --cpu-test + - --memory-test + - --activity=org.mozilla.reference.browser.BrowserTestActivity + +raptor-scn-cpu-memory-power-idle-bg-refbrow: + description: "Raptor idle-browser (backgrounded) cpu/memory/power on Reference Browser" + try-name: raptor-scn-cpu-memory-power-idle-bg-refbrow + treeherder-symbol: Rap-refbrow(idlbg-cmp) + tier: 3 + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-scn-power-idle-bg + - --app=refbrow + - --binary-path=org.mozilla.reference.browser.raptor + - --cpu-test + - --memory-test + - --power-test + - --activity=org.mozilla.reference.browser.BrowserTestActivity diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml new file mode 100644 index 0000000000..fc57f43a14 --- /dev/null +++ b/taskcluster/ci/test/raptor.yml @@ -0,0 +1,784 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-app: + firefox: + by-test-platform: + .*-qr/.*: 2400 + .*-ref-hw-2017/.*: 3600 + default: 1800 + default: + by-test-platform: + .*-qr/.*: 2400 + .*-ux/.*: 2400 + default: 1800 + suite: raptor + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: + by-app: + firefox: + by-test-name: + raptor-tp6-firefox: + by-pageload: + warm: + by-test-platform: + windows7-32.*/opt: [] + (linux|windows10|macos)(?!.*shippable).*: [] + linux.*shippable.*: [] + macos.*shippable.*: [] + default: ["mozilla-central"] + cold: + by-test-platform: + windows7-32.*/opt: [] + macosx.*64(?:-qr)?/opt: ["mozilla-central"] + (linux|windows10|macos)(?!.*shippable).*: [] + linux.*shippable.*: [] + macos.*shippable.*: [] + default: ["trunk", "mozilla-beta"] + default: + by-test-platform: + windows7-32.*/opt: [] + macosx.*64(?:-qr)?/opt: ["mozilla-central"] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ["trunk", "mozilla-beta"] + chrome: [] + chromium: [] + default: # tests that dont have "apps: []" + by-test-name: + raptor-.*-profiling.*: + by-test-platform: + windows7-32.*/opt: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ["mozilla-central"] + default: + by-test-platform: + windows7-32.*/opt: [] + macosx.*64(?:-qr)?/opt: ["mozilla-central"] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ["trunk", "mozilla-beta"] + limit-platforms: + by-app: + chrome: + - linux64-shippable/opt + - windows7-32-shippable/opt + - windows10-64-shippable/opt + - macosx1014-64-shippable/opt + chromium: + - linux64-shippable/opt + - windows7-32-shippable/opt + - windows10-64-shippable/opt + - macosx1014-64-shippable/opt + default: + - .* + tier: + by-app: + firefox: + by-test-platform: + windows10-64-ref-hw-2017/opt: 2 + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + default: 2 + fission-tier: 2 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + windows10-64-ccov.*/.*: + - raptor/windows_vm_config.py + linux64-ccov.*/.*: + - raptor/linux64_config_taskcluster.py + default: + - raptor/linux_config.py + extra-options: + # Bug 1626522: Temporarily enable extension logging for + # an easier investigation of intermittents on mobile devices + - --setpref=extensions.logging.enabled=true + # Bug 1663449: Disable conditioned profiles for raptor tests because of test failures + - --no-conditioned-profile + fission-run-on-projects: + by-app: + chrome: [] + chromium: [] + default: + by-test-platform: + linux64-shippable-qr/opt: ["mozilla-central"] + windows10-64-shippable-qr/opt: ["mozilla-central"] + default: [] + fetches: + fetch: + by-app: + chromium: + by-test-platform: + win.*64.*: + - win64-chromium + win.*32.*: + - win32-chromium + macosx.*: + - mac-chromium + default: + - linux64-chromium + default: [] + +raptor-tp6: + description: "Raptor (webextension) tp6" + apps: ["firefox", "chrome", "chromium"] + variants: + # Used for defining webrender and fission build variant tests, + by-app: + firefox: + by-subtest: + yahoo-mail: [] + wikipedia: [] + default: ["fission"] + chrome: [] + chromium: [] + default: [] + pageload: + by-app: + firefox: "both" + chrome: "both" + chromium: "both" + default: "cold" + limit-platforms: + by-app: + chrome: + # Run on non-QR shippable, but test-platforms.yml doesn't have + # any more non-QR macosx shippable things. So for macosx we want + # to run it on shippable-qr, otherwise there's nothing running + # this on macosx at all. + - .*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + chromium: + # Ditto as for chrome + - .*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + default: [] + treeherder-symbol: Rap(tp6) + max-run-time: 4000 + raptor-test: tp6 + raptor-subtests: + - amazon + - apple + - bing + - ebay + - [facebook, fb] + - [facebook-redesign, fb-r] + - fandom + - [docs, gdoc] + - google + - [google-mail, gmail] + - [sheets, gsheets] + - [slides, gslides] + - imdb + - imgur + - instagram + - linkedin + - microsoft + - netflix + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - yahoo-news + - yandex + - youtube + tier: + by-app: + firefox: + by-subtest: + amazon: 1 + .*gmail.*: 1 + .*gslides.*: 1 + imgur: 1 + tumblr: 1 + twitch: 1 + twitter: 1 + default: 2 + default: 2 + +raptor-tp6-live: + description: "Raptor (webextension) tp6 on live-sites" + apps: ["firefox"] + variants: [] + pageload: "both" + treeherder-symbol: Rap-live(tp6) + raptor-test: tp6 + raptor-subtests: + - [cnn-ampstories, cnn-amp] + run-on-projects: + by-pageload: + warm: + by-test-platform: + windows7-32.*/opt: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ["mozilla-central"] + cold: + by-test-platform: + windows7-32.*/opt: [] + macosx.*64(?:-qr)?/opt: ["mozilla-central"] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ["mozilla-central", "autoland"] + tier: 2 + mozharness: + extra-options: + - --no-conditioned-profile + - --live-sites + +raptor-speedometer-firefox: + description: "Raptor Speedometer on Firefox" + variants: ["fission"] + try-name: raptor-speedometer-firefox + treeherder-symbol: Rap(sp) + mozharness: + extra-options: + - --test=raptor-speedometer + +raptor-speedometer-firefox-profiling: + description: "Raptor Speedometer on Firefox with Gecko Profiling" + try-name: raptor-speedometer-firefox-profiling + treeherder-symbol: Rap-Prof(sp) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-speedometer + - --gecko-profile + +raptor-stylebench-firefox: + description: "Raptor StyleBench on Firefox" + variants: ["fission"] + try-name: raptor-stylebench-firefox + treeherder-symbol: Rap(sb) + mozharness: + extra-options: + - --test=raptor-stylebench + +raptor-stylebench-firefox-profiling: + description: "Raptor StyleBench on Firefox with Gecko Profiling" + try-name: raptor-stylebench-firefox-profiling + treeherder-symbol: Rap-Prof(sb) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-stylebench + - --gecko-profile + +raptor-motionmark-htmlsuite-firefox: + description: "Raptor MotionMark HtmlSuite on Firefox" + variants: ["fission"] + try-name: raptor-motionmark-htmlsuite-firefox + treeherder-symbol: Rap(mm-h) + mozharness: + extra-options: + - --test=raptor-motionmark-htmlsuite + +raptor-motionmark-htmlsuite-firefox-profiling: + description: "Raptor MotionMark HtmlSuite on Firefox with Gecko Profiling" + try-name: raptor-motionmark-htmlsuite-firefox-profiling + treeherder-symbol: Rap-Prof(mm-h) + mozharness: + extra-options: + - --test=raptor-motionmark-htmlsuite + - --gecko-profile + +raptor-motionmark-animometer-firefox: + description: "Raptor MotionMark Animometer on Firefox" + variants: ["fission"] + try-name: raptor-motionmark-animometer-firefox + treeherder-symbol: Rap(mm-a) + tier: 1 + mozharness: + extra-options: + - --test=raptor-motionmark-animometer + +raptor-motionmark-animometer-firefox-profiling: + description: "Raptor MotionMark Animometer on Firefox with Gecko Profiling" + try-name: raptor-motionmark-animometer-firefox-profiling + treeherder-symbol: Rap-Prof(mm-a) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-motionmark-animometer + - --gecko-profile + +raptor-webaudio-firefox: + description: "Raptor WebAudio on Firefox" + variants: ["fission"] + try-name: raptor-webaudio-firefox + treeherder-symbol: Rap(wa) + tier: 1 + mozharness: + extra-options: + - --test=raptor-webaudio + +raptor-webaudio-firefox-profiling: + description: "Raptor WebAudio on Firefox with Gecko Profiling" + try-name: raptor-webaudio-firefox + treeherder-symbol: Rap-Prof(wa) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-webaudio + - --gecko-profile + +raptor-sunspider-firefox: + description: "Raptor SunSpider on Firefox" + variants: ["fission"] + try-name: raptor-sunspider-firefox + treeherder-symbol: Rap(ss) + mozharness: + extra-options: + - --test=raptor-sunspider + +raptor-sunspider-firefox-profiling: + description: "Raptor SunSpider on Firefox with Gecko Profiling" + try-name: raptor-sunspider-firefox-profiling + treeherder-symbol: Rap-Prof(ss) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-sunspider + - --gecko-profile + +raptor-unity-webgl-firefox: + description: "Raptor Unity WebGL on Firefox" + variants: ["fission"] + try-name: raptor-unity-webgl-firefox + treeherder-symbol: Rap(ugl) + run-on-projects: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ["autoland"] + tier: 3 + mozharness: + extra-options: + - --test=raptor-unity-webgl + fetches: + fetch: + - unity-webgl + +raptor-unity-webgl-firefox-profiling: + description: "Raptor Unity WebGL on Firefox with Gecko Profiling" + try-name: raptor-unity-webgl-firefox-profiling + treeherder-symbol: Rap-Prof(ugl) + mozharness: + extra-options: + - --test=raptor-unity-webgl + - --gecko-profile + fetches: + fetch: + - unity-webgl + +raptor-wasm-misc-firefox: + description: "Raptor WASM Misc on Firefox" + variants: ["fission"] + try-name: raptor-wasm-misc-firefox + treeherder-symbol: Rap(wm) + mozharness: + extra-options: + - --test=raptor-wasm-misc + fetches: + fetch: + - wasm-misc + +raptor-wasm-misc-firefox-profiling: + description: "Raptor WASM Misc on Firefox with Gecko Profiling" + try-name: raptor-wasm-misc-firefox-profiling + treeherder-symbol: Rap-Prof(wm) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-misc + - --gecko-profile + fetches: + fetch: + - wasm-misc + +raptor-wasm-misc-baseline-firefox: + description: "Raptor WASM Misc on Firefox with baseline JIT" + variants: ["fission"] + try-name: raptor-wasm-misc-baseline-firefox + treeherder-symbol: Rap(wm-b) + mozharness: + extra-options: + - --test=raptor-wasm-misc-baseline + fetches: + fetch: + - wasm-misc + +raptor-wasm-misc-baseline-firefox-profiling: + description: "Raptor WASM Misc on Firefox with baseline JIT and Gecko Profiling" + try-name: raptor-wasm-misc-baseline-firefox-profiling + treeherder-symbol: Rap-Prof(wm-b) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-misc-baseline + - --gecko-profile + fetches: + fetch: + - wasm-misc + +raptor-wasm-misc-optimizing-firefox: + description: "Raptor WASM Misc on Firefox with optimizing JIT" + variants: ["fission"] + try-name: raptor-wasm-misc-optimizing-firefox + treeherder-symbol: Rap(wm-i) + mozharness: + extra-options: + - --test=raptor-wasm-misc-optimizing + fetches: + fetch: + - wasm-misc + +raptor-wasm-misc-optimizing-firefox-profiling: + description: "Raptor WASM Misc on Firefox with optimizing JIT and Gecko Profiling" + try-name: raptor-wasm-misc-optimizing-firefox-profiling + treeherder-symbol: Rap-Prof(wm-i) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-misc-optimizing + - --gecko-profile + fetches: + fetch: + - wasm-misc + +raptor-assorted-dom-firefox: + description: "Raptor Assorted-Dom on Firefox" + variants: ["fission"] + try-name: raptor-assorted-dom-firefox + treeherder-symbol: Rap(dom) + max-run-time: + by-test-platform: + .*-qr/.*: 2100 + .*-ref-hw-2017/.*: 2100 + default: 1500 + mozharness: + extra-options: + - --test=raptor-assorted-dom + fetches: + fetch: + - assorted-dom + +raptor-assorted-dom-firefox-profiling: + description: "Raptor Assorted-Dom on Firefox with Gecko Profiling" + try-name: raptor-assorted-dom-firefox-profiling + treeherder-symbol: Rap-Prof(dom) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-assorted-dom + - --gecko-profile + fetches: + fetch: + - assorted-dom + +raptor-wasm-godot-firefox: + description: "Raptor Wasm Godot on Firefox" + variants: ["fission"] + try-name: raptor-wasm-godot-firefox + treeherder-symbol: Rap(godot) + max-run-time: + by-test-platform: + .*-qr/.*: 2100 + .*-ref-hw-2017/.*: 2100 + default: 1500 + mozharness: + extra-options: + - --test=raptor-wasm-godot + +raptor-wasm-godot-firefox-profiling: + description: "Raptor Wasm Godot on Firefox with Gecko Profiling" + try-name: raptor-wasm-godot-firefox-profiling + treeherder-symbol: Rap-Prof(godot) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-godot + - --gecko-profile + +raptor-wasm-godot-baseline-firefox: + description: "Raptor Wasm Godot on Firefox with baseline JIT" + variants: ["fission"] + try-name: raptor-wasm-godot-baseline-firefox + treeherder-symbol: Rap(godot-b) + max-run-time: + by-test-platform: + .*-qr/.*: 2100 + .*-ref-hw-2017/.*: 2100 + default: 1500 + mozharness: + extra-options: + - --test=raptor-wasm-godot-baseline + +raptor-wasm-godot-baseline-firefox-profiling: + description: "Raptor Wasm Godot on Firefox with baseline JIT and Gecko Profiling" + try-name: raptor-wasm-godot-baseline-firefox-profiling + treeherder-symbol: Rap-Prof(godot-b) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-godot-baseline + - --gecko-profile + +raptor-wasm-godot-optimizing-firefox: + description: "Raptor WASM godot on Firefox with optimizing JIT" + variants: ["fission"] + try-name: raptor-wasm-godot-optimizing-firefox + treeherder-symbol: Rap(godot-i) + max-run-time: + by-test-platform: + .*-qr/.*: 2100 + .*-ref-hw-2017/.*: 2100 + default: 1500 + mozharness: + extra-options: + - --test=raptor-wasm-godot-optimizing + +raptor-wasm-godot-optimizing-firefox-profiling: + description: "Raptor WASM godot on Firefox with optimizing JIT and Gecko Profiling" + try-name: raptor-wasm-godot-optimizing-firefox-profiling + treeherder-symbol: Rap-Prof(godot-i) + max-run-time: 900 + mozharness: + extra-options: + - --test=raptor-wasm-godot-optimizing + - --gecko-profile + +raptor-youtube-playback-v9-power-firefox: + description: "Raptor V9 YouTube Playback on Firefox with Power Testing" + try-name: raptor-youtube-playback-v9-power-firefox + treeherder-symbol: Rap(ytp-v9-p) + run-on-projects: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ["mozilla-central"] + max-run-time: 2700 + tier: 3 + mozharness: + extra-options: + - --test=raptor-youtube-playback-v9 + - --power-test + +raptor-youtube-playback-h264-power-firefox: + description: "Raptor H264 YouTube Playback on Firefox with Power Testing" + try-name: raptor-youtube-playback-h264-power-firefox + treeherder-symbol: Rap(ytp-h264-p) + run-on-projects: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ["mozilla-central"] + max-run-time: 2700 + tier: 3 + mozharness: + extra-options: + - --test=raptor-youtube-playback-h264 + - --power-test + +raptor-youtube-playback-av1-sfr-firefox: + description: "Raptor YouTube Playback AV1 SFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-av1-sfr-firefox + treeherder-symbol: Rap(ytp-av1-sfr) + run-on-projects: + by-test-platform: + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + limit-platforms: + - windows10-64-ref-hw-2017/opt + - macosx1014-64-shippable-qr/opt + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-av1-sfr + +raptor-youtube-playback-h264-sfr-firefox: + description: "Raptor YouTube Playback H264 SFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-h264-sfr-firefox + treeherder-symbol: Rap(ytp-h264-sfr) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-h264-sfr + +raptor-youtube-playback-hfr-firefox: + description: "Raptor YouTube Playback HFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-hfr-firefox + treeherder-symbol: Rap(ytp-hfr) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: 3600 + tier: 2 + mozharness: + extra-options: + - --test=raptor-youtube-playback-hfr + +raptor-youtube-playback-vp9-sfr-firefox: + description: "Raptor YouTube Playback VP9 SFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-vp9-sfr-firefox + treeherder-symbol: Rap(ytp-vp9-sfr) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + limit-platforms: + - android-hw-p2.* + mozharness: + extra-options: + - --test=raptor-youtube-playback-vp9-sfr + +raptor-youtube-playback-widevine-hfr-firefox: + description: "Raptor YouTube Playback Widevine HFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-widevine-hfr-firefox + treeherder-symbol: Rap(ytp-widevine-hfr) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-hfr + +raptor-youtube-playback-widevine-h264-sfr-firefox: + description: "Raptor YouTube Playback Widevine H264 SFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-widevine-h264-sfr-firefox + treeherder-symbol: Rap(ytp-widevine-h264-sfr) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-h264-sfr + +raptor-youtube-playback-widevine-vp9-sfr-firefox: + description: "Raptor YouTube Playback Widevine VP9 SFR on Firefox" + variants: ["fission"] + try-name: raptor-youtube-playback-widevine-vp9-sfr-firefox + treeherder-symbol: Rap(ytp-widevine-vp9-sfr) + run-on-projects: + by-test-platform: + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: + by-test-platform: + windows10-aarch64/opt: 3600 + default: 2700 + limit-platforms: + - android-hw-p2.* + mozharness: + extra-options: + - --test=raptor-youtube-playback-widevine-vp9-sfr + +raptor-ares6-firefox: + description: "Raptor Ares6 on Firefox" + variants: ["fission"] + try-name: raptor-ares6-firefox + treeherder-symbol: Rap(ares6) + max-run-time: 2500 + mozharness: + extra-options: + - --test=raptor-ares6 + +raptor-ares6-firefox-profiling: + description: "Raptor Ares6 on Firefox with Gecko Profiling" + try-name: raptor-ares6-firefox-profiling + treeherder-symbol: Rap-Prof(ares6) + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + - --gecko-profile + +raptor-jetstream2-firefox: + description: "Raptor Jetstream2 on Firefox" + variants: ["fission"] + try-name: raptor-jetstream2-firefox + treeherder-symbol: Rap(js2) + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + tier: 3 + max-run-time: 8000 + mozharness: + extra-options: + - --test=raptor-jetstream2 + fetches: + fetch: + - jetstream2 + +raptor-jetstream2-firefox-profiling: + description: "Raptor Jetstream2 on Firefox with Gecko Profiling" + try-name: raptor-jetstream2-firefox-profiling + treeherder-symbol: Rap-Prof(js2) + max-run-time: 8000 + tier: 3 + mozharness: + extra-options: + - --test=raptor-jetstream2 + - --gecko-profile + fetches: + fetch: + - jetstream2 diff --git a/taskcluster/ci/test/reftest.yml b/taskcluster/ci/test/reftest.yml new file mode 100644 index 0000000000..4847147ef3 --- /dev/null +++ b/taskcluster/ci/test/reftest.yml @@ -0,0 +1,235 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: reftest + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + android-hw-.*: geckoview-androidTest.apk + default: null + variants: ['fission'] + fission-tier: 2 + run-on-projects: + by-test-platform: + android-hw-.*-qr/debug: ['mozilla-central'] + android-hw-.*-qr/opt: ['mozilla-central', 'autoland'] + windows10-aarch64/.*: [] # Specific repos enabled below + default: built-projects + test-manifest-loader: null # don't load tests in the taskgraph + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +crashtest: + description: "Crashtest run" + schedules-component: crashtest + treeherder-symbol: R(C) + chunks: + by-test-platform: + android-em-7.*: 1 + default: 1 + e10s: true + run-on-projects: + by-test-platform: + windows10-aarch64/.*: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + default: built-projects + max-run-time: 3600 + tier: + by-test-platform: + windows10-aarch64.*: 2 + android-hw-.*-qr/.*: 2 + default: default + fission-tier: + by-test-platform: + linux1804-64-qr/debug: 1 + default: 2 + fission-run-on-projects: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + default: [] + +crashtest-qr: + description: "Crashtest Webrender run" + schedules-component: crashtest + treeherder-symbol: R(C) + chunks: 1 + e10s: true + run-on-projects: built-projects + max-run-time: 1200 + tier: 2 + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + +jsreftest: + description: "JS Reftest run" + schedules-component: jsreftest + treeherder-symbol: R(J) + instance-size: default + chunks: + by-test-platform: + android-hw.*\/debug: 12 + android-hw.*/(opt|pgo)?: 6 + windows.*\/opt: 2 + windows10-64-asan/opt: 3 + windows10-64-ccov-qr/opt: 5 + linux.*64-ccov.*/.*: 5 + linux1804-64-qr/opt: 4 + linux1804-64-qr/debug: 5 + linux1804-64/debug: 5 + macosx.*-64-ccov.*/.*: 5 + macosx.*-64(-shippable)?/opt: 2 + default: 3 + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 7200 + macosx.*64-ccov.*/.*: 7200 + linux.*64-ccov.*/.*: 7200 + default: 3600 + tier: default + run-on-projects: + by-test-platform: + android-hw-.*-aarch64(?:-shippable)?/.*: ['mozilla-central', 'release'] + android-hw-.*-api-16(?:-shippable)?/.*: ['mozilla-central', 'release'] + default: built-projects + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + +reftest: + description: "Reftest run" + treeherder-symbol: R(R) + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + variants: + by-test-platform: + linux1804-64-qr/debug: ['webrender-sw', 'fission'] + windows10-64-qr/debug: ['webrender-sw', 'fission'] + default: ['fission'] + chunks: + by-test-platform: + android-em-7.*: 2 + linux1804-64(-shippable|-devedition|-qr)?/opt: 5 + macosx101.*-64-shippable(-qr)?/opt: 3 + macosx101.*-64/debug: 4 + macosx101.*-64(-ccov.*|-qr)/debug: 6 + windows.*/opt: 2 + windows.*/debug: 4 + windows10-64-asan/opt: 3 + windows10-64-ccov-qr/opt: 9 + default: 8 + e10s: true + run-on-projects: + by-test-platform: + windows10-aarch64/.*: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] + default: built-projects + max-run-time: + by-test-platform: + windows7-32-mingwclang/debug: 5400 + windows10-64-ccov.*/.*: 5400 + windows10-64-asan/opt: 5400 + macosx.*64-ccov.*/.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + android-em.*: false + macosx.*64/opt: false + windows10-64.*/opt: false + default: true + tier: + by-test-platform: + windows10-aarch64.*: 2 + android-hw-.*-qr/.*: 2 + default: default + fission-tier: + by-test-platform: + linux1804-64-qr/debug: 1 + default: 2 + fission-run-on-projects: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + default: [] + +reftest-qr: + description: "Reftest webrender run" + treeherder-symbol: R(R) + chunks: 2 + e10s: true + run-on-projects: built-projects + max-run-time: 2400 + mozharness: + chunked: true + tier: 2 + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + + +reftest-gpu: + description: "Reftest GPU run" + treeherder-symbol: R(Rg) + chunks: + by-test-platform: + windows.*/opt: 2 + default: 4 + run-on-projects: + by-test-platform: + windows10.*: [] + default: built-projects + instance-size: default + virtualization: virtual-with-gpu + max-run-time: 3600 + mozharness: + chunked: true + tier: default + +reftest-no-accel: + description: "Reftest not accelerated run" + treeherder-symbol: R(Ru) + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + run-on-projects: + by-test-platform: + windows10.*: [] + default: built-projects + chunks: + by-test-platform: + macosx.*: 1 + windows.*: 4 + linux1804-64(-shippable|-devedition)?/opt: 4 + default: 8 + mozharness: + chunked: + by-test-platform: + windows10-64.*/opt: false + macosx.*: false + default: true diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml new file mode 100644 index 0000000000..4e758416ba --- /dev/null +++ b/taskcluster/ci/test/talos.yml @@ -0,0 +1,646 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: talos + tier: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: default + variants: ['fission'] + fission-tier: 2 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + mozharness: + script: talos_script.py + config: + by-test-platform: + macosx.*: + - talos/mac_config.py + windows.*: + - talos/windows_config.py + windows10-64-ccov-qr/opt: + - talos/windows_vm_config.py + linux64-ccov.*/.*: + - talos/linux64_config_taskcluster.py + default: + - talos/linux_config.py + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: + by-test-name: + talos-.*-profiling: + by-test-platform: + windows7-32.*: [] + (linux|windows10|macos)(?!.*shippable).*: [] + .*(?:shippable-qr).*: [] + windows10.*(?!shippable-qr)shippable.*: [] + default: ['mozilla-central'] + default: + by-test-platform: + windows7-32.*: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-beta', 'trunk'] + fission-run-on-projects: + by-test-platform: + (linux64|windows10-64|macosx1014-64)-shippable-qr/opt: ['mozilla-central'] + default: [] + +talos-bcv: + description: "Talos basic compositor video" + try-name: bcv + treeherder-symbol: T(bcv) + variants: + by-test-platform: + (linux64|windows10|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + run-on-projects: + by-test-platform: + .(linux|windows10|macos)(?:shippable-qr|-qr)/.*: + by-variant: + webrender-sw: ['trunk'] + default: [] + windows7-32.*: [] + (linux|windows10|macos)(?!.*shippable)(?!.*-qr).*: [] + default: ['mozilla-beta', 'trunk'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=bcv + +talos-bcv-profiling: + description: "Talos basic compositor video with gecko profiling" + try-name: bcv-prof + treeherder-symbol: T-Prof(bcv) + tier: 2 + run-on-projects: + by-test-platform: + windows7-32.*: [] + .*(?:shippable-qr|-qr)/.*: [] + (linux|windows10|macos)(?!.*shippable)(?!.*-qr).*: [] + windows10-64-shippable/opt: [] + default: ['mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=bcv + - --gecko-profile + +talos-chrome: + description: "Talos chrome" + try-name: chromez + treeherder-symbol: T(c) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=chromez + +talos-chrome-profiling: + description: "Talos chrome with gecko profiling" + try-name: chromez-prof + treeherder-symbol: T-Prof(c) + tier: 2 + max-run-time: 1200 + mozharness: + extra-options: + - --suite=chromez + - --gecko-profile + +talos-damp: + description: "Talos devtools (damp)" + try-name: damp + treeherder-symbol: T(damp) + max-run-time: 5400 + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + run-on-projects: + by-test-platform: + windows7-32-shippable/.*: [] + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-beta', 'trunk'] + fission-run-on-projects: + by-test-platform: + (linux64|windows10-64)-shippable-qr/opt: ['mozilla-central'] + # Bug 1657864 - Disabled on macos for high frequency intermittents + default: [] + + mozharness: + extra-options: + - --suite=damp + +talos-damp-profiling: + description: "Talos devtools (damp) with gecko profiling" + try-name: damp-prof + treeherder-symbol: T-Prof(damp) + tier: 2 + max-run-time: 5400 + run-on-projects: + by-test-platform: + windows7-32-shippable/.*: [] + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + mozharness: + extra-options: + - --suite=damp + - --gecko-profile + +talos-dromaeojs: + description: "Talos dromaeojs" + try-name: dromaeojs + treeherder-symbol: T(d) + max-run-time: 2100 + mozharness: + extra-options: + - --suite=dromaeojs + +talos-dromaeojs-profiling: + description: "Talos dromaeojs with gecko profiling" + try-name: dromaeojs-prof + treeherder-symbol: T-Prof(d) + tier: 2 + max-run-time: 2100 + mozharness: + extra-options: + - --suite=dromaeojs + - --gecko-profile + +talos-flex: + description: "Talos XUL flexbox emulation enabled" + try-name: flex + treeherder-symbol: T(f) + tier: 3 + run-on-projects: + by-test-platform: + windows7-32.*: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=flex + +talos-flex-profiling: + description: "Talos XUL flexbox emulation enabled with gecko profiling" + try-name: flex-prof + treeherder-symbol: T-Prof(f) + tier: 3 + run-on-projects: + by-test-platform: + windows7-32.*: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=flex + - --gecko-profile + +talos-g1: + description: "Talos g1" + try-name: g1 + treeherder-symbol: T(g1) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: + by-test-platform: + linux64.*: 3600 + default: 7200 + mozharness: + extra-options: + - --suite=g1 + +talos-g1-profiling: + description: "Talos g1 with gecko profiling" + try-name: g1-prof + treeherder-symbol: T-Prof(g1) + tier: 2 + max-run-time: + by-test-platform: + linux64.*: 3600 + default: 7200 + mozharness: + extra-options: + - --suite=g1 + - --gecko-profile + +talos-g3: + description: "Talos g3" + try-name: g3 + treeherder-symbol: T(g3) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 900 + mozharness: + extra-options: + - --suite=g3 + +talos-g3-profiling: + description: "Talos g3 with gecko profiling" + try-name: g3-prof + treeherder-symbol: T-Prof(g3) + tier: 2 + max-run-time: 900 + mozharness: + extra-options: + - --suite=g3 + - --gecko-profile + +talos-g4: + description: "Talos g4" + try-name: g4 + treeherder-symbol: T(g4) + variants: + by-test-platform: + windows7-32.*: [] + (linux64|windows10|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + run-on-projects: + by-test-platform: + windows7-32.*: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-beta', 'trunk'] + max-run-time: + by-test-platform: + linux64.*: 1500 + default: 1800 + mozharness: + extra-options: + - --suite=g4 + +talos-g4-profiling: + description: "Talos g4 with gecko profiling" + try-name: g4-prof + treeherder-symbol: T-Prof(g4) + tier: 2 + max-run-time: + by-test-platform: + linux64.*: 1500 + default: 1800 + mozharness: + extra-options: + - --suite=g4 + - --gecko-profile + +talos-g5: + description: "Talos g5" + try-name: g5 + treeherder-symbol: T(g5) + mozharness: + extra-options: + - --suite=g5 + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: + by-test-platform: + linux64.*: 1200 + macosx.*: 2400 + default: 1800 + +talos-g5-profiling: + description: "Talos g5 with gecko profiling" + try-name: g5-prof + treeherder-symbol: T-Prof(g5) + tier: 2 + mozharness: + extra-options: + - --suite=g5 + - --gecko-profile + max-run-time: + by-test-platform: + linux64.*: 1200 + default: 1800 + +talos-h1: + description: "Talos h1" + try-name: h1 + treeherder-symbol: T(h1) + mozharness: + extra-options: + - --suite=h1 + max-run-time: + by-test-platform: + linux64.*: 900 + default: 1800 + +talos-h2: + description: "Talos h2" + try-name: h2 + treeherder-symbol: T(h2) + max-run-time: + by-test-platform: + linux64.*: 900 + default: 7200 + mozharness: + extra-options: + - --suite=h2 + +talos-motionmark: + description: "Talos motionmark" + try-name: motionmark + treeherder-symbol: T(mm) + run-on-projects: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: 3600 + tier: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + mozharness: + extra-options: + - --suite=motionmark + +talos-motionmark-profiling: + description: "Talos motionmark with gecko profiling" + try-name: motionmark-prof + treeherder-symbol: T-Prof(mm) + max-run-time: 3600 + tier: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + mozharness: + extra-options: + - --suite=motionmark + - --gecko-profile + +talos-other: + description: "Talos other" + try-name: other + treeherder-symbol: T(o) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=other + +talos-other-profiling: + description: "Talos other with gecko profiling" + try-name: other-prof + treeherder-symbol: T-Prof(o) + tier: 2 + max-run-time: 1500 + mozharness: + extra-options: + - --suite=other + - --gecko-profile + +talos-sessionrestore-many-windows: + description: "Talos sessionrestore-many-windows" + try-name: sessionrestore-many-windows + treeherder-symbol: T(smw) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + run-on-projects: + by-test-platform: + windows7-32.*/opt: [] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=sessionrestore-many-windows + +talos-sessionrestore-many-windows-profiling: + description: "Talos sessionrestore-many-windows with gecko profiling" + try-name: sessionrestore-many-windows-prof + treeherder-symbol: T-Prof(smw) + tier: 2 + max-run-time: 1500 + mozharness: + extra-options: + - --suite=sessionrestore-many-windows + - --gecko-profile + +talos-perf-reftest: + description: "Talos perf-reftest" + try-name: perf-reftest + treeherder-symbol: T(p) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1200 + mozharness: + extra-options: + - --suite=perf-reftest + +talos-perf-reftest-profiling: + description: "Talos perf-reftest with gecko profiling" + try-name: perf-reftest-prof + treeherder-symbol: T-Prof(p) + tier: 2 + max-run-time: 1200 + mozharness: + extra-options: + - --suite=perf-reftest + - --gecko-profile + +talos-perf-reftest-singletons: + description: "Talos perf-reftest singletons" + try-name: perf-reftest-singletons + treeherder-symbol: T(ps) + max-run-time: 1500 + mozharness: + extra-options: + - --suite=perf-reftest-singletons + +talos-perf-reftest-singletons-profiling: + description: "Talos perf-reftest singletons with gecko profiling" + try-name: perf-reftest-singletons-prof + treeherder-symbol: T-Prof(ps) + tier: 2 + max-run-time: 1500 + mozharness: + extra-options: + - --suite=perf-reftest-singletons + - --gecko-profile + +talos-svgr: + description: "Talos svgr" + try-name: svgr + treeherder-symbol: T(s) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=svgr + +talos-svgr-profiling: + description: "Talos svgr with gecko profiling" + try-name: svgr-prof + treeherder-symbol: T-Prof(s) + tier: 2 + max-run-time: 1800 + mozharness: + extra-options: + - --suite=svgr + - --gecko-profile + +talos-tp5o: + description: "Talos tp5o" + try-name: tp5o + treeherder-symbol: T(tp) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=tp5o + +talos-tp5o-profiling: + description: "Talos tp5o with gecko profiling" + try-name: tp5o-prof + treeherder-symbol: T-Prof(tp) + tier: 2 + max-run-time: 1800 + mozharness: + extra-options: + - --suite=tp5o + - --gecko-profile + +talos-tabswitch: + description: "Talos page scroll (tabswitch)" + try-name: tabswitch + treeherder-symbol: T(tabswitch) + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + max-run-time: 1200 + mozharness: + extra-options: + - --suite=tabswitch + +talos-tabswitch-profiling: + description: "Talos page scroll (tabswitch) with gecko profiling" + try-name: tabswitch-prof + treeherder-symbol: T-Prof(tabswitch) + tier: 2 + max-run-time: 1200 + mozharness: + extra-options: + - --suite=tabswitch + - --gecko-profile + +talos-webgl: + description: "Talos WebGL" + try-name: webgl + treeherder-symbol: T(webgl) + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + windows7-32.*: [] + (linux64|windows10|macos).*-qr.*: ['webgl-ipc', 'webrender-sw'] + windows10-64-ccov-qr/opt: [] + default: ['webgl-ipc'] + run-on-projects: + by-test-platform: + windows7-32.*: [] + windows10-64-ref-hw-2017/opt: ['mozilla-central'] + (linux|windows10|macos)(?!.*shippable).*: [] + default: ['mozilla-beta', 'trunk'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=webgl + +talos-webgl-profiling: + description: "Talos WebGL with gecko profiling" + try-name: webgl + treeherder-symbol: T-Prof(webgl) + tier: 2 + run-on-projects: [] + variants: + by-test-platform: + .*-ccov.*/.*: [] + android.*: [] + windows7-32.*: [] + default: ['webgl-ipc'] + max-run-time: 1800 + mozharness: + extra-options: + - --suite=webgl + - --gecko-profile + +talos-xperf: + description: "Talos xperf" + try-name: xperf + treeherder-symbol: T(x) + virtualization: virtual + os-groups: ['Administrators'] + run-as-administrator: + by-test-platform: + windows7-32.*: false + windows10-64.*: true + variants: + by-test-platform: + (linux64|windows|macos).*-qr.*: ['fission', 'webrender-sw'] + default: ['fission'] + run-on-projects: + by-test-platform: + windows7-32-shippable(?:-qr)?/.*: [] + windows10-64-shippable(?:-qr)?/.*: ['mozilla-beta', 'trunk'] + default: [] + tier: + by-test-platform: + windows10-64-ccov.*/.*: 3 + default: default + mozharness: + extra-options: + - --suite=xperf + config: + by-test-platform: + windows.*: + - talos/windows_vm_config.py + +talos-realworld-webextensions: + description: "Talos tests with realworld WebExtensions" + try-name: "realworld-webextensions" + treeherder-symbol: T(realwebext) + max-run-time: 1500 + mozharness: + extra-options: + - --suite=realworld-webextensions + +talos-realworld-webextensions-profiling: + description: "Talos tests with realworld WebExtensions with gecko profiling" + try-name: "realworld-webextensions-prof" + treeherder-symbol: T-Prof(realwebext) + tier: 2 + max-run-time: 1500 + mozharness: + extra-options: + - --suite=realworld-webextensions + - --gecko-profile diff --git a/taskcluster/ci/test/test-platforms.yml b/taskcluster/ci/test/test-platforms.yml new file mode 100644 index 0000000000..24bd836ad2 --- /dev/null +++ b/taskcluster/ci/test/test-platforms.yml @@ -0,0 +1,537 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file maps build platforms to test platforms. In some cases, a +# single build may be tested on multiple test platforms, but a single test +# platform can only link to one build platform. Both build and test platforms +# are represented as <platform>/<type>, where <type> is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + + +## +# Linux platforms (matching /linux.*/) +# Linux1804-64 uses Ubuntu 18.04 and is run on docker + +linux1804-32-shippable/opt: + build-platform: linux-shippable/opt + test-sets: + - linux32-tests + +linux1804-64/opt: + build-platform: linux64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - marionette-framescript + - marionette-headless + - mochitest-headless + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +linux1804-64/debug: + build-platform: linux64/debug + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-shippable/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - marionette-framescript + - marionette-headless + - mochitest-headless + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +linux1804-64-devedition/opt: + build-platform: linux64-devedition/opt + test-sets: + - desktop-screenshot-capture + - linux1804-tests + - web-platform-tests + +linux1804-64-asan/opt: + build-platform: linux64-asan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-tsan/opt: + build-platform: linux64-tsan/opt + test-sets: + - linux-tsan-tests + +linux1804-64-qr/opt: + build-platform: linux64/opt + test-sets: + - awsy + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-qr/debug: + build-platform: linux64/debug + test-sets: + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-asan-qr/opt: + build-platform: linux64-asan/opt + test-sets: + - linux-qr-smoketests + +linux1804-64-ccov/opt: + build-platform: linux64-ccov/opt + test-sets: + - awsy + - linux1804-tests + - linux1804-ccov-tests + - web-platform-tests + - ccov-code-coverage-tests + - mochitest-webgpu + +# Linux64 uses Ubuntu 16.04 and is run on hardware + +linux64/opt: + build-platform: linux64/opt + test-sets: + - talos + - linux-talos-flex + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-shippable/opt: + build-platform: linux64-shippable/opt + test-sets: + - talos + - linux-talos-flex + - raptor-chrome + - raptor-chromium + - raptor-fetch-chromium + - raptor-firefox + - browsertime + - raptor-fetch-firefox + - raptor-profiling + - raptor-fetch-profiling + - talos-profiling + +# QR builds just run a subset right now. +linux64-qr/opt: + build-platform: linux64/opt + test-sets: + - talos + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - talos + - talos-profiling + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-ccov/opt: + build-platform: linux64-ccov/opt + test-sets: + - talos + +## +# Windows platforms (matching /windows.*/) + +# win32 +windows7-32/debug: + build-platform: win32/debug + test-sets: + - windows-reftest-gpu + - windows-tests + - web-platform-tests + - mochitest-webgpu + +windows7-32/opt: + build-platform: win32/opt + test-sets: + - awsy + - desktop-screenshot-capture + - marionette-framescript + - windows-reftest-gpu + - windows-talos + - windows-tests + - web-platform-tests + - raptor-firefox + - browsertime + - mochitest-webgpu + - web-platform-tests-wdspec-headless + +windows7-32-shippable/opt: + build-platform: win32-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - marionette-framescript + - windows-reftest-gpu + - windows-talos + - windows-tests + - web-platform-tests + - web-platform-tests-wdspec-headless + - raptor-chrome + - raptor-chromium + - raptor-firefox + - browsertime + - mochitest-webgpu + +# win7 qr builds for try purposes +windows7-32-qr/debug: + build-platform: win32/debug + test-sets: + - mochitest-webgpu + - windows-qr-tests + - web-platform-tests + +windows7-32-qr/opt: + build-platform: win32/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - web-platform-tests + +# win64 +windows10-64-ccov-qr/opt: + build-platform: win64-ccov/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-reftest-gpu + - windows-talos + - windows-tests + - windows-ccov-tests + - windows10-tests + - web-platform-tests + - ccov-code-coverage-tests + - mochitest-webgpu + +windows10-64/debug: + build-platform: win64/debug + test-sets: + - mochitest-webgpu + - windows-tests + - windows10-tests + - web-platform-tests + +windows10-64/opt: + build-platform: win64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-talos + - marionette-framescript + - windows-tests + - windows10-tests + - web-platform-tests + - web-platform-tests-backlog + - raptor-firefox + - browsertime + - mochitest-webgpu + - web-platform-tests-wdspec-headless + +windows10-aarch64/opt: + build-platform: win64-aarch64-shippable/opt + test-sets: + - windows-aarch64-tests + +windows10-64-ref-hw-2017/opt: + build-platform: win64-shippable/opt + test-sets: + - raptor-firefox + - browsertime + - talos-ref-hw + +windows10-64-shippable/opt: + build-platform: win64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-talos + - marionette-framescript + - windows-tests + - windows10-tests + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + - raptor-chrome + - raptor-chromium + - raptor-firefox + - browsertime + - raptor-profiling + - talos-profiling + - mochitest-webgpu + +windows10-64-devedition/opt: + build-platform: win64-devedition/opt + test-sets: + - desktop-screenshot-capture + - windows-tests + - windows10-tests + - web-platform-tests + - mochitest-webgpu + +windows10-64-asan/opt: + build-platform: win64-asan/opt + test-sets: + - windows-tests + - windows10-tests + - mochitest-webgpu + +# QR builds just run a subset right now. +windows10-64-qr/debug: + build-platform: win64/debug + test-sets: + - mochitest-webgpu + - windows-qr-tests + - web-platform-tests + +windows10-64-qr/opt: + build-platform: win64/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - raptor-firefox + - browsertime + - web-platform-tests + +windows10-64-shippable-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - talos-profiling + - raptor-firefox + - browsertime + - web-platform-tests + - mochitest-webgpu + +windows10-64-mingwclang/debug: + build-platform: win64-mingwclang/debug + test-sets: + - mingwclang-tests-debug + +windows10-64-mingwclang/opt: + build-platform: win64-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +windows7-32-mingwclang/debug: + build-platform: win32-mingwclang/debug + test-sets: + - mingwclang-win7-tests-debug + +windows7-32-mingwclang/opt: + build-platform: win32-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +## +# MacOS X platforms (matching /macosx.*/) + +macosx1014-64/debug: + build-platform: macosx64/debug + test-sets: + - macosx1014-64-tests + +macosx1014-64-devedition/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1014-64-tests + +macosx1014-64-qr/opt: + build-platform: macosx64/opt + test-sets: + - macosx1014-64-qr-tests + - desktop-screenshot-capture + - marionette-framescript + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +macosx1014-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - macosx1014-64-qr-tests + - macosx64-talos + - awsy + - browsertime + - desktop-screenshot-capture + - marionette-framescript + - raptor-chrome + - raptor-chromium + - raptor-firefox + - raptor-firefox-power + - raptor-profiling + - talos-profiling + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +macosx1014-64-devedition-qr/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1014-64-qr-tests + - web-platform-tests + +macosx1014-64-qr/debug: + build-platform: macosx64/debug + test-sets: + - macosx1014-64-qr-tests + - web-platform-tests + +macosx1014-64-ccov-qr/opt: + build-platform: macosx64-ccov/opt + test-sets: + - macosx1014-64-qr-tests + +## +# Android platforms (matching /android-em.*/) +# +# android-em test platforms execute on android emulators. + +android-em-7.0-x86_64/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-tests + +android-em-7.0-x86_64-qr/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-qr-tests + +android-em-7.0-x86_64-shippable-qr/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-qr-tests + +android-em-7.0-x86_64-qr/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-qr-tests + +# android-hw test platforms execute on real devices attached to Autophone hosts. + +# android-hw-g5-7-0 Motorola Moto G5 Android 7.0 + +android-hw-g5-7-0-arm7-api-16/opt: + build-platform: android-api-16/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-g5-7-0-arm7-api-16-shippable/opt: + build-platform: android-api-16-shippable/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +# android-hw-p2-8-0 Google Pixel 2 Android 8.0 + +android-hw-p2-8-0-arm7-api-16/opt: + build-platform: android-api-16/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-pgo-unittests + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-p2-8-0-arm7-api-16-shippable/opt: + build-platform: android-api-16-shippable/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-pgo-unittests + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-p2-8-0-arm7-api-16/debug: + build-platform: android-api-16/debug + test-sets: + - android-hw-arm7-debug-unittests + +android-hw-p2-8-0-android-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64-qr/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64-browsertime + - android-hw-aarch64-opt-unittests + - android-hw-aarch64-raptor-cpu-memory + - android-hw-aarch64-raptor-cpu-memory-power + +android-hw-p2-8-0-android-aarch64-shippable/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64-browsertime + - android-hw-aarch64-pgo-unittests + - android-hw-aarch64-raptor-cpu-memory + - android-hw-aarch64-raptor-cpu-memory-power + +android-hw-p2-8-0-android-aarch64/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64-debug-unittests diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml new file mode 100644 index 0000000000..0465c99a9f --- /dev/null +++ b/taskcluster/ci/test/test-sets.yml @@ -0,0 +1,560 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# +# Test names given here reference tests.yml. + +### Feature-specific test groups + +web-platform-tests: + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + +web-platform-tests-backlog: + - web-platform-tests-backlog + - web-platform-tests-reftest-backlog + +web-platform-tests-wdspec-headless: + - web-platform-tests-wdspec-headless + +mochitest-webgpu: + - mochitest-webgpu + +talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g3 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions + +talos-profiling: + - talos-bcv-profiling + - talos-chrome-profiling + # - talos-damp-profiling - Disabled b/c of Bug 1597734 + - talos-dromaeojs-profiling + # - talos-flex-profiling - Disabled b/c of Bug 1597736 + - talos-g1-profiling + - talos-g3-profiling + - talos-g4-profiling + - talos-g5-profiling + - talos-motionmark-profiling + - talos-other-profiling + - talos-sessionrestore-many-windows-profiling + - talos-perf-reftest-profiling + - talos-perf-reftest-singletons-profiling + - talos-svgr-profiling + - talos-tp5o-profiling + - talos-tabswitch-profiling + - talos-realworld-webextensions-profiling + - talos-webgl-profiling + +talos-ref-hw: + - talos-g4 + - talos-webgl + +raptor-firefox: + - raptor-tp6 + - raptor-tp6-live + - raptor-speedometer-firefox + - raptor-stylebench-firefox + - raptor-motionmark-htmlsuite-firefox + - raptor-motionmark-animometer-firefox + - raptor-webaudio-firefox + - raptor-sunspider-firefox + - raptor-wasm-godot-firefox + - raptor-youtube-playback-av1-sfr-firefox + - raptor-youtube-playback-h264-sfr-firefox + - raptor-youtube-playback-hfr-firefox + - raptor-youtube-playback-vp9-sfr-firefox + - raptor-youtube-playback-widevine-hfr-firefox + - raptor-youtube-playback-widevine-h264-sfr-firefox + - raptor-youtube-playback-widevine-vp9-sfr-firefox + - raptor-ares6-firefox + - raptor-jetstream2-firefox + +raptor-chrome: + - raptor-speedometer-chrome + - raptor-stylebench-chrome + - raptor-motionmark-htmlsuite-chrome + - raptor-motionmark-animometer-chrome + - raptor-webaudio-chrome + - raptor-sunspider-chrome + - raptor-wasm-godot-chrome + - raptor-ares6-chrome + - raptor-jetstream2-chrome + - raptor-youtube-playback-av1-sfr-chrome + - raptor-youtube-playback-h264-sfr-chrome + - raptor-youtube-playback-hfr-chrome + - raptor-youtube-playback-vp9-sfr-chrome + - raptor-youtube-playback-widevine-hfr-chrome + - raptor-youtube-playback-widevine-h264-sfr-chrome + - raptor-youtube-playback-widevine-vp9-sfr-chrome + +raptor-profiling: + - raptor-speedometer-firefox-profiling + - raptor-stylebench-firefox-profiling + - raptor-motionmark-htmlsuite-firefox-profiling + - raptor-motionmark-animometer-firefox-profiling + - raptor-webaudio-firefox-profiling + - raptor-sunspider-firefox-profiling + - raptor-wasm-godot-firefox-profiling + - raptor-ares6-firefox-profiling + - raptor-jetstream2-firefox-profiling + +raptor-chromium: + - raptor-speedometer-chromium + - raptor-stylebench-chromium + - raptor-motionmark-htmlsuite-chromium + - raptor-motionmark-animometer-chromium + - raptor-webaudio-chromium + - raptor-sunspider-chromium + - raptor-wasm-godot-chromium + - raptor-ares6-chromium + - raptor-jetstream2-chromium + +# Fetch tasks are only supported on Linux for now, +# so these need to be separate sets. +raptor-fetch-firefox: + - raptor-unity-webgl-firefox + - raptor-wasm-misc-firefox + - raptor-wasm-misc-baseline-firefox + - raptor-wasm-misc-optimizing-firefox + - raptor-wasm-godot-baseline-firefox + - raptor-wasm-godot-optimizing-firefox + - raptor-assorted-dom-firefox + +raptor-fetch-profiling: + - raptor-unity-webgl-firefox-profiling + - raptor-wasm-misc-firefox-profiling + - raptor-wasm-misc-baseline-firefox-profiling + - raptor-wasm-misc-optimizing-firefox-profiling + - raptor-wasm-godot-baseline-firefox-profiling + - raptor-wasm-godot-optimizing-firefox-profiling + - raptor-assorted-dom-firefox-profiling + +raptor-fetch-chromium: + - raptor-unity-webgl-chromium + - raptor-wasm-misc-chromium + - raptor-assorted-dom-chromium + - raptor-jetstream2-chromium + +browsertime: + - browsertime-tp6 + - browsertime-speedometer + - browsertime-ares6 + - browsertime-tp6-profiling + +# Raptor desktop power tests +# (requires Intel Power Gadget to be pre-installed) +raptor-firefox-power: + - raptor-youtube-playback-v9-power-firefox + - raptor-youtube-playback-h264-power-firefox + +awsy: + - awsy + - awsy-base + - awsy-dmd + - awsy-base-dmd + - awsy-tp6 + +ccov-code-coverage-tests: + - test-coverage + - test-coverage-wpt + +mochitest-headless: + - mochitest-plain-headless + +mochitest-valgrind: + - mochitest-valgrind + +marionette-framescript: + - marionette-framescript + +marionette-headless: + - marionette-headless + +desktop-screenshot-capture: + - browser-screenshots + + +### Specific platform test sets + +# Linux + +linux32-tests: + - web-platform-tests + +linux1804-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - reftest-no-accel + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +linux1804-qr-tests: + - cppunit + - crashtest + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - mochitest-a11y + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - xpcshell + +linux1804-ccov-tests: + # jittest run on regular linux builds is redundant with SM(p), so we omit it + # from the linux1804-tests and linux1804-qr-tests sets. But, we still want + # to run it on ccov builds, so we have this set that has it and which runs + # on ccov builds. + - jittest + +linux-qr-smoketests: + - crashtest + - reftest + +linux-talos-flex: + - talos-flex + +# Windows + +windows-qr-tests: + - crashtest + - mochitest-chrome-gpu + - mochitest-browser-chrome + - mochitest-plain + - mochitest-plain-gpu + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl2-core + - reftest + +windows-reftest-gpu: + - reftest-gpu + +windows-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + # - jittest # See comment in windows-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - reftest + - reftest-no-accel + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +windows-ccov-tests: + # jittest run on regular windows builds is redundant with SM(p), so we omit + # it from the windows-tests set. But, we still want to run it on ccov builds, + # so we have this set that has it and which runs on ccov builds. + - jittest + +windows10-tests: + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-devtools-chrome + +mingwclang-tests-debug: + - cppunit + - firefox-ui-functional-local + - firefox-ui-functional-remote + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + +# The following is the same as the above block, minus reftests. +# Reftests were disabled due to frequent hangs; see bug 1642719 +# for re-enabling. +mingwclang-win7-tests-debug: + - cppunit + - firefox-ui-functional-local + - firefox-ui-functional-remote + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + # - reftest - disabled, see bug 1642719 + - telemetry-tests-client + +mingwclang-tests-opt: + - cppunit + - mochitest-chrome-gpu + - mochitest-plain-gpu + +windows-aarch64-tests: + - crashtest + - mochitest-media + - reftest + +windows-talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-svgr + - talos-tp5o + - talos-xperf + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + +# macOS + +macosx1014-64-tests: + - crashtest + - reftest + - web-platform-tests-reftest + +macosx1014-64-qr-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + - jittest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-plain + - mochitest-plain-gpu + - mochitest-media + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +macosx64-talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-realworld-webextensions + - talos-webgl + # - talos-tabswitch # Bug 1453007 times out + # - talos-h1 # too long to unpack profile- Bug 1442893 + +# Android + +android-x86_64-tests: + - cppunit + - crashtest + - geckoview-junit + - gtest + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + # - mochitest-webgl1-core # Harness issues? + # - mochitest-webgl1-ext + # - mochitest-webgl2-core # Emulator not creating ES3? + # - mochitest-webgl2-ext + - reftest + - test-verify + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - xpcshell + +android-x86_64-qr-tests: + - crashtest + - geckoview-junit + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - reftest + +android-hw-aarch64-opt-unittests: + - jittest + +android-hw-arm7-pgo-unittests: + - jittest + - jsreftest + +android-hw-aarch64-pgo-unittests: + - jittest + +android-hw-arm7-debug-unittests: + - jittest + - jsreftest + +android-hw-aarch64-debug-unittests: + - jittest + +android-hw-aarch64-qr: + - crashtest-qr + - reftest-qr + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + +android-hw-arm7-raptor-cpu-memory: + - raptor-speedometer-geckoview-cpu-memory + - raptor-scn-cpu-memory-idle-geckoview + - raptor-scn-cpu-memory-idle-refbrow + - raptor-scn-cpu-memory-idle-bg-geckoview + - raptor-scn-cpu-memory-idle-bg-refbrow + +android-hw-arm7-raptor-cpu-memory-power: + - raptor-speedometer-geckoview-cpu-memory-power + - raptor-scn-cpu-memory-power-idle-geckoview + - raptor-scn-cpu-memory-power-idle-refbrow + - raptor-scn-cpu-memory-power-idle-bg-geckoview + - raptor-scn-cpu-memory-power-idle-bg-refbrow + +android-hw-aarch64-raptor-cpu-memory: + - raptor-speedometer-geckoview-cpu-memory + - raptor-scn-cpu-memory-idle-geckoview + - raptor-scn-cpu-memory-idle-refbrow + - raptor-scn-cpu-memory-idle-bg-geckoview + - raptor-scn-cpu-memory-idle-bg-refbrow + +android-hw-aarch64-raptor-cpu-memory-power: + - raptor-speedometer-geckoview-cpu-memory-power + - raptor-scn-cpu-memory-power-idle-geckoview + - raptor-scn-cpu-memory-power-idle-refbrow + - raptor-scn-cpu-memory-power-idle-bg-geckoview + - raptor-scn-cpu-memory-power-idle-bg-refbrow + +android-hw-aarch64-browsertime: + - browsertime-power + - browsertime-tp6m + - browsertime-tp6m-live + - browsertime-tp6m-profiling + - browsertime-speedometer-mobile + - browsertime-youtube-playback-mobile + - browsertime-unity-webgl-mobile + +android-hw-arm7-browsertime: + - browsertime-power + - browsertime-tp6m + - browsertime-tp6m-live + - browsertime-tp6m-profiling + - browsertime-speedometer-mobile + - browsertime-youtube-playback-mobile + - browsertime-unity-webgl-mobile + +linux-tsan-tests: + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-media + - mochitest-plain + - web-platform-tests + - web-platform-tests-reftest + - xpcshell diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml new file mode 100644 index 0000000000..b669421775 --- /dev/null +++ b/taskcluster/ci/test/web-platform.yml @@ -0,0 +1,342 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: web-platform-tests + instance-size: xlarge + max-run-time: + by-test-platform: + android-em-7.0-x86_64/debug: 7200 + default: 5400 + variants: + by-test-platform: + linux.*64/debug: ['fission'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + linux.*64(-shippable)?-qr/opt: ['trunk'] + windows10-64(-shippable)?-qr/opt: ['trunk'] + default: [] + fission-tier: + by-test-platform: + linux.*64-qr/debug: 1 + linux.*64(-shippable)?-qr/opt: 1 + windows10-64(-shippable)?-qr/opt: 1 + default: 2 + mozharness: + script: web_platform_tests.py + config: + by-test-platform: + windows.*: + - web_platform_tests/prod_config_windows_taskcluster.py + macosx.*: + - web_platform_tests/prod_config_mac.py + android-em.*: + - android/androidx86_7_0.py + - web_platform_tests/prod_config_android.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + target: + by-test-platform: + android-em-7.0-x86_64-shippable(-qr)?/opt: geckoview-androidTest.apk + android-em-7.0-x86_64(-qr)?/opt: geckoview-androidTest.apk + android-em-7.0-x86_64(-qr)?/debug: geckoview-androidTest.apk + default: null + python-3: true + +web-platform-tests: + description: "Web platform test run" + suite: web-platform-tests + treeherder-symbol: W(wpt) + virtualization: + by-test-platform: + windows10-64.*/.*: hardware + default: virtual + chunks: + by-test-platform: + android.*: 16 + linux.*64-tsan/opt: 26 + linux.*64-asan/opt: 22 + linux.*64-ccov.*/opt: 20 + linux.*/debug: 16 + macosx.*/debug: 18 + windows7.*/opt: 12 + windows.*/debug: 14 + windows10-64-ccov.*/opt: 14 + default: 10 + max-run-time: + by-test-platform: + .*-ccov.*/.*: 10800 + linux.*64-tsan/opt: 14400 + linux.*64-asan/opt: 14400 + linux.*64(-qr)?/debug: 10800 + macosx.*/debug: 10800 + android-em.*/.*: 10800 + default: 7200 + e10s: true + run-on-projects: + by-test-platform: + linux1804-32-shippable/opt: ['release'] # skip integration, Bug 1599197 + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-backlog: + description: "Web platform test (backlog) run" + treeherder-symbol: W-b(wpt) + chunks: + by-test-platform: + android.*: 3 + default: 1 + max-run-time: + by-test-platform: + android.*/opt: 7200 + default: 3600 + variants: [] + e10s: true + run-on-projects: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-reftest: + description: "Web platform reftest run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W(Wr) + virtualization: + by-test-platform: + windows10-64.*-qr/.*: virtual-with-gpu + default: virtual + chunks: + by-test-platform: + .*-ccov.*/.*: 8 + linux.*64-tsan/opt: 8 + linux1804-64(-qr|-asan)/.*: 6 + linux1804-64(-shippable|-devedition)?/opt: 3 + macosx10.*-64/debug: 6 + macosx10.*-64/opt: 3 + windows.*-(32|64)(-qr)?/debug: 5 + android.*: 6 + default: 4 + e10s: true + run-on-projects: built-projects + tier: + by-test-platform: + linux1804-64-asan/opt: 2 + default: default + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-reftest-backlog: + description: "Web platform reftest (backlog) run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W-b(Wr) + virtualization: virtual + chunks: 1 + variants: [] + e10s: true + run-on-projects: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-wdspec: + description: "Web platform webdriver-spec run" + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(Wd) + chunks: + by-test-platform: + .*-ccov.*/.*: 4 + default: 3 + mozharness: + extra-options: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + run-on-projects: + by-test-platform: + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + tier: + by-test-platform: + android.*: 3 + linux.*64-asan/opt: 2 + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + fetches: + toolchain: + by-test-platform: + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + + +web-platform-tests-wdspec-headless: + description: "Web platform webdriver-spec headless run" + suite: + name: web-platform-tests-wdspec + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(WdH) + chunks: 2 + mozharness: + extra-options: + by-test-platform: + windows.*: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + - --headless + - --headless-width=1600 + - --headless-height=1200 + default: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + - --headless + fetches: + toolchain: + by-test-platform: + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + +web-platform-tests-crashtest: + description: "Web platform crashtests run" + schedules-component: web-platform-tests-crashtest + treeherder-symbol: W(Wc) + chunks: 1 + mozharness: + extra-options: + - --test-type=crashtest + run-on-projects: + by-test-platform: + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + tier: + by-test-platform: + linux1804-64-asan/opt: 2 + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + +web-platform-tests-print-reftest: + description: "Web platform print-reftest run" + schedules-component: web-platform-tests-print-reftest + treeherder-symbol: W(Wp) + chunks: 1 + test-manifest-loader: null # don't load tests in the taskgraph + tier: + by-test-platform: + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + run-on-projects: built-projects + mozharness: + extra-options: + - --test-type=print-reftest + +test-verify-wpt: + description: "Extra verification of web-platform tests modified on this push" + suite: + category: test-verify + name: test-verify-wpt + variants: [] + treeherder-symbol: TVw + max-run-time: 10800 + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + test-manifest-loader: null # don't load tests in the taskgraph + optimization: + test-verify: ["test-verify-wpt"] + tier: 2 + mozharness: + extra-options: + - --verify + +test-coverage-wpt: + description: "Per web-platform test coverage" + suite: + category: test-coverage + name: test-coverage-wpt + schedules-component: test-coverage-wpt + treeherder-symbol: TCw + max-run-time: 10800 + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --per-test-coverage diff --git a/taskcluster/ci/test/xpcshell.yml b/taskcluster/ci/test/xpcshell.yml new file mode 100644 index 0000000000..7bd3d366e0 --- /dev/null +++ b/taskcluster/ci/test/xpcshell.yml @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: xpcshell + variants: + by-test-platform: + linux1804-64/debug: ['fission', 'socketprocess_networking'] + default: ['fission'] + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + fission-tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + requires-signed-builds: + by-test-platform: + windows10-64-asan/opt: false # No XPCShell on ASAN yet + windows7-32-mingwclang/opt: false + windows7-32-mingwclang/debug: false + windows10-64-mingwclang/opt: false + windows10-64-mingwclang/debug: false + windows.*: true + default: false + +xpcshell: + description: "xpcshell test run" + treeherder-symbol: X(X) + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + windows10-64-asan/opt: [] # No XPCShell on ASAN yet + default: built-projects + chunks: + by-test-platform: + .*-ccov.*/.*: 6 + windows10-64-ccov-qr/opt: 6 + android-em-7.*: 4 + macosx1014-64(-shippable|-devedition|-asan)?(-qr)?/.*: 2 + linux1804-64(-qr)?/debug: 6 + .*-tsan/opt: 8 + windows(7-32|10-64)(-shippable|-devedition|-asan|.*-qr)?/.*: 2 + default: 5 + max-run-time: + by-test-platform: + macosx1014-64(-shippable|-devedition|-asan)?(-qr)?/.*: 5400 + windows(7-32|10-64)(-shippable)?/opt: 5400 + .*-ccov.*/.*: 7200 + .*-tsan/opt: 7200 + android-em-7.*: 5400 + default: 2700 + allow-software-gl-layers: false + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: default + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-emulator-linux + - linux64-node + - linux64-minidump-stackwalk |