From 2aa4a82499d4becd2284cdb482213d541b8804dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 16:29:10 +0200 Subject: Adding upstream version 86.0.1. Signed-off-by: Daniel Baumann --- taskcluster/ci/test/awsy.yml | 86 +++ taskcluster/ci/test/browsertime-desktop.yml | 151 ++++++ taskcluster/ci/test/browsertime-mobile.yml | 426 +++++++++++++++ taskcluster/ci/test/compiled.yml | 105 ++++ taskcluster/ci/test/firefox-ui.yml | 38 ++ taskcluster/ci/test/kind.yml | 72 +++ taskcluster/ci/test/marionette.yml | 69 +++ taskcluster/ci/test/misc.yml | 254 +++++++++ taskcluster/ci/test/mochitest.yml | 607 +++++++++++++++++++++ taskcluster/ci/test/raptor-chrome.yml | 247 +++++++++ taskcluster/ci/test/raptor-chromium.yml | 219 ++++++++ taskcluster/ci/test/raptor-gve.yml | 120 +++++ taskcluster/ci/test/raptor-refbrow.yml | 86 +++ taskcluster/ci/test/raptor.yml | 784 ++++++++++++++++++++++++++++ taskcluster/ci/test/reftest.yml | 235 +++++++++ taskcluster/ci/test/talos.yml | 646 +++++++++++++++++++++++ taskcluster/ci/test/test-platforms.yml | 537 +++++++++++++++++++ taskcluster/ci/test/test-sets.yml | 560 ++++++++++++++++++++ taskcluster/ci/test/web-platform.yml | 342 ++++++++++++ taskcluster/ci/test/xpcshell.yml | 99 ++++ 20 files changed, 5683 insertions(+) create mode 100644 taskcluster/ci/test/awsy.yml create mode 100644 taskcluster/ci/test/browsertime-desktop.yml create mode 100644 taskcluster/ci/test/browsertime-mobile.yml create mode 100644 taskcluster/ci/test/compiled.yml create mode 100644 taskcluster/ci/test/firefox-ui.yml create mode 100644 taskcluster/ci/test/kind.yml create mode 100644 taskcluster/ci/test/marionette.yml create mode 100644 taskcluster/ci/test/misc.yml create mode 100644 taskcluster/ci/test/mochitest.yml create mode 100644 taskcluster/ci/test/raptor-chrome.yml create mode 100644 taskcluster/ci/test/raptor-chromium.yml create mode 100644 taskcluster/ci/test/raptor-gve.yml create mode 100644 taskcluster/ci/test/raptor-refbrow.yml create mode 100644 taskcluster/ci/test/raptor.yml create mode 100644 taskcluster/ci/test/reftest.yml create mode 100644 taskcluster/ci/test/talos.yml create mode 100644 taskcluster/ci/test/test-platforms.yml create mode 100644 taskcluster/ci/test/test-sets.yml create mode 100644 taskcluster/ci/test/web-platform.yml create mode 100644 taskcluster/ci/test/xpcshell.yml (limited to 'taskcluster/ci/test') diff --git a/taskcluster/ci/test/awsy.yml b/taskcluster/ci/test/awsy.yml new file mode 100644 index 0000000000..ea50703abb --- /dev/null +++ b/taskcluster/ci/test/awsy.yml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: awsy + max-run-time: 7200 + instance-size: xlarge + virtualization: + by-test-platform: + windows10-64(?:-shippable)?-qr/.*: virtual-with-gpu + windows10-64/.*: virtual-with-gpu + default: virtual + allow-software-gl-layers: false + mozharness: + script: awsy_script.py + config: + by-test-platform: + windows.*: + - awsy/taskcluster_windows_config.py + macosx.*/opt: + - awsy/macosx_config.py + default: + - awsy/linux_config.py + tier: default + fission-tier: + by-test-platform: + linux1804-64(?!-shippable).*: 3 + windows10-64(-qr)?/opt: 3 + default: 2 + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition + windows7-32.*/opt: [] + (linux|windows10|mac)(?!.*shippable).*/opt: [] + default: ['integration', 'mozilla-central', 'mozilla-beta', 'mozilla-release'] + test-manifest-loader: null # don't load tests in the taskgraph + fission-run-on-projects: + by-test-platform: + linux1804-64-shippable-qr/opt: ['mozilla-central'] + windows10-64-shippable-qr/opt: ['mozilla-central'] + default: [] + + +awsy: + description: "Are we slim yet" + treeherder-symbol: SY(sy) + +awsy-tp6: + description: "Are we slim yet - tp6 pageset" + variants: ["fission"] + treeherder-symbol: SY(sy-tp6) + run-on-projects: + by-test-platform: + .*-ccov.*/.*: [] # don't run on coverage + .*-devedition/.*: [] # don't run on devedition + windows7-32.*/opt: [] + (linux|windows10|mac)(?!.*shippable)(?!.*ccov).*/opt: [] + default: ['integration', 'mozilla-central', 'mozilla-beta', 'mozilla-release'] + mozharness: + extra-options: + - --tp6 + +awsy-dmd: + description: "Are we slim yet - dmd enabled" + treeherder-symbol: SY(sy-d) + run-on-projects: [] + mozharness: + extra-options: + - --dmd + +awsy-base: + description: "Are we slim yet - about:blank base case" + treeherder-symbol: SY(ab) + mozharness: + extra-options: + - --base + +awsy-base-dmd: + description: "Are we slim yet - about:blank base case - dmd enabled" + treeherder-symbol: SY(ab-d) + run-on-projects: [] + mozharness: + extra-options: + - --base + - --dmd diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..1111f85168 --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + .*-ref-hw-2017/.*: 3600 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: ['mozilla-central'] + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + fission-tier: 3 + virtualization: + by-test-platform: + windows10-64-ccov.*/.*: virtual + default: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + windows10-64-ccov.*/.*: + - raptor/windows_vm_config.py + linux64-ccov.*/.*: + - raptor/linux64_config_taskcluster.py + default: + - raptor/linux_config.py + extra-options: + - --browsertime + - --no-conditioned-profile + fission-run-on-projects: [] + +browsertime-tp6: + description: "Raptor (browsertime) tp6 page-load tests" + raptor-test: tp6 + raptor-subtests: + - amazon + - apple + - [bing-search, bing] + - ebay + - [facebook, fb] + - [facebook-redesign, fbr] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-sheets, gsheets] + - [google-slides, gslides] + - imdb + - imgur + - instagram + - linkedin + - microsoft + - netflix + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - yahoo-news + - yandex + - youtube + tier: + by-app: + firefox: + by-subtest: + amazon: 1 + google-mail: 1 + google-slides: 1 + imgur: 1 + tumblr: 1 + twitch: 1 + twitter: 1 + default: 2 + default: 3 + apps: ['firefox', 'chrome', 'chromium'] + variants: + by-app: + firefox: ['fission'] + default: [] + pageload: cold + fission-tier: 2 + fission-run-on-projects: ["mozilla-central"] + run-on-projects: + by-app: + firefox: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + linux.*shippable.*: ["trunk", "mozilla-beta"] + macos.*shippable.*: ["trunk", "mozilla-beta"] + default: [] + default: [] + limit-platforms: + by-app: + chrome: + - linux.*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + chromium: + # Run on non-QR shippable, but test-platforms.yml doesn't have + # any more non-QR macosx shippable things. So for macosx we want + # to run it on shippable-qr, otherwise there's nothing running + # this on macosx at all. + - .*shippable[^-qr].* + - macosx1014-64-shippable-qr/opt + default: [] + treeherder-symbol: Btime(tp6) + max-run-time: 4000 + run-visual-metrics: true + mozharness: + extra-options: + - --chimera + +browsertime-speedometer: + description: "Raptor (browsertime) Speedometer on Firefox" + raptor-test: speedometer + variants: ["fission"] + apps: ["firefox"] + treeherder-symbol: Btime(sp) + +browsertime-ares6: + description: "Raptor (browsertime) ares6 on Firefox" + raptor-test: raptor-ares6 + variants: ["fission"] + apps: ["firefox"] + treeherder-symbol: Btime(ares6) + +browsertime-tp6-profiling: + description: "Raptor (browsertime) tp6 page-load tests with Gecko Profiling" + raptor-test: tp6 + raptor-subtests: ['amazon'] + apps: ['firefox'] + pageload: 'both' + treeherder-symbol: Btime-Prof(tp6) + max-run-time: 4000 + mozharness: + extra-options: + - --gecko-profile diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml new file mode 100644 index 0000000000..3a1440d810 --- /dev/null +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -0,0 +1,426 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: raptor + run-on-projects: + by-app: + refbrow: [] + fennec: [] + default: + by-test-platform: + android-hw-p2-.*aarch64.*/pgo: ["mozilla-central"] + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central"] + default: [] + webrender-run-on-projects: [] + tier: 3 + test-manifest-loader: null # don't load tests in the taskgraph + target: + by-app: + fenix: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.fenix.nightly.latest.arm64-v8a + name: build/arm64-v8a/target.apk + default: + index: mobile.v2.fenix.nightly.latest.armeabi-v7a + name: build/armeabi-v7a/target.apk + fennec: + by-test-platform: + android-hw-p2-8-0-android-aarch64.*: + index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-aarch64-release-opt + name: build/target.apk + default: + index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt + name: build/target.apk + refbrow: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.reference-browser.raptor.latest.arm64-v8a + name: target.arm64-v8a.apk + default: + index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a + name: target.armeabi-v7a.apk + default: geckoview_example.apk + e10s: true + virtualization: hardware + activity: + by-app: + fenix: org.mozilla.fenix.IntentReceiverActivity + geckoview: org.mozilla.geckoview_example.GeckoViewActivity + fennec: org.mozilla.gecko.BrowserApp + refbrow: org.mozilla.reference.browser.BrowserTestActivity + default: None + binary-path: + by-app: + fenix: org.mozilla.fenix + geckoview: org.mozilla.geckoview_example + fennec: org.mozilla.firefox + refbrow: org.mozilla.reference.browser.raptor + default: None + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --browsertime + # Bug 1635749, disable window recorder temporarily + - --browsertime-no-ffwindowrecorder + - --no-conditioned-profile + optimization: + skip-unless-backstop: null + variants: + by-app: + chrome-m: [] + fennec: [] + default: ["webrender"] + +browsertime-tp6m: + description: "Raptor (browsertime) tp6 page-load tests on android" + raptor-test: tp6m + raptor-subtests: + by-app: + chrome-m: + - amazon + - youtube + default: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bbc + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - cnn + - [cnn-ampstories, cnn-amp] + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - facebook + - [facebook-cristiano, fb-cris] + - google + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - jianshu + - [microsoft-support, micros-sup] + - reddit + - [stackoverflow, stacko] + - web-de + - wikipedia + - youtube + - [youtube-watch, youtube-w] + apps: ["geckoview", "fenix", "fennec", "chrome-m", "refbrow"] + tier: + by-raptor-test: + amazon: + by-app: + fenix: 1 + geckoview: 1 + refbrow: 2 + fennec: 2 + default: 3 + default: + by-app: + geckoview: 1 + refbrow: 2 + fennec: 2 + default: 3 + webrender-run-on-projects: + by-raptor-test: + amazon: + by-app: + fenix: [] + chrome-m: [] + fennec: [] + geckoview: + by-pageload: + cold: + by-test-platform: + &android-cold-settings + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + warm: + by-test-platform: + &android-warm-settings + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['mozilla-central'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['mozilla-central'] + default: [] + default: + by-test-platform: + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central"] + default: [] + youtube: + by-app: + fenix: [] + chrome-m: [] + refbrow: [] + fennec: [] + geckoview: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + default: + by-test-platform: + android-hw-p2-.*aarch64-shippable/opt: ["mozilla-central", "try"] + default: [] + default: + by-pageload: + cold: + by-app: + geckoview: + by-test-platform: *android-cold-settings + default: [] + warm: + by-app: + geckoview: + by-test-platform: *android-warm-settings + default: [] + run-on-projects: + by-app: + geckoview: + by-subtest: + allrecipes: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + amazon-search: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + espn: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + facebook: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + google: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + microsoft-support: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + youtube-watch: + by-pageload: + cold: + by-test-platform: *android-cold-settings + warm: + by-test-platform: *android-warm-settings + default: [] + default: [] + pageload: cold + run-visual-metrics: true + treeherder-symbol: Btime(tp6m) + mozharness: + extra-options: + - --chimera + +browsertime-youtube-playback-mobile: + description: "Browsertime YouTube Playback on Android" + treeherder-symbol: Btime(ytp) + raptor-subtests: + by-test-platform: + android-hw-g5.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + android-hw-p2.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + apps: ["fenix", "geckoview", "refbrow"] + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ["mozilla-central"] + android-hw-p2-8-0-android-aarch64-shippable/opt: ["mozilla-central"] + default: [] + default: [] + test-url-param: + by-subtest: + youtube-playback-h264-sfr: # remove high resolution tests + "exclude=1,2, + 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38" + youtube-playback-hfr: + by-test-platform: + android-hw-g5.*: # remove VP9(1-34) and AV1(51-74) tests + "exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74" + android-hw-p2.*: # remove AV1(51-74) tests + "exclude=1,2, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74" + default: "" + youtube-playback-widevine-hfr: + by-test-platform: + android-hw-g5.*: # remove VP9(1-34) tests + "exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34" + default: "" + default: "" + max-run-time: 3600 + +browsertime-tp6m-live: + description: "Raptor (browsertime) tp6 live site page-load tests on android" + raptor-test: tp6m + raptor-subtests: + by-app: + geckoview: + - [cnn-ampstories, cnn-amp] + default: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bbc + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - booking-sf + - cnn + - [cnn-ampstories, cnn-amp] + - discord + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - expedia + - facebook + - [facebook-cristiano, fb-cris] + - [fashionbeans, fashbea] + - google + - [google-accounts, gacc] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - [imdb-firefox, imdb-f] + - jianshu + - [medium-article, medium] + - [microsoft-support, micros-sup] + - nytimes + - [people-article, people] + - reddit + - [reddit-thread, reddit-thr] + - rumble-fox + - [stackoverflow, stacko] + - [stackoverflow-question, stack-q] + - [urbandictionary-define, urbict-def] + - web-de + - [wikia-marvel, wikia-m] + - wikipedia + - youtube + - [youtube-watch, youtube-w] + apps: ["fenix", "chrome-m", "geckoview"] + run-on-projects: [] + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ["autoland"] + android-hw-p2-8-0-android-aarch64-shippable/opt: ["autoland"] + default: [] + default: [] + pageload: cold + run-visual-metrics: true + treeherder-symbol: Btime-live(tp6m) + mozharness: + extra-options: + - --live-sites + - --chimera + +browsertime-tp6m-profiling: + description: "Raptor (browsertime) tp6 page-load tests on android with Gecko profiling" + raptor-test: tp6m + raptor-subtests: ["amazon", "youtube"] + apps: ["geckoview"] + pageload: cold + treeherder-symbol: Btime-Prof(tp6m) + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --browsertime + - --gecko-profile + +browsertime-speedometer-mobile: + description: "Browsertime Speedometer on Android" + apps: ["geckoview", "fenix", "fennec", "chrome-m"] + tier: + by-app: + fenix: 1 + default: 3 + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + default: [] + treeherder-symbol: Btime(sp) + mozharness: + extra-options: + - --browsertime + - --test=speedometer + +browsertime-unity-webgl-mobile: + description: "Browsertime Unity WebGL on Android" + apps: ["geckoview", "refbrow", "fenix", "chrome-m"] + treeherder-symbol: Btime(ugl) + webrender-run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-g5-7-0-arm7-api-16-shippable/opt: ['trunk', 'mozilla-beta'] + android-hw-p2-8-0-android-aarch64-shippable/opt: ['trunk', 'mozilla-beta'] + default: [] + default: [] + max-run-time: 900 + mozharness: + extra-options: + - --test=unity-webgl + fetches: + fetch: + - unity-webgl + +browsertime-power: + description: "Browsertime Power Usage Tests on Android" + apps: ["geckoview", "fenix", "refbrow"] + tier: 2 + raptor-subtests: + - idle + - idle-bg + - [speedometer, sp] + variants: ["webrender"] + run-on-projects: [] + treeherder-symbol: Btime-P(power) + mozharness: + extra-options: + - --power-test diff --git a/taskcluster/ci/test/compiled.yml b/taskcluster/ci/test/compiled.yml new file mode 100644 index 0000000000..aa4e803747 --- /dev/null +++ b/taskcluster/ci/test/compiled.yml @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + e10s: false + test-manifest-loader: null # don't load tests in the taskgraph + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + python-3: true + +cppunit: + description: "CPP Unit Tests" + suite: cppunittest + treeherder-symbol: cppunit + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + tier: default + run-on-projects: built-projects + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: GTest + instance-size: xlarge + run-on-projects: + by-test-platform: + windows.*-shippable/.*: [] # permafails on shippable + windows.*-nightly/.*: [] # permafails on nightly too + .*-devedition/.*: [] # don't run on devedition + default: built-projects + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + tier: + by-test-platform: + windows7-32-shippable.*: 3 + windows10-64-shippable.*: 3 + windows10-64-asan.*: 3 + linux.*64-ccov.*/opt: 3 + default: default + +jittest: + description: "JIT Test run" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-test-platform: + android-hw-.*-aarch64/debug: ['mozilla-central', 'release'] + android-hw-.*-api-16(?:-shippable)?/.*: ['mozilla-central', 'release'] + default: built-projects + chunks: + by-test-platform: + windows.*: 1 + windows10-64-ccov-qr/opt: 8 + macosx.*/opt: 1 + macosx.*/debug: 3 + android.*: 10 + default: 6 + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 7200 + macosx.*-ccov.*/.*: 7200 + android-hw.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + windows.*: false + macosx.*: false + default: true + target: + by-test-platform: + android-.*: geckoview-androidTest.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default diff --git a/taskcluster/ci/test/firefox-ui.yml b/taskcluster/ci/test/firefox-ui.yml new file mode 100644 index 0000000000..4d911fa0a2 --- /dev/null +++ b/taskcluster/ci/test/firefox-ui.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: firefox-ui + max-run-time: 5400 + test-manifest-loader: null # don't load tests in the taskgraph + mozharness: + script: firefox_ui_tests/functional.py + config: + by-test-platform: + windows.*: + - firefox_ui_tests/taskcluster_windows.py + macosx.*: + - firefox_ui_tests/taskcluster.py + - firefox_ui_tests/taskcluster_mac.py + linux.*: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + +firefox-ui-functional-local: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn-l(en-US) + mozharness: + extra-options: + - "--tag" + - "local" + +firefox-ui-functional-remote: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn-r(en-US) + mozharness: + extra-options: + - "--tag" + - "remote" + tier: 2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml new file mode 100644 index 0000000000..507baf080f --- /dev/null +++ b/taskcluster/ci/test/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.test:loader + +kind-dependencies: + - build + - build-signing + - fetch + - toolchain + +transforms: + - taskgraph.transforms.tests:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +# Each stanza in a file pointed to by 'jobs-from' describes a particular test +# suite or sub-suite. These are processed through the transformations described +# above to produce a bunch of tasks. See the schema in +# `taskcluster/taskgraph/transforms/tests.py` for a description of the fields +# used in these files. + +# Adding a new test type or running tests on a new platform? Be sure to review +# https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy + +jobs-from: + - awsy.yml + - browsertime-desktop.yml + - browsertime-mobile.yml + - compiled.yml + - firefox-ui.yml + - marionette.yml + - misc.yml + - mochitest.yml + - raptor.yml + - raptor-chrome.yml + - raptor-chromium.yml + - raptor-gve.yml + - raptor-refbrow.yml + - reftest.yml + - talos.yml + - web-platform.yml + - xpcshell.yml + +job-defaults: + attributes: + retrigger: true + require-signed-extensions: + by-release-type: + release|esr.*: true + beta: + by-test-platform: + .*-devedition/.*: false + default: true + default: false + fetches: + toolchain: + by-test-platform: + win.*: + - win32-minidump-stackwalk + - win32-fix-stacks + macosx.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + android-em-7.*: + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks diff --git a/taskcluster/ci/test/marionette.yml b/taskcluster/ci/test/marionette.yml new file mode 100644 index 0000000000..65fcbcc0df --- /dev/null +++ b/taskcluster/ci/test/marionette.yml @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: marionette + mozharness: + script: marionette.py + config: + by-test-platform: + macosx.*: + - marionette/prod_config.py + - marionette/mac_taskcluster_config.py + windows.*: + - marionette/windows_taskcluster_config.py + default: + - marionette/prod_config.py + - remove_executables.py + extra-options: + - --setpref=toolkit.asyncshutdown.log=true + + run-on-projects: built-projects + test-manifest-loader: null # don't load tests in the taskgraph + tier: default + fission-run-on-projects: + by-test-platform: + linux.*64/.*: ['autoland'] + linux.*64-shippable/.*: ['mozilla-central'] + windows10-64/.*: ['autoland'] + windows10-64-shippable/.*: ['mozilla-central'] + default: [] + fission-tier: 2 + variants: ["fission"] + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + +marionette: + description: "Marionette unittest run" + treeherder-symbol: Mn + max-run-time: 5400 + instance-size: default + chunks: 1 + +marionette-framescript: + description: "Marionette unittest run with its JSWindowActors disabled" + treeherder-symbol: MnFr + max-run-time: 5400 + instance-size: default + tier: 2 + run-on-projects: + # Platforms for which we release geckodriver + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: built-projects + default: [] + mozharness: + extra-options: + - --disable-actors + +marionette-headless: + description: "Marionette headless unittest run" + treeherder-symbol: MnH + max-run-time: 5400 + instance-size: default + tier: 2 + mozharness: + extra-options: + - --headless diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml new file mode 100644 index 0000000000..8c11fbe247 --- /dev/null +++ b/taskcluster/ci/test/misc.yml @@ -0,0 +1,254 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + +geckoview-junit: + description: "Geckoview junit run" + suite: geckoview-junit + variants: ["geckoview-e10s-single", "geckoview-fission"] + treeherder-symbol: gv-junit + loopback-video: true + e10s: true + target: geckoview-androidTest.apk + max-run-time: 3600 + tier: default + fission-run-on-projects: ['trunk'] + mozharness: + script: android_emulator_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + extra-options: + - --test-suite=geckoview-junit + +telemetry-tests-client: + description: "Telemetry tests client run" + suite: telemetry-tests-client + treeherder-symbol: tt(c) + max-run-time: 1200 + tier: default + mozharness: + script: telemetry/telemetry_client.py + config: + by-test-platform: + linux.*: + - remove_executables.py + windows.*: [] + macosx.*: [] + +test-verify: + description: "Extra verification of tests modified on this push" + suite: test-verify + treeherder-symbol: TV + loopback-video: true + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + optimization: + test-verify: ["test-verify"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-verify-gpu: + description: "Extra verification of tests modified on this push on gpu instances" + suite: + category: test-verify + name: test-verify-gpu + treeherder-symbol: TVg + loopback-video: true + virtualization: virtual-with-gpu + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + optimization: + test-verify: ["test-verify-gpu"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + - --gpu-required + +test-coverage: + description: "Per-test coverage" + suite: test-coverage + treeherder-symbol: TC + loopback-video: true + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-sdk-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-coverage-gpu: + description: "Per-test coverage for tests that require gpu instances" + suite: test-coverage + treeherder-symbol: TCg + loopback-video: true + virtualization: virtual-with-gpu + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # only run on mozilla-central and try. + .*-ccov.*/.*: ['mozilla-central'] + default: [] + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + - --gpu-required diff --git a/taskcluster/ci/test/mochitest.yml b/taskcluster/ci/test/mochitest.yml new file mode 100644 index 0000000000..7906d8b626 --- /dev/null +++ b/taskcluster/ci/test/mochitest.yml @@ -0,0 +1,607 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: mochitest + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + android-hw.*: geckoview-androidTest.apk + default: null + variants: + by-test-platform: + linux.*64/debug: ['fission'] + default: ['fission'] + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + default: built-projects + fission-run-on-projects: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + linux.*64-shippable-qr/opt: ['mozilla-central'] + linux.*64-shippable/.*: ['mozilla-central'] + linux.*64/debug: ['mozilla-central'] + windows10-64-shippable(-qr)?/opt: ['mozilla-central'] + default: [] + fission-tier: + by-test-platform: + linux.*64.*-qr/debug: 1 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +mochitest-plain: + description: "Mochitest plain run" + treeherder-symbol: M() + schedules-component: mochitest-plain + loopback-video: true + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual + variants: + by-test-platform: + linux1804-64/opt: ['fission-xorigin'] + linux1804-64/debug: ['fission-xorigin'] + default: ['fission'] + chunks: + by-test-platform: + android-em-7.*: 4 + linux.*/debug: 16 + linux.*64-asan/opt: 10 + linux.*64-tsan/opt: 20 + linux.*64-.*cov/opt: 10 + windows10-64-ccov.*/.*: 10 + macosx.*64-ccov.*/.*: 10 + default: 5 + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + e10s: true + max-run-time: 5400 + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android-em.*: + - --test-suite=mochitest-plain + default: [] + chunked: + by-test-platform: + android-em.*: false + default: true + +mochitest-a11y: + description: "Mochitest a11y run" + treeherder-symbol: M(a11y) + schedules-component: mochitest-a11y + test-manifest-loader: default # ensure we don't run with manifest-scheduling + loopback-video: true + tier: default + e10s: false + run-on-projects: built-projects + mozharness: + mochitest-flavor: a11y + +mochitest-browser-chrome: + description: "Mochitest browser-chrome run" + suite: + name: mochitest-browser-chrome + treeherder-symbol: M(bc) + schedules-component: mochitest-browser-chrome + loopback-video: true + variants: + by-test-platform: + linux.*64(-shippable)?/opt: ['fission', 'a11y-checks'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64(-qr)?/debug: ['trunk'] + linux.*64-shippable(-qr)?/opt: ['mozilla-central'] + windows10-64-shippable(-qr)?/opt: ['mozilla-central'] + default: [] + tier: default + fission-tier: + by-test-platform: + linux.*64/debug: 1 + default: 2 + chunks: + by-test-platform: + linux.*/debug: 16 + linux.*64-asan/opt: 16 + linux.*64-tsan/opt: 32 + macosx.*64/debug: 16 + windows10-64-ccov.*/.*: 14 + windows10.*-asan/opt: 9 + default: 7 + max-run-time: + by-test-platform: + linux.*64-ccov.*/.*: 9000 + windows7-32/debug: 5400 + windows10-64/debug: 5400 + macosx.*64(-qr)?/debug: 5400 + windows10-64-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 10800 + linux.*/debug: 5400 + windows10-64-qr/debug: 5400 + linux.*64-tsan/opt: 9000 + default: 3600 + mozharness: + mochitest-flavor: browser + chunked: true + # Bug 1281241: migrating to m3.large instances + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + allow-software-gl-layers: false + +browser-screenshots: + description: "Browser Screenshots" + suite: + name: mochitest-browser-chrome-screenshots + treeherder-symbol: M(ss) + loopback-video: true + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: + by-test-platform: + windows7-32(?:-shippable)(?:-qr)?/opt: ['mozilla-central'] + windows10-64(?:-shippable)(?:-qr)?/opt: ['mozilla-central'] + (?:windows10-64|windows7-32|linux1804-64|macosx1014-64)(?:-qr)?/opt: ['integration'] + linux1804-64-(?:shippable)(?:-qr)?/opt: ['mozilla-central'] + macosx.*64-shippable/opt: ['mozilla-central'] + default: [] + fission-run-on-projects: [] + fission-tier: 2 + max-run-time: 3600 + mozharness: + mochitest-flavor: browser + allow-software-gl-layers: false + +mochitest-chrome: + description: "Mochitest chrome run" + treeherder-symbol: M(c) + schedules-component: mochitest-chrome + loopback-video: true + tier: default + chunks: + by-test-platform: + .*(-ccov|-qr).*/.*: 3 + windows10-64-ccov-qr/opt: 3 + .*-asan/opt: 3 + .*-tsan/opt: 6 + (linux.*64|windows.*-..|macosx10..)/debug: 3 + default: 2 + max-run-time: 3600 + e10s: false + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-devtools-chrome: + description: "Mochitest devtools-chrome run" + suite: + name: mochitest-devtools-chrome + schedules-component: mochitest-browser-chrome + treeherder-symbol: M(dt) + loopback-video: true + variants: + by-test-platform: + linux.*64(-shippable)?/opt: ['fission', 'a11y-checks'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64(-qr)?/debug: ['trunk'] + linux.*64-shippable(-qr)?/opt: ['trunk'] + windows10-64-shippable(-qr)?/opt: ['trunk'] + default: [] + tier: default + fission-tier: + by-test-platform: + linux.*64(-qr)?/debug: 1 + linux.*64-shippable(-qr)?/opt: 1 + windows10-64-shippable(-qr)?/opt: 1 + default: 2 + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 9000 + linux.*64-ccov.*/.*: 7200 + linux.*64-tsan/opt: 7200 + default: 5400 + chunks: + by-test-platform: + .*-ccov.*/.*: 16 + linux.*64/debug: 12 + macosx.*64/debug: 8 + .*-asan/opt: 8 + .*-tsan/opt: 16 + default: 5 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-[at]san/opt: xlarge # runs out of memory on default/m3.large + default: default + # Bug 1296086: high number of intermittents observed with software GL and large instances + allow-software-gl-layers: false + +mochitest-plain-gpu: + description: "Mochitest plain GPU run" + suite: + name: mochitest-plain-gpu + treeherder-symbol: M(gpu) + schedules-component: mochitest-plain + loopback-video: true + fission-run-on-projects: [] + fission-tier: 2 + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + e10s: true + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-plain-gpu + default: + - --mochitest-suite=mochitest-plain-gpu + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-chrome-gpu: + description: "Mochitest chrome GPU run" + suite: + name: mochitest-chrome-gpu + treeherder-symbol: M(gpu-c) + loopback-video: true + fission-run-on-projects: [] + fission-tier: 2 + tier: default + virtualization: + by-test-platform: + windows10-64.*: hardware + default: virtual-with-gpu + e10s: true + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android.*/.*: [] + default: built-projects + mozharness: + mochitest-flavor: chrome + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-chrome-gpu + default: + - --mochitest-suite=mochitest-chrome-gpu + instance-size: + by-test-platform: + linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large + default: default + +mochitest-media: + description: "Mochitest media run" + treeherder-symbol: M(mda) + schedules-component: mochitest-plain + max-run-time: + by-test-platform: + windows10-64-ccov.*/.*: 7200 + macosx.*64-ccov.*/.*: 7200 + linux.*64-tsan/opt: 7200 + default: 5400 + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + android-hw-.*(?/, where is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + + +## +# Linux platforms (matching /linux.*/) +# Linux1804-64 uses Ubuntu 18.04 and is run on docker + +linux1804-32-shippable/opt: + build-platform: linux-shippable/opt + test-sets: + - linux32-tests + +linux1804-64/opt: + build-platform: linux64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - marionette-framescript + - marionette-headless + - mochitest-headless + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +linux1804-64/debug: + build-platform: linux64/debug + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-shippable/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - marionette-framescript + - marionette-headless + - mochitest-headless + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +linux1804-64-devedition/opt: + build-platform: linux64-devedition/opt + test-sets: + - desktop-screenshot-capture + - linux1804-tests + - web-platform-tests + +linux1804-64-asan/opt: + build-platform: linux64-asan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-tsan/opt: + build-platform: linux64-tsan/opt + test-sets: + - linux-tsan-tests + +linux1804-64-qr/opt: + build-platform: linux64/opt + test-sets: + - awsy + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-qr/debug: + build-platform: linux64/debug + test-sets: + - linux1804-qr-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-asan-qr/opt: + build-platform: linux64-asan/opt + test-sets: + - linux-qr-smoketests + +linux1804-64-ccov/opt: + build-platform: linux64-ccov/opt + test-sets: + - awsy + - linux1804-tests + - linux1804-ccov-tests + - web-platform-tests + - ccov-code-coverage-tests + - mochitest-webgpu + +# Linux64 uses Ubuntu 16.04 and is run on hardware + +linux64/opt: + build-platform: linux64/opt + test-sets: + - talos + - linux-talos-flex + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-shippable/opt: + build-platform: linux64-shippable/opt + test-sets: + - talos + - linux-talos-flex + - raptor-chrome + - raptor-chromium + - raptor-fetch-chromium + - raptor-firefox + - browsertime + - raptor-fetch-firefox + - raptor-profiling + - raptor-fetch-profiling + - talos-profiling + +# QR builds just run a subset right now. +linux64-qr/opt: + build-platform: linux64/opt + test-sets: + - talos + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - talos + - talos-profiling + - raptor-firefox + - browsertime + - raptor-fetch-firefox + +linux64-ccov/opt: + build-platform: linux64-ccov/opt + test-sets: + - talos + +## +# Windows platforms (matching /windows.*/) + +# win32 +windows7-32/debug: + build-platform: win32/debug + test-sets: + - windows-reftest-gpu + - windows-tests + - web-platform-tests + - mochitest-webgpu + +windows7-32/opt: + build-platform: win32/opt + test-sets: + - awsy + - desktop-screenshot-capture + - marionette-framescript + - windows-reftest-gpu + - windows-talos + - windows-tests + - web-platform-tests + - raptor-firefox + - browsertime + - mochitest-webgpu + - web-platform-tests-wdspec-headless + +windows7-32-shippable/opt: + build-platform: win32-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - marionette-framescript + - windows-reftest-gpu + - windows-talos + - windows-tests + - web-platform-tests + - web-platform-tests-wdspec-headless + - raptor-chrome + - raptor-chromium + - raptor-firefox + - browsertime + - mochitest-webgpu + +# win7 qr builds for try purposes +windows7-32-qr/debug: + build-platform: win32/debug + test-sets: + - mochitest-webgpu + - windows-qr-tests + - web-platform-tests + +windows7-32-qr/opt: + build-platform: win32/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - web-platform-tests + +# win64 +windows10-64-ccov-qr/opt: + build-platform: win64-ccov/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-reftest-gpu + - windows-talos + - windows-tests + - windows-ccov-tests + - windows10-tests + - web-platform-tests + - ccov-code-coverage-tests + - mochitest-webgpu + +windows10-64/debug: + build-platform: win64/debug + test-sets: + - mochitest-webgpu + - windows-tests + - windows10-tests + - web-platform-tests + +windows10-64/opt: + build-platform: win64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-talos + - marionette-framescript + - windows-tests + - windows10-tests + - web-platform-tests + - web-platform-tests-backlog + - raptor-firefox + - browsertime + - mochitest-webgpu + - web-platform-tests-wdspec-headless + +windows10-aarch64/opt: + build-platform: win64-aarch64-shippable/opt + test-sets: + - windows-aarch64-tests + +windows10-64-ref-hw-2017/opt: + build-platform: win64-shippable/opt + test-sets: + - raptor-firefox + - browsertime + - talos-ref-hw + +windows10-64-shippable/opt: + build-platform: win64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - windows-talos + - marionette-framescript + - windows-tests + - windows10-tests + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + - raptor-chrome + - raptor-chromium + - raptor-firefox + - browsertime + - raptor-profiling + - talos-profiling + - mochitest-webgpu + +windows10-64-devedition/opt: + build-platform: win64-devedition/opt + test-sets: + - desktop-screenshot-capture + - windows-tests + - windows10-tests + - web-platform-tests + - mochitest-webgpu + +windows10-64-asan/opt: + build-platform: win64-asan/opt + test-sets: + - windows-tests + - windows10-tests + - mochitest-webgpu + +# QR builds just run a subset right now. +windows10-64-qr/debug: + build-platform: win64/debug + test-sets: + - mochitest-webgpu + - windows-qr-tests + - web-platform-tests + +windows10-64-qr/opt: + build-platform: win64/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - raptor-firefox + - browsertime + - web-platform-tests + +windows10-64-shippable-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - awsy + - windows-qr-tests + - windows-talos + - talos-profiling + - raptor-firefox + - browsertime + - web-platform-tests + - mochitest-webgpu + +windows10-64-mingwclang/debug: + build-platform: win64-mingwclang/debug + test-sets: + - mingwclang-tests-debug + +windows10-64-mingwclang/opt: + build-platform: win64-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +windows7-32-mingwclang/debug: + build-platform: win32-mingwclang/debug + test-sets: + - mingwclang-win7-tests-debug + +windows7-32-mingwclang/opt: + build-platform: win32-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +## +# MacOS X platforms (matching /macosx.*/) + +macosx1014-64/debug: + build-platform: macosx64/debug + test-sets: + - macosx1014-64-tests + +macosx1014-64-devedition/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1014-64-tests + +macosx1014-64-qr/opt: + build-platform: macosx64/opt + test-sets: + - macosx1014-64-qr-tests + - desktop-screenshot-capture + - marionette-framescript + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +macosx1014-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - macosx1014-64-qr-tests + - macosx64-talos + - awsy + - browsertime + - desktop-screenshot-capture + - marionette-framescript + - raptor-chrome + - raptor-chromium + - raptor-firefox + - raptor-firefox-power + - raptor-profiling + - talos-profiling + - web-platform-tests + - web-platform-tests-backlog + - web-platform-tests-wdspec-headless + +macosx1014-64-devedition-qr/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1014-64-qr-tests + - web-platform-tests + +macosx1014-64-qr/debug: + build-platform: macosx64/debug + test-sets: + - macosx1014-64-qr-tests + - web-platform-tests + +macosx1014-64-ccov-qr/opt: + build-platform: macosx64-ccov/opt + test-sets: + - macosx1014-64-qr-tests + +## +# Android platforms (matching /android-em.*/) +# +# android-em test platforms execute on android emulators. + +android-em-7.0-x86_64/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-tests + +android-em-7.0-x86_64-qr/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-qr-tests + +android-em-7.0-x86_64-shippable-qr/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-qr-tests + +android-em-7.0-x86_64-qr/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-qr-tests + +# android-hw test platforms execute on real devices attached to Autophone hosts. + +# android-hw-g5-7-0 Motorola Moto G5 Android 7.0 + +android-hw-g5-7-0-arm7-api-16/opt: + build-platform: android-api-16/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-g5-7-0-arm7-api-16-shippable/opt: + build-platform: android-api-16-shippable/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +# android-hw-p2-8-0 Google Pixel 2 Android 8.0 + +android-hw-p2-8-0-arm7-api-16/opt: + build-platform: android-api-16/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-pgo-unittests + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-p2-8-0-arm7-api-16-shippable/opt: + build-platform: android-api-16-shippable/opt + test-sets: + - android-hw-arm7-browsertime + - android-hw-arm7-pgo-unittests + - android-hw-arm7-raptor-cpu-memory + - android-hw-arm7-raptor-cpu-memory-power + +android-hw-p2-8-0-arm7-api-16/debug: + build-platform: android-api-16/debug + test-sets: + - android-hw-arm7-debug-unittests + +android-hw-p2-8-0-android-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64-qr/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64-qr + +android-hw-p2-8-0-android-aarch64/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64-browsertime + - android-hw-aarch64-opt-unittests + - android-hw-aarch64-raptor-cpu-memory + - android-hw-aarch64-raptor-cpu-memory-power + +android-hw-p2-8-0-android-aarch64-shippable/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64-browsertime + - android-hw-aarch64-pgo-unittests + - android-hw-aarch64-raptor-cpu-memory + - android-hw-aarch64-raptor-cpu-memory-power + +android-hw-p2-8-0-android-aarch64/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64-debug-unittests diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml new file mode 100644 index 0000000000..0465c99a9f --- /dev/null +++ b/taskcluster/ci/test/test-sets.yml @@ -0,0 +1,560 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# +# Test names given here reference tests.yml. + +### Feature-specific test groups + +web-platform-tests: + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + +web-platform-tests-backlog: + - web-platform-tests-backlog + - web-platform-tests-reftest-backlog + +web-platform-tests-wdspec-headless: + - web-platform-tests-wdspec-headless + +mochitest-webgpu: + - mochitest-webgpu + +talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g3 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions + +talos-profiling: + - talos-bcv-profiling + - talos-chrome-profiling + # - talos-damp-profiling - Disabled b/c of Bug 1597734 + - talos-dromaeojs-profiling + # - talos-flex-profiling - Disabled b/c of Bug 1597736 + - talos-g1-profiling + - talos-g3-profiling + - talos-g4-profiling + - talos-g5-profiling + - talos-motionmark-profiling + - talos-other-profiling + - talos-sessionrestore-many-windows-profiling + - talos-perf-reftest-profiling + - talos-perf-reftest-singletons-profiling + - talos-svgr-profiling + - talos-tp5o-profiling + - talos-tabswitch-profiling + - talos-realworld-webextensions-profiling + - talos-webgl-profiling + +talos-ref-hw: + - talos-g4 + - talos-webgl + +raptor-firefox: + - raptor-tp6 + - raptor-tp6-live + - raptor-speedometer-firefox + - raptor-stylebench-firefox + - raptor-motionmark-htmlsuite-firefox + - raptor-motionmark-animometer-firefox + - raptor-webaudio-firefox + - raptor-sunspider-firefox + - raptor-wasm-godot-firefox + - raptor-youtube-playback-av1-sfr-firefox + - raptor-youtube-playback-h264-sfr-firefox + - raptor-youtube-playback-hfr-firefox + - raptor-youtube-playback-vp9-sfr-firefox + - raptor-youtube-playback-widevine-hfr-firefox + - raptor-youtube-playback-widevine-h264-sfr-firefox + - raptor-youtube-playback-widevine-vp9-sfr-firefox + - raptor-ares6-firefox + - raptor-jetstream2-firefox + +raptor-chrome: + - raptor-speedometer-chrome + - raptor-stylebench-chrome + - raptor-motionmark-htmlsuite-chrome + - raptor-motionmark-animometer-chrome + - raptor-webaudio-chrome + - raptor-sunspider-chrome + - raptor-wasm-godot-chrome + - raptor-ares6-chrome + - raptor-jetstream2-chrome + - raptor-youtube-playback-av1-sfr-chrome + - raptor-youtube-playback-h264-sfr-chrome + - raptor-youtube-playback-hfr-chrome + - raptor-youtube-playback-vp9-sfr-chrome + - raptor-youtube-playback-widevine-hfr-chrome + - raptor-youtube-playback-widevine-h264-sfr-chrome + - raptor-youtube-playback-widevine-vp9-sfr-chrome + +raptor-profiling: + - raptor-speedometer-firefox-profiling + - raptor-stylebench-firefox-profiling + - raptor-motionmark-htmlsuite-firefox-profiling + - raptor-motionmark-animometer-firefox-profiling + - raptor-webaudio-firefox-profiling + - raptor-sunspider-firefox-profiling + - raptor-wasm-godot-firefox-profiling + - raptor-ares6-firefox-profiling + - raptor-jetstream2-firefox-profiling + +raptor-chromium: + - raptor-speedometer-chromium + - raptor-stylebench-chromium + - raptor-motionmark-htmlsuite-chromium + - raptor-motionmark-animometer-chromium + - raptor-webaudio-chromium + - raptor-sunspider-chromium + - raptor-wasm-godot-chromium + - raptor-ares6-chromium + - raptor-jetstream2-chromium + +# Fetch tasks are only supported on Linux for now, +# so these need to be separate sets. +raptor-fetch-firefox: + - raptor-unity-webgl-firefox + - raptor-wasm-misc-firefox + - raptor-wasm-misc-baseline-firefox + - raptor-wasm-misc-optimizing-firefox + - raptor-wasm-godot-baseline-firefox + - raptor-wasm-godot-optimizing-firefox + - raptor-assorted-dom-firefox + +raptor-fetch-profiling: + - raptor-unity-webgl-firefox-profiling + - raptor-wasm-misc-firefox-profiling + - raptor-wasm-misc-baseline-firefox-profiling + - raptor-wasm-misc-optimizing-firefox-profiling + - raptor-wasm-godot-baseline-firefox-profiling + - raptor-wasm-godot-optimizing-firefox-profiling + - raptor-assorted-dom-firefox-profiling + +raptor-fetch-chromium: + - raptor-unity-webgl-chromium + - raptor-wasm-misc-chromium + - raptor-assorted-dom-chromium + - raptor-jetstream2-chromium + +browsertime: + - browsertime-tp6 + - browsertime-speedometer + - browsertime-ares6 + - browsertime-tp6-profiling + +# Raptor desktop power tests +# (requires Intel Power Gadget to be pre-installed) +raptor-firefox-power: + - raptor-youtube-playback-v9-power-firefox + - raptor-youtube-playback-h264-power-firefox + +awsy: + - awsy + - awsy-base + - awsy-dmd + - awsy-base-dmd + - awsy-tp6 + +ccov-code-coverage-tests: + - test-coverage + - test-coverage-wpt + +mochitest-headless: + - mochitest-plain-headless + +mochitest-valgrind: + - mochitest-valgrind + +marionette-framescript: + - marionette-framescript + +marionette-headless: + - marionette-headless + +desktop-screenshot-capture: + - browser-screenshots + + +### Specific platform test sets + +# Linux + +linux32-tests: + - web-platform-tests + +linux1804-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - reftest-no-accel + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +linux1804-qr-tests: + - cppunit + - crashtest + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - mochitest-a11y + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - xpcshell + +linux1804-ccov-tests: + # jittest run on regular linux builds is redundant with SM(p), so we omit it + # from the linux1804-tests and linux1804-qr-tests sets. But, we still want + # to run it on ccov builds, so we have this set that has it and which runs + # on ccov builds. + - jittest + +linux-qr-smoketests: + - crashtest + - reftest + +linux-talos-flex: + - talos-flex + +# Windows + +windows-qr-tests: + - crashtest + - mochitest-chrome-gpu + - mochitest-browser-chrome + - mochitest-plain + - mochitest-plain-gpu + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl2-core + - reftest + +windows-reftest-gpu: + - reftest-gpu + +windows-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + # - jittest # See comment in windows-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - reftest + - reftest-no-accel + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +windows-ccov-tests: + # jittest run on regular windows builds is redundant with SM(p), so we omit + # it from the windows-tests set. But, we still want to run it on ccov builds, + # so we have this set that has it and which runs on ccov builds. + - jittest + +windows10-tests: + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-devtools-chrome + +mingwclang-tests-debug: + - cppunit + - firefox-ui-functional-local + - firefox-ui-functional-remote + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + +# The following is the same as the above block, minus reftests. +# Reftests were disabled due to frequent hangs; see bug 1642719 +# for re-enabling. +mingwclang-win7-tests-debug: + - cppunit + - firefox-ui-functional-local + - firefox-ui-functional-remote + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + # - reftest - disabled, see bug 1642719 + - telemetry-tests-client + +mingwclang-tests-opt: + - cppunit + - mochitest-chrome-gpu + - mochitest-plain-gpu + +windows-aarch64-tests: + - crashtest + - mochitest-media + - reftest + +windows-talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-svgr + - talos-tp5o + - talos-xperf + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + +# macOS + +macosx1014-64-tests: + - crashtest + - reftest + - web-platform-tests-reftest + +macosx1014-64-qr-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + - jittest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-plain + - mochitest-plain-gpu + - mochitest-media + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +macosx64-talos: + - talos-bcv + - talos-chrome + - talos-damp + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-realworld-webextensions + - talos-webgl + # - talos-tabswitch # Bug 1453007 times out + # - talos-h1 # too long to unpack profile- Bug 1442893 + +# Android + +android-x86_64-tests: + - cppunit + - crashtest + - geckoview-junit + - gtest + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + # - mochitest-webgl1-core # Harness issues? + # - mochitest-webgl1-ext + # - mochitest-webgl2-core # Emulator not creating ES3? + # - mochitest-webgl2-ext + - reftest + - test-verify + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - xpcshell + +android-x86_64-qr-tests: + - crashtest + - geckoview-junit + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - reftest + +android-hw-aarch64-opt-unittests: + - jittest + +android-hw-arm7-pgo-unittests: + - jittest + - jsreftest + +android-hw-aarch64-pgo-unittests: + - jittest + +android-hw-arm7-debug-unittests: + - jittest + - jsreftest + +android-hw-aarch64-debug-unittests: + - jittest + +android-hw-aarch64-qr: + - crashtest-qr + - reftest-qr + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + +android-hw-arm7-raptor-cpu-memory: + - raptor-speedometer-geckoview-cpu-memory + - raptor-scn-cpu-memory-idle-geckoview + - raptor-scn-cpu-memory-idle-refbrow + - raptor-scn-cpu-memory-idle-bg-geckoview + - raptor-scn-cpu-memory-idle-bg-refbrow + +android-hw-arm7-raptor-cpu-memory-power: + - raptor-speedometer-geckoview-cpu-memory-power + - raptor-scn-cpu-memory-power-idle-geckoview + - raptor-scn-cpu-memory-power-idle-refbrow + - raptor-scn-cpu-memory-power-idle-bg-geckoview + - raptor-scn-cpu-memory-power-idle-bg-refbrow + +android-hw-aarch64-raptor-cpu-memory: + - raptor-speedometer-geckoview-cpu-memory + - raptor-scn-cpu-memory-idle-geckoview + - raptor-scn-cpu-memory-idle-refbrow + - raptor-scn-cpu-memory-idle-bg-geckoview + - raptor-scn-cpu-memory-idle-bg-refbrow + +android-hw-aarch64-raptor-cpu-memory-power: + - raptor-speedometer-geckoview-cpu-memory-power + - raptor-scn-cpu-memory-power-idle-geckoview + - raptor-scn-cpu-memory-power-idle-refbrow + - raptor-scn-cpu-memory-power-idle-bg-geckoview + - raptor-scn-cpu-memory-power-idle-bg-refbrow + +android-hw-aarch64-browsertime: + - browsertime-power + - browsertime-tp6m + - browsertime-tp6m-live + - browsertime-tp6m-profiling + - browsertime-speedometer-mobile + - browsertime-youtube-playback-mobile + - browsertime-unity-webgl-mobile + +android-hw-arm7-browsertime: + - browsertime-power + - browsertime-tp6m + - browsertime-tp6m-live + - browsertime-tp6m-profiling + - browsertime-speedometer-mobile + - browsertime-youtube-playback-mobile + - browsertime-unity-webgl-mobile + +linux-tsan-tests: + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-media + - mochitest-plain + - web-platform-tests + - web-platform-tests-reftest + - xpcshell diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml new file mode 100644 index 0000000000..b669421775 --- /dev/null +++ b/taskcluster/ci/test/web-platform.yml @@ -0,0 +1,342 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: web-platform-tests + instance-size: xlarge + max-run-time: + by-test-platform: + android-em-7.0-x86_64/debug: 7200 + default: 5400 + variants: + by-test-platform: + linux.*64/debug: ['fission'] + default: ['fission'] + fission-run-on-projects: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + linux.*64(-shippable)?-qr/opt: ['trunk'] + windows10-64(-shippable)?-qr/opt: ['trunk'] + default: [] + fission-tier: + by-test-platform: + linux.*64-qr/debug: 1 + linux.*64(-shippable)?-qr/opt: 1 + windows10-64(-shippable)?-qr/opt: 1 + default: 2 + mozharness: + script: web_platform_tests.py + config: + by-test-platform: + windows.*: + - web_platform_tests/prod_config_windows_taskcluster.py + macosx.*: + - web_platform_tests/prod_config_mac.py + android-em.*: + - android/androidx86_7_0.py + - web_platform_tests/prod_config_android.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + target: + by-test-platform: + android-em-7.0-x86_64-shippable(-qr)?/opt: geckoview-androidTest.apk + android-em-7.0-x86_64(-qr)?/opt: geckoview-androidTest.apk + android-em-7.0-x86_64(-qr)?/debug: geckoview-androidTest.apk + default: null + python-3: true + +web-platform-tests: + description: "Web platform test run" + suite: web-platform-tests + treeherder-symbol: W(wpt) + virtualization: + by-test-platform: + windows10-64.*/.*: hardware + default: virtual + chunks: + by-test-platform: + android.*: 16 + linux.*64-tsan/opt: 26 + linux.*64-asan/opt: 22 + linux.*64-ccov.*/opt: 20 + linux.*/debug: 16 + macosx.*/debug: 18 + windows7.*/opt: 12 + windows.*/debug: 14 + windows10-64-ccov.*/opt: 14 + default: 10 + max-run-time: + by-test-platform: + .*-ccov.*/.*: 10800 + linux.*64-tsan/opt: 14400 + linux.*64-asan/opt: 14400 + linux.*64(-qr)?/debug: 10800 + macosx.*/debug: 10800 + android-em.*/.*: 10800 + default: 7200 + e10s: true + run-on-projects: + by-test-platform: + linux1804-32-shippable/opt: ['release'] # skip integration, Bug 1599197 + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-backlog: + description: "Web platform test (backlog) run" + treeherder-symbol: W-b(wpt) + chunks: + by-test-platform: + android.*: 3 + default: 1 + max-run-time: + by-test-platform: + android.*/opt: 7200 + default: 3600 + variants: [] + e10s: true + run-on-projects: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-reftest: + description: "Web platform reftest run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W(Wr) + virtualization: + by-test-platform: + windows10-64.*-qr/.*: virtual-with-gpu + default: virtual + chunks: + by-test-platform: + .*-ccov.*/.*: 8 + linux.*64-tsan/opt: 8 + linux1804-64(-qr|-asan)/.*: 6 + linux1804-64(-shippable|-devedition)?/opt: 3 + macosx10.*-64/debug: 6 + macosx10.*-64/opt: 3 + windows.*-(32|64)(-qr)?/debug: 5 + android.*: 6 + default: 4 + e10s: true + run-on-projects: built-projects + tier: + by-test-platform: + linux1804-64-asan/opt: 2 + default: default + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + +web-platform-tests-reftest-backlog: + description: "Web platform reftest (backlog) run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W-b(Wr) + virtualization: virtual + chunks: 1 + variants: [] + e10s: true + run-on-projects: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + +web-platform-tests-wdspec: + description: "Web platform webdriver-spec run" + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(Wd) + chunks: + by-test-platform: + .*-ccov.*/.*: 4 + default: 3 + mozharness: + extra-options: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + run-on-projects: + by-test-platform: + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + tier: + by-test-platform: + android.*: 3 + linux.*64-asan/opt: 2 + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + fetches: + toolchain: + by-test-platform: + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + + +web-platform-tests-wdspec-headless: + description: "Web platform webdriver-spec headless run" + suite: + name: web-platform-tests-wdspec + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(WdH) + chunks: 2 + mozharness: + extra-options: + by-test-platform: + windows.*: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + - --headless + - --headless-width=1600 + - --headless-height=1200 + default: + - --test-type=wdspec + - --setpref=toolkit.asyncshutdown.log=true + - --headless + fetches: + toolchain: + by-test-platform: + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + +web-platform-tests-crashtest: + description: "Web platform crashtests run" + schedules-component: web-platform-tests-crashtest + treeherder-symbol: W(Wc) + chunks: 1 + mozharness: + extra-options: + - --test-type=crashtest + run-on-projects: + by-test-platform: + linux.*-qr/.*: ['release'] # skip on integration branches due to high load + windows.*-qr/.*: ['release'] # skip on integration branches due to high load + default: built-projects + tier: + by-test-platform: + linux1804-64-asan/opt: 2 + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + +web-platform-tests-print-reftest: + description: "Web platform print-reftest run" + schedules-component: web-platform-tests-print-reftest + treeherder-symbol: W(Wp) + chunks: 1 + test-manifest-loader: null # don't load tests in the taskgraph + tier: + by-test-platform: + .*-qr/.*: 2 # can't be tier-1 if it's not running on integration branches + default: default + run-on-projects: built-projects + mozharness: + extra-options: + - --test-type=print-reftest + +test-verify-wpt: + description: "Extra verification of web-platform tests modified on this push" + suite: + category: test-verify + name: test-verify-wpt + variants: [] + treeherder-symbol: TVw + max-run-time: 10800 + run-on-projects: + by-test-platform: + # do not run on ccov or asan + .*-ccov.*/.*: [] + .*-asan/.*: [] + .*shippable.*: [] + # do not run on beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + test-manifest-loader: null # don't load tests in the taskgraph + optimization: + test-verify: ["test-verify-wpt"] + tier: 2 + mozharness: + extra-options: + - --verify + +test-coverage-wpt: + description: "Per web-platform test coverage" + suite: + category: test-coverage + name: test-coverage-wpt + schedules-component: test-coverage-wpt + treeherder-symbol: TCw + max-run-time: 10800 + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --per-test-coverage diff --git a/taskcluster/ci/test/xpcshell.yml b/taskcluster/ci/test/xpcshell.yml new file mode 100644 index 0000000000..7bd3d366e0 --- /dev/null +++ b/taskcluster/ci/test/xpcshell.yml @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: xpcshell + variants: + by-test-platform: + linux1804-64/debug: ['fission', 'socketprocess_networking'] + default: ['fission'] + target: + by-test-platform: + android-em-7.*: geckoview-androidTest.apk + default: null + fission-tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/androidx86_7_0.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + requires-signed-builds: + by-test-platform: + windows10-64-asan/opt: false # No XPCShell on ASAN yet + windows7-32-mingwclang/opt: false + windows7-32-mingwclang/debug: false + windows10-64-mingwclang/opt: false + windows10-64-mingwclang/debug: false + windows.*: true + default: false + +xpcshell: + description: "xpcshell test run" + treeherder-symbol: X(X) + run-on-projects: + by-test-platform: + .*-tsan/opt: ['trunk'] + windows10-64-asan/opt: [] # No XPCShell on ASAN yet + default: built-projects + chunks: + by-test-platform: + .*-ccov.*/.*: 6 + windows10-64-ccov-qr/opt: 6 + android-em-7.*: 4 + macosx1014-64(-shippable|-devedition|-asan)?(-qr)?/.*: 2 + linux1804-64(-qr)?/debug: 6 + .*-tsan/opt: 8 + windows(7-32|10-64)(-shippable|-devedition|-asan|.*-qr)?/.*: 2 + default: 5 + max-run-time: + by-test-platform: + macosx1014-64(-shippable|-devedition|-asan)?(-qr)?/.*: 5400 + windows(7-32|10-64)(-shippable)?/opt: 5400 + .*-ccov.*/.*: 7200 + .*-tsan/opt: 7200 + android-em-7.*: 5400 + default: 2700 + allow-software-gl-layers: false + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: default + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*aarch64.*: + - win32-node + - win32-minidump-stackwalk + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-emulator-linux + - linux64-node + - linux64-minidump-stackwalk -- cgit v1.2.3