diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:14:29 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:14:29 +0000 |
| commit | fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8 (patch) | |
| tree | 4c1ccaf5486d4f2009f9a338a98a83e886e29c97 /taskcluster | |
| parent | Releasing progress-linux version 124.0.1-1~progress7.99u1. (diff) | |
| download | firefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.tar.xz firefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.zip | |
Merging upstream version 125.0.1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster')
97 files changed, 1340 insertions, 321 deletions
diff --git a/taskcluster/ci/artifact-build/kind.yml b/taskcluster/ci/artifact-build/kind.yml index aa15674935..76eb0003a4 100644 --- a/taskcluster/ci/artifact-build/kind.yml +++ b/taskcluster/ci/artifact-build/kind.yml @@ -21,7 +21,7 @@ job-defaults: treeherder: kind: build symbol: AB - tier: 2 + tier: 1 run-on-projects: ['mozilla-central'] worker-type: b-linux-gcp worker: diff --git a/taskcluster/ci/beetmover-geckoview/kind.yml b/taskcluster/ci/beetmover-geckoview/kind.yml index 109b42eb6a..e608ded508 100644 --- a/taskcluster/ci/beetmover-geckoview/kind.yml +++ b/taskcluster/ci/beetmover-geckoview/kind.yml @@ -20,6 +20,7 @@ only-for-attributes: not-for-build-platforms: - linux-shippable/opt - linux64-shippable/opt + - linux-aarch64-shippable/opt - linux64-nightlyasrelease/opt - macosx64-shippable/opt - macosx64-nightlyasrelease/opt diff --git a/taskcluster/ci/build/ios.yml b/taskcluster/ci/build/ios.yml new file mode 100644 index 0000000000..1e8c18caab --- /dev/null +++ b/taskcluster/ci/build/ios.yml @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + index: + product: firefox + treeherder: + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run-on-projects: ['trunk'] + run: + using: mozharness + actions: [get-secrets, build] + secrets: true + script: "mozharness/scripts/fx_desktop_build.py" + config: + - builds/releng_base_firefox.py + extra-config: + app_name: mobile/ios + stage_platform: ios + disable_package_metrics: true + use-sccache: true + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - linux64-dump_syms + - linux64-hfsplus + - linux64-libdmg + - linux64-rust-ios + - linux64-rust-size + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-sccache + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +ios/debug: + description: "ios debug" + index: + job-name: ios-debug + treeherder: + platform: ios/debug + symbol: B + run: + extra-config: + mozconfig_platform: ios + mozconfig-variant: debug + fetches: + toolchain: + - ios-sdk + +ios/opt: + description: "ios opt" + index: + job-name: ios-opt + treeherder: + platform: ios/opt + symbol: B + run: + extra-config: + mozconfig_platform: ios + mozconfig-variant: opt + fetches: + toolchain: + - ios-sdk + +ios-non-unified/plain: + description: "ios non-unified" + index: + job-name: ios-non-unified-plain + treeherder: + platform: ios/plain + symbol: Bp-nu + worker: + env: + PERFHERDER_EXTRA_OPTIONS: non-unified + MOZ_AUTOMATION_BUILD_SYMBOLS: '0' + run: + extra-config: + mozconfig_platform: ios + extra_mozconfig_content: + - ac_add_options --disable-unified-build + mozconfig-variant: opt + fetches: + toolchain: + - ios-sdk + +ios-sim/debug: + description: "ios-sim debug" + index: + job-name: ios-sim-debug + treeherder: + platform: ios/debug + symbol: Bsim + worker: + env: + PERFHERDER_EXTRA_OPTIONS: ios-sim + run: + extra-config: + mozconfig_platform: ios-sim + mozconfig-variant: debug + fetches: + toolchain: + - ios-sim-sdk + +ios-sim/opt: + description: "ios-sim opt" + index: + job-name: ios-sim-opt + treeherder: + platform: ios/opt + symbol: Bsim + worker: + env: + PERFHERDER_EXTRA_OPTIONS: ios-sim + run: + extra-config: + mozconfig_platform: ios-sim + mozconfig-variant: opt + fetches: + toolchain: + - ios-sim-sdk diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index 886f1ef433..b09cbe6926 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -22,6 +22,7 @@ jobs-from: - android.yml - android-asan.yml - android-stuff.yml + - ios.yml - linux.yml - linux-base-toolchains.yml - macosx.yml diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml index 73301313d6..1e38416e1c 100644 --- a/taskcluster/ci/build/linux.yml +++ b/taskcluster/ci/build/linux.yml @@ -1262,7 +1262,7 @@ linux64-asan-fuzzing-ccov/opt: fetches: toolchain: - linux64-clang - - linux64-rust + - linux64-rust-dev - linux64-rust-size - linux64-cbindgen - linux64-dump_syms @@ -1756,8 +1756,7 @@ linux64-add-on-devel/opt: - sysroot-wasm32-wasi linux64-aarch64/opt: - description: "Linux64 aarch64 Opt" - use-pgo: linux64-shippable/opt + description: "Linux AArch64 (a.k.a. ARM64) Opt" index: product: firefox job-name: linux64-aarch64-opt @@ -1766,7 +1765,6 @@ linux64-aarch64/opt: treeherder: platform: linux64-aarch64/opt symbol: B - tier: 2 worker: max-run-time: 7200 run: @@ -1783,6 +1781,60 @@ linux64-aarch64/opt: secrets: true tooltool-downloads: public run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-node + - linux64-pkgconf + - sysroot-aarch64-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-aarch64-shippable/opt: + description: "Linux AArch64 (a.k.a. ARM64) Shippable" + use-pgo: linux64-shippable/opt + index: + product: firefox + job-name: linux-aarch64-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: firefox + treeherder: + platform: linux-aarch64-shippable/opt + symbol: Bpgo(B) + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 9000 + env: + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + extra_mozconfig_content: + - ac_add_options --target=aarch64 + - unset MOZ_STDCXX_COMPAT + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + run-on-projects: ['mozilla-central'] fetches: toolchain: - linux64-clang @@ -1795,5 +1847,3 @@ linux64-aarch64/opt: - sysroot-aarch64-linux-gnu - sysroot-x86_64-linux-gnu - sysroot-wasm32-wasi - optimization: - skip-unless-expanded: null diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 73496c538b..894ce9399a 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -798,6 +798,7 @@ mac-signing: - "/Contents/MacOS/XUL" - "/Contents/MacOS/pingsender" - "/Contents/MacOS/minidump-analyzer" + - "/Contents/MacOS/nmhproxy" - "/Contents/MacOS/*.dylib" - "/Contents/Resources/gmp-clearkey/*/*.dylib" @@ -839,6 +840,7 @@ mac-signing: - "/Contents/Library/LaunchServices/org.mozilla.updater" - "/Contents/MacOS/pingsender" - "/Contents/MacOS/minidump-analyzer" + - "/Contents/MacOS/nmhproxy" - deep: false runtime: true diff --git a/taskcluster/ci/fetch/browsertime.yml b/taskcluster/ci/fetch/browsertime.yml index 614ddce593..84cb862b7a 100644 --- a/taskcluster/ci/fetch/browsertime.yml +++ b/taskcluster/ci/fetch/browsertime.yml @@ -126,9 +126,9 @@ win32-chromedriver-122: type: static-url artifact-name: chromedriver_win32_122.tar.zst add-prefix: "122" - url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/win32/chrome-win32.zip - sha256: 4778e327a30972fa1d39a9aabc60c04e82553b7cafa2f8f768eaec9ee0935eab - size: 134566218 + url: https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.57/win32/chromedriver-win32.zip + sha256: f6309630e014ffb00dd78813e75d8bf664e755ea2ebc79974c8a0478ca96a154 + size: 8001652 linux64-chromedriver-122: description: "Linux64 chromedriver v122" @@ -136,9 +136,9 @@ linux64-chromedriver-122: type: static-url artifact-name: chromedriver_linux64_122.tar.zst add-prefix: "122" - url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/linux64/chrome-linux64.zip - sha256: 5514382fc80e092f76a6a7ea85bdfcafacd7e62e1c831a7842f0eca003383c5d - size: 149627735 + url: https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.57/linux64/chromedriver-linux64.zip + sha256: 0c80bb130eda0afb583155d55ec9cc22d75e2cf0dd90d657647df923465d584b + size: 8613042 mac64-chromedriver-122: description: "Mac64 chromedriver v122" @@ -146,9 +146,9 @@ mac64-chromedriver-122: type: static-url artifact-name: chromedriver_mac64_122.tar.zst add-prefix: "122" - url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/mac-x64/chrome-mac-x64.zip - sha256: 753d2814bb82d723289eea76628e9b043bc36666d7cf285876dc4f19ca811c04 - size: 155446914 + url: https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.57/mac-x64/chromedriver-mac-x64.zip + sha256: 31d9be68e77c733357ee825c34249dbb94c5c08961e58dfb38b7623e9383fa69 + size: 9209421 mac-arm-chromedriver-122: description: "Mac64 arm chromedriver v122" @@ -156,9 +156,9 @@ mac-arm-chromedriver-122: type: static-url artifact-name: chromedriver_mac-arm_122.tar.zst add-prefix: "122" - url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/mac-arm64/chrome-mac-arm64.zip - sha256: 1edd4096771669e6c429eff5a459f520defd43afb798fc86d58608d95355be6e - size: 146946870 + url: https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.57/mac-arm64/chromedriver-mac-arm64.zip + sha256: 645f277c05d7f7d238c7a45412f42489767f48cba05d8632ce00b54b3704d4e1 + size: 8293896 upload-speed-test-file: description: "File used for the upload speed performance test" diff --git a/taskcluster/ci/fetch/toolchains.yml b/taskcluster/ci/fetch/toolchains.yml index 50ffb065b0..4ab075aacd 100644 --- a/taskcluster/ci/fetch/toolchains.yml +++ b/taskcluster/ci/fetch/toolchains.yml @@ -296,8 +296,8 @@ sccache: fetch: type: git repo: https://github.com/mozilla/sccache - # 0.7.6 release - revision: 794c987b7843f404b9e50513dae978f807ba0a60 + # 0.7.7 release + revision: e976c42c2141969915bdd39dd3177db6f27405a8 fxc2: description: fxc2 source code @@ -491,28 +491,6 @@ nodejs-12-linux64: strip-components: 1 add-prefix: node/ -nodejs-16-linux64: - description: nodejs 16 for x64 linux - fetch: - type: static-url - url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz - sha256: c88b52497ab38a3ddf526e5b46a41270320409109c3f74171b241132984fd08f - size: 22492000 - artifact-name: nodejs.tar.zst - strip-components: 1 - add-prefix: node/ - -nodejs-16-macosx64: - description: nodejs 16 for x64 macOS - fetch: - type: static-url - url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-darwin-x64.tar.xz - sha256: 601d1d632b9815dbb647f5cd440a95fa651110b5b4bcaf07ed3aa7886525716b - size: 20975524 - artifact-name: nodejs.tar.zst - strip-components: 1 - add-prefix: node/ - nodejs-18-macosx64: description: nodejs 18 for x64 macOS fetch: @@ -524,17 +502,6 @@ nodejs-18-macosx64: strip-components: 1 add-prefix: node/ -nodejs-16-macosx64-arm64: - description: nodejs 16 for arm64 macOS - fetch: - type: static-url - url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-darwin-arm64.tar.xz - sha256: 7313c9db5f5140ece847a97f366032a10bddb6d87fa28a361ada918addcd5c73 - size: 19363804 - artifact-name: nodejs.tar.zst - strip-components: 1 - add-prefix: node/ - nodejs-18-macosx64-arm64: description: nodejs 18 for arm64 macOS fetch: @@ -546,17 +513,6 @@ nodejs-18-macosx64-arm64: strip-components: 1 add-prefix: node/ -nodejs-16-win32: - description: nodejs 16 for x86 Windows - fetch: - type: static-url - url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-win-x86.zip - sha256: 17e2bf8ed00b3d15321743104104c8b6ef55677dfc18ee2ed44de64fbf2c4def - size: 25424808 - artifact-name: nodejs.tar.zst - strip-components: 1 - add-prefix: node/ - nodejs-18-win32: description: nodejs 18 for x86 Windows fetch: @@ -568,17 +524,6 @@ nodejs-18-win32: strip-components: 1 add-prefix: node/ -nodejs-16-win64: - description: nodejs 16 for x64 Windows - fetch: - type: static-url - url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-win-x64.zip - sha256: 534ca7a24e999c81cec847a498cc43d47e2bb158f6edf639e5297f2718350e96 - size: 27072850 - artifact-name: nodejs.tar.zst - strip-components: 1 - add-prefix: node/ - nodejs-18-win64: description: nodejs 18 for x64 Windows fetch: @@ -712,3 +657,14 @@ windows-rs: artifact-name: windows-rs.tar.zst strip-components: 1 add-prefix: windows-rs/ + +libxml2: + description: Source of the libxml2 library + fetch: + type: static-url + url: https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.12/libxml2-v2.9.12.tar.gz + sha256: 98bfa7a9a5e2a75638422050740448ee9f02bf4dc2075c9822d7747d5ff9e617 + size: 5233674 + artifact-name: libxml2.tar.zst + strip-components: 1 + add-prefix: libxml2/ diff --git a/taskcluster/ci/mar-signing/kind.yml b/taskcluster/ci/mar-signing/kind.yml index f1337058d0..b91498cf62 100644 --- a/taskcluster/ci/mar-signing/kind.yml +++ b/taskcluster/ci/mar-signing/kind.yml @@ -16,6 +16,7 @@ kind-dependencies: only-for-build-platforms: - linux-shippable/opt - linux64-shippable/opt + - linux-aarch64-shippable/opt - macosx64-shippable/opt - win32-shippable/opt - win64-shippable/opt diff --git a/taskcluster/ci/repackage/kind.yml b/taskcluster/ci/repackage/kind.yml index e67ce5566b..2644e9d5f5 100644 --- a/taskcluster/ci/repackage/kind.yml +++ b/taskcluster/ci/repackage/kind.yml @@ -27,6 +27,7 @@ only-for-build-platforms: - linux64-devedition/opt - linux64-asan-reporter-shippable/opt - linux64/opt + - linux-aarch64-shippable/opt - macosx64-shippable/opt - macosx64-devedition/opt - macosx64/opt diff --git a/taskcluster/ci/searchfox/kind.yml b/taskcluster/ci/searchfox/kind.yml index 98c2235fe1..71f94197ce 100644 --- a/taskcluster/ci/searchfox/kind.yml +++ b/taskcluster/ci/searchfox/kind.yml @@ -99,6 +99,46 @@ jobs: - sysroot-x86_64-linux-gnu - sysroot-wasm32-wasi + ios-searchfox/debug: + description: "ios Debug Cross-compile Searchfox" + index: + job-name: ios-searchfox-debug + treeherder: + platform: ios/debug + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "ios" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + keep-artifacts: false + extra-config: + app_name: mobile/ios + stage_platform: ios + mozconfig_platform: ios + disable_package_metrics: true + mozconfig-variant: debug-searchfox + fetches: + toolchain: + - linux64-clang + - linux64-hfsplus + - linux64-libdmg + - linux64-nasm + - linux64-node + - linux64-rust-ios + - linux64-cbindgen + - linux64-dump_syms + - ios-sdk + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + win64-searchfox/debug: description: "Win64 Searchfox Debug (clang-cl)" index: diff --git a/taskcluster/ci/source-test/mozlint.yml b/taskcluster/ci/source-test/mozlint.yml index 28f2381827..e101049103 100644 --- a/taskcluster/ci/source-test/mozlint.yml +++ b/taskcluster/ci/source-test/mozlint.yml @@ -607,3 +607,15 @@ trojan-source: - '**/*.py' - '**/*.rs' - 'tools/lint/trojan-source.yml' + +ignorefile-test: + description: .gitignore vs .hgignore consistency check + platform: lint/opt + treeherder: + symbol: ignorefile + run: + mach: lint -v -l ignorefile .gitignore .hgignore + when: + files-changed: + - '.gitignore' + - '.hgignore' diff --git a/taskcluster/ci/source-test/python.yml b/taskcluster/ci/source-test/python.yml index 3fd8b25b32..d9e0ede88b 100644 --- a/taskcluster/ci/source-test/python.yml +++ b/taskcluster/ci/source-test/python.yml @@ -24,6 +24,7 @@ job-defaults: PATH: "/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin" default: max-run-time: 3600 + use-system-python: false treeherder: kind: test tier: 1 diff --git a/taskcluster/ci/source-test/taskgraph.yml b/taskcluster/ci/source-test/taskgraph.yml index 132d5e488c..ab6883919a 100644 --- a/taskcluster/ci/source-test/taskgraph.yml +++ b/taskcluster/ci/source-test/taskgraph.yml @@ -28,9 +28,7 @@ diff: using: run-task cwd: '{checkout}' command: >- - mkdir -p /builds/worker/artifacts/diffs && - ./mach taskgraph target --fast -p "taskcluster/test/params" -o "/builds/worker/artifacts/diffs/diff.txt" --diff {base_rev} && - python taskcluster/scripts/misc/summarize-tgdiff.py /builds/worker/artifacts/diffs/ 20 + echo 'disabled' when: files-changed: - 'taskcluster/ci/**' diff --git a/taskcluster/ci/spidermonkey/linux.yml b/taskcluster/ci/spidermonkey/linux.yml index cf7604bf5a..fbf3c8d0e2 100644 --- a/taskcluster/ci/spidermonkey/linux.yml +++ b/taskcluster/ci/spidermonkey/linux.yml @@ -202,6 +202,28 @@ sm-pbl-linux64/opt: - linux64-pkgconf - sysroot-x86_64-linux-gnu +sm-pbl-linux64/debug: + description: "Spidermonkey PBL Debug" + index: + job-name: sm-pbl-linux64-debug + treeherder: + symbol: SM(pbl) + platform: linux64/debug + tier: 2 + run: + spidermonkey-variant: pbl-debug + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + sm-plain-linux32/debug: description: "Spidermonkey Plain" index: diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml index b1422e1b2f..fab92c40a6 100644 --- a/taskcluster/ci/test/browsertime-desktop.yml +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -103,6 +103,11 @@ browsertime-tp6: - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --extra-profiler-run + macosx1015.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run default: - --chimera - --conditioned-profile=settled @@ -179,6 +184,11 @@ browsertime-responsiveness: - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled - --extra-profiler-run + macosx1015.*: + - --cold + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run default: - --cold - --conditioned-profile=settled @@ -208,6 +218,11 @@ browsertime-tp6-essential: - --conditioned-profile=settled - --collect-perfstats - --extra-profiler-run + macosx1015.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run default: - --chimera - --conditioned-profile=settled @@ -283,6 +298,13 @@ browsertime-tp6-bytecode: - --conditioned-profile=settled - --test-bytecode-cache - --extra-profiler-run + macosx1015.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run default: - --chimera - --browsertime @@ -304,6 +326,8 @@ browsertime-benchmark: macosx1400.*shippable.*: - [speedometer, sp] - [speedometer3, sp3] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] default: by-app: firefox: @@ -315,6 +339,8 @@ browsertime-benchmark: - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] - [motionmark-htmlsuite, mm-h] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] @@ -331,6 +357,8 @@ browsertime-benchmark: - [motionmark-htmlsuite, mm-h] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] # - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535) - [stylebench, sb] @@ -342,6 +370,8 @@ browsertime-benchmark: - [speedometer3, sp3] - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] chrome: by-test-platform: macosx1015.*shippable.*: @@ -351,6 +381,8 @@ browsertime-benchmark: - [matrix-react-bench, mrb] - [motionmark-animometer, mm-a] - [motionmark-htmlsuite, mm-h] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] @@ -366,6 +398,8 @@ browsertime-benchmark: - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite, mm-h] - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] @@ -381,6 +415,8 @@ browsertime-benchmark: - [motionmark-animometer-ramp, mm-ar] - [motionmark-htmlsuite, mm-h] - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] - [speedometer, sp] - [speedometer3, sp3] - [stylebench, sb] @@ -462,6 +498,8 @@ browsertime-benchmark: motionmark-htmlsuite: 1500 motionmark-htmlsuite-ramp: 3000 motionmark-animometer-ramp: 2100 + motionmark-1-3: 3000 + motionmark-htmlsuite-1-3: 3000 unity-webgl: 2400 speedometer3: by-test-platform: @@ -863,6 +901,11 @@ browsertime-tp6-webextensions: - --browsertime - --browsertime-no-ffwindowrecorder - --conditioned-profile=settled-webext + macosx1015.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run default: - --chimera - --browsertime diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml index 89bbc5fb6b..8e685a590e 100644 --- a/taskcluster/ci/test/browsertime-mobile.yml +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -87,6 +87,8 @@ browsertime-tp6m: mozharness: extra-options: - --chimera + # Bug 1884667 - Fresh profiles cause high frequency intermittents + - --conditioned-profile=settled tier: by-app: geckoview: 1 @@ -115,7 +117,7 @@ browsertime-tp6m: - reddit - sina - [stackoverflow, stacko] - - web-de + - bild-de - wikipedia - youtube run-on-projects: @@ -292,7 +294,7 @@ browsertime-tp6m-live: - [stackoverflow, stacko] - [stackoverflow-question, stack-q] - [urbandictionary-define, urbict-def] - - web-de + - bild-de - [wikia-marvel, wikia-m] - wikipedia - youtube @@ -378,11 +380,12 @@ browsertime-benchmark-speedometer3-mobile: by-test-platform: android-hw-a51.*shippable-qr.*: [trunk-only] default: [] - cstm-car-m: + cstm-car-m: [mozilla-central] + chrome-m: by-test-platform: - android-hw-p6.*shippable-qr.*: [] - android-hw-s21.*shippable-qr.*: [] - default: [mozilla-central] + android-hw-p6.*shippable-qr.*: [mozilla-central] + android-hw-s21.*shippable-qr.*: [mozilla-central] + default: [] default: [] treeherder-symbol: Btime(sp3) mozharness: @@ -409,6 +412,24 @@ browsertime-benchmark-motionmark: treeherder-symbol: Btime() tier: 2 +browsertime-benchmark-motionmark-1-3: + description: Raptor (browsertime) Motionmark 1.3 benchmark tests + run-without-variant: false + raptor: + apps: [geckoview, fenix, chrome-m, cstm-car-m] + subtests: + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] + run-on-projects: + by-app: + chrome-m: [] + default: + by-test-platform: + android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central] + default: [] + treeherder-symbol: Btime() + tier: 2 + browsertime-benchmark-unity-webgl-mobile: description: Browsertime Unity WebGL on Android run-without-variant: false diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml index b31860a165..ba6fd7aec8 100644 --- a/taskcluster/ci/test/misc.yml +++ b/taskcluster/ci/test/misc.yml @@ -8,7 +8,7 @@ job-defaults: geckoview-junit: description: "Geckoview junit run" suite: geckoview-junit - variants: ["geckoview-fission", "no-fission"] + variants: ["geckoview-fission", "no-fission", "geckoview-no-fission-ship"] treeherder-symbol: gv-junit loopback-video: true target: geckoview-androidTest.apk @@ -17,6 +17,7 @@ geckoview-junit: by-variant: geckoview-fission: ['trunk'] no-fission: built-projects + geckoview-no-fission-ship: built-projects default: [] tier: default mozharness: diff --git a/taskcluster/ci/test/mochitest.yml b/taskcluster/ci/test/mochitest.yml index f2399d7116..afe9530364 100644 --- a/taskcluster/ci/test/mochitest.yml +++ b/taskcluster/ci/test/mochitest.yml @@ -783,8 +783,8 @@ mochitest-media: default: [] media-gpu: by-test-platform: - .*hw-ref.*: [] - (macosx.*64|windows11.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central', 'mozilla-beta'] + windows11.*hw-ref.*: ['autoland', 'mozilla-central', 'mozilla-beta'] + (macosx.*64)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central', 'mozilla-beta'] default: [] # TODO : add Android, it hasn't supported 'tags' # TODO : add Linux, it would crash on gfxFeature.cpp:180 diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml index f622eef18e..28da3b47f4 100644 --- a/taskcluster/ci/test/talos.yml +++ b/taskcluster/ci/test/talos.yml @@ -677,6 +677,8 @@ talos-webgl: webgl-ipc: by-test-platform: (linux1804|windows10|macosx1015)-64.*: [trunk] + windows11-64-2009-hw-ref-shippable.*: [trunk] + windows11(?!.*shippable).*: [] default: [] webrender-sw: by-test-platform: @@ -685,14 +687,14 @@ talos-webgl: default: by-test-platform: windows.*-32.*: [] - (linux|windows10-64|macos)(?!.*shippable).*: [] - (linux|windows10-64|macos)(?!.*-qr).*: [] + (linux|windows10-64|windows11|macos)(?!.*shippable).*: [] + (linux|windows10-64|windows11|macos)(?!.*-qr).*: [] default: [trunk] max-run-time: 2100 mozharness: extra-options: by-test-platform: - windows10-64-ref-hw.*: + windows(10-64-ref-hw|11-64-2009-hw-ref).*: - --suite=webgl - --skip-preflight default: diff --git a/taskcluster/ci/test/test-platforms.yml b/taskcluster/ci/test/test-platforms.yml index 777652f191..0f124e0878 100644 --- a/taskcluster/ci/test/test-platforms.yml +++ b/taskcluster/ci/test/test-platforms.yml @@ -203,6 +203,7 @@ windows11-64-2009-hw-ref-shippable/opt: msix: repackage-signing-msix-win64/opt default: build-signing-win64/opt test-sets: + - talos-ref-hw - windows-ref-hw-tests windows11-64-2009-hw-ref/opt: @@ -212,6 +213,7 @@ windows11-64-2009-hw-ref/opt: msix: repackage-signing-msix-win64/opt default: build-signing-win64/opt test-sets: + - talos-ref-hw - windows-ref-hw-tests windows10-64-shippable-qr/opt: diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 90ef2145fe..16807b5b9e 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -490,6 +490,7 @@ android-hw-browsertime: - browsertime-youtube-playback-mobile - browsertime-benchmark-unity-webgl-mobile - browsertime-benchmark-motionmark + - browsertime-benchmark-motionmark-1-3 android-hw-browsertime-profiling: - browsertime-tp6m-profiling diff --git a/taskcluster/ci/test/variants.yml b/taskcluster/ci/test/variants.yml index 1bfe00d001..c57a332a91 100644 --- a/taskcluster/ci/test/variants.yml +++ b/taskcluster/ci/test/variants.yml @@ -19,6 +19,15 @@ headless: mozinfo: "headless" replace: tier: 2 + when: + $eval: '( + "linux" in task["test-platform"] + && ( + "mochitest-plain" in task["try-name"] + || "marionette" in task["try-name"] + ) + ) + || "web-platform-tests-wdspec" == task["try-name"]' merge: mozharness: extra-options: @@ -27,11 +36,24 @@ headless: a11y-checks: description: "{description} with accessibility checks enabled" component: "Core::Disability Access APIs" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "a11y-checks" mozinfo: "a11y_checks" replace: tier: 2 + when: + $eval: '( + "linux" in task["test-platform"] + && "opt" in task["test-platform"] + && !( + "asan" in task["test-platform"] + || "tsan" in task["test-platform"] + || "ccov" in task["test-platform"] + ) + ) && ( + "mochitest-devtools-chrome" == task["try-name"] + || "mochitest-browser-chrome" == task["try-name"] + )' merge: mozharness: extra-options: @@ -40,7 +62,7 @@ a11y-checks: aab: description: "{description} with aab test_runner" component: "GeckoView::General" - expiration: "2024-03-01" + expiration: "2024-09-01" when: &gv_e10s_filter $eval: '"android" in task["test-platform"]' suffix: "aab" @@ -61,7 +83,7 @@ aab: geckoview-no-fission: description: "{description} without fission enabled" component: "GeckoView::Sandboxing" - expiration: "2024-03-01" + expiration: "2024-09-01" when: <<: *gv_e10s_filter suffix: "nofis" @@ -74,7 +96,7 @@ geckoview-no-fission: geckoview-fission: description: "{description} with fission enabled and default isolation strategy" component: "GeckoView::Sandboxing" - expiration: "2024-03-01" + expiration: "2024-09-01" when: <<: *gv_e10s_filter suffix: "fis" @@ -83,7 +105,7 @@ geckoview-fission: geckoview-fission-isolate-high-value: description: "{description} with fission enabled and isolation strategy isolateHighValue" component: "GeckoView::Sandboxing" - expiration: "2024-03-01" + expiration: "2024-09-01" when: <<: *gv_e10s_filter suffix: "fis-hv" @@ -96,13 +118,31 @@ geckoview-fission-isolate-high-value: no-fission: description: "{description} without fission enabled" component: "Core::DOM: Content Processes" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "nofis" mozinfo: "fission" + when: + $eval: ' + "linux" in task["test-platform"] + || "android" in task["test-platform"] + ' + merge: + mozharness: + extra-options: + - "--disable-fission" + +geckoview-no-fission-ship: + description: "{description} with fission disabled and ship enabled" + component: "GeckoView::Sandboxing" + expiration: "2024-09-01" + suffix: "nofis-ship" + when: + <<: *gv_e10s_filter merge: mozharness: extra-options: - "--disable-fission" + - "--setpref=fission.disableSessionHistoryInParent=false" xorigin: description: "{description} with cross-origin" @@ -112,6 +152,8 @@ xorigin: mozinfo: "xorigin" replace: e10s: true + when: + $eval: '"mochitest-plain" in task["try-name"]' merge: mozharness: extra-options: @@ -121,9 +163,25 @@ xorigin: conditioned_profile: description: "{description} with conditioned profile" component: "Testing::General" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "condprof" mozinfo: "condprof" + when: + $eval: ' + ( + "linux" in task["test-platform"] + || "windows" in task["test-platform"] + ) && ( + "opt" in task["test-platform"] + && !( + "asan" in task["test-platform"] + || "tsan" in task["test-platform"] + || "ccov" in task["test-platform"] + ) + ) && ( + "mochitest-plain" == task["try-name"] + || "xpcshell" == task["try-name"] + )' merge: mozharness: extra-options: @@ -137,6 +195,8 @@ socketprocess: expiration: "2024-09-01" suffix: "spi" mozinfo: "socketprocess_e10s" + when: + $eval: '"mochitest-media" in task["try-name"]' merge: mozharness: extra-options: @@ -149,6 +209,16 @@ socketprocess_networking: expiration: "2024-09-01" suffix: "spi-nw" mozinfo: "socketprocess_networking" + when: + $eval: ' + ( + "linux" in task["test-platform"] + || "windows" in task["test-platform"] + || "macosx" in task["test-platform"] + ) && ( + "mochitest" in task["try-name"] + || "xpcshell" == task["try-name"] + )' merge: mozharness: extra-options: @@ -161,7 +231,7 @@ socketprocess_networking: webrender-dcomp-disabled: description: "{description} with dcomp disabled" component: "Core::Graphics: WebRender" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "wr-dc0" when: $eval: '"windows1" in task["test-platform"]' @@ -177,7 +247,7 @@ webrender-dcomp-disabled: webrender-dcomp-present: # Default variant ends up like this. description: "{description} with dcomp present but not overlays" component: "Core::Graphics: WebRender" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "wr-dc1-p" when: $eval: '"windows1" in task["test-platform"]' @@ -193,7 +263,7 @@ webrender-dcomp-present: # Default variant ends up like this. webrender-dcomp-overlays: description: "{description} with dcomp overlays but not compositing" component: "Core::Graphics: WebRender" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "wr-dc2-o" when: $eval: '"windows1" in task["test-platform"]' @@ -209,7 +279,7 @@ webrender-dcomp-overlays: webrender-dcomp-compositing: description: "{description} with dcomp compositing" component: "Core::Graphics: WebRender" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "wr-dc3-c" when: $eval: '"windows1" in task["test-platform"]' @@ -227,7 +297,7 @@ webrender-dcomp-compositing: webrender-sw: description: "{description} with software webrender enabled" component: "Core::Graphics: WebRender" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "swr" mozinfo: "swgl" merge: @@ -239,7 +309,7 @@ webrender-sw: webgl-ipc: description: "{description} with WebGL IPC process enabled" component: "Core::Graphics: CanvasWebGL" - expiration: "2024-03-01" + expiration: "2024-09-01" suffix: "gli" merge: mozharness: @@ -252,6 +322,17 @@ devtools-no-eft: component: "DevTools::Framework" expiration: "2024-03-01" suffix: "dt-no-eft" + when: + $eval: ' + "linux" in task["test-platform"] + && "opt" in task["test-platform"] + && "mochitest-devtools-chrome" == task["try-name"] + && !( + "asan" in task["test-platform"] + || "tsan" in task["test-platform"] + || "ccov" in task["test-platform"] + ) + ' merge: mozharness: extra-options: @@ -260,7 +341,7 @@ devtools-no-eft: wmf-media-engine: description: "{description} with Windows Media Foundation media engine enabled" component: "Core::Audio/Video: Playback" - expiration: "2024-04-10" + expiration: "2024-10-10" when: $eval: '"windows1" in task["test-platform"]' suffix: "wmfme" @@ -274,6 +355,7 @@ wmf-media-engine: - "--setpref=media.wmf.media-engine.enabled=1" - "--setpref=media.wmf.media-engine.channel-decoder.enabled=true" - "--setpref=media.eme.wmf.clearkey.enabled=true" + - "--setpref=media.wmf.media-engine.bypass-gfx-blocklist=true" media-gpu: description: "{description} for running media tests on the GPU worker" @@ -281,6 +363,11 @@ media-gpu: expiration: "2024-04-21" suffix: "mda-gpu" mozinfo: "mda_gpu" + when: + $eval: ' + "windows" in task["test-platform"] + || "macosx" in task["test-platform"] + ' merge: virtualization: virtual-with-gpu mozharness: @@ -295,7 +382,17 @@ msix: suffix: "msix" mozinfo: "msix" when: - $eval: '"windows1" in task["test-platform"] && "-64" in task["test-platform"] && ("opt" in task["test-platform"] || "debug" in task["test-platform"]) && !("asan" in task["test-platform"]) && !("ccov" in task["test-platform"])' + $eval: ' + "windows1" in task["test-platform"] + && "-64" in task["test-platform"] + && ( + "opt" in task["test-platform"] + || "debug" in task["test-platform"] + ) && !( + "asan" in task["test-platform"] + || "ccov" in task["test-platform"] + ) + ' merge: mozharness: extra-options: @@ -307,6 +404,14 @@ http3: expiration: "2024-09-01" suffix: "http3" mozinfo: "http3" + when: + $eval: '( + "linux" in task["test-platform"] + && "debug" in task["test-platform"] + ) && ( + "mochitest-plain" == task["try-name"] + || "mochitest-devtools-chrome" == task["try-name"] + )' merge: mozharness: extra-options: @@ -318,6 +423,13 @@ http2: expiration: "2024-09-01" suffix: "http2" mozinfo: "http2" + when: + $eval: '( + "linux" in task["test-platform"] + && "debug" in task["test-platform"] + ) + && "mochitest-plain" == task["try-name"] + ' merge: mozharness: extra-options: diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml index 5f850d16c1..4ef9721fe6 100644 --- a/taskcluster/ci/test/web-platform.yml +++ b/taskcluster/ci/test/web-platform.yml @@ -378,9 +378,9 @@ web-platform-tests-wdspec: treeherder-symbol: W(Wd) chunks: by-test-platform: - .*-ccov.*/.*: 4 + .*-ccov.*/.*: 6 .*-tsan.*/.*: 8 - default: 3 + default: 5 mozharness: extra-options: by-test-platform: diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml index ed8cef3dc1..3c76f7cad4 100644 --- a/taskcluster/ci/toolchain/android.yml +++ b/taskcluster/ci/toolchain/android.yml @@ -140,7 +140,7 @@ linux64-jdk-repack: toolchain-artifact: project/gecko/jdk/jdk-linux.tar.zst toolchain-alias: linux64-jdk toolchain-env: - JAVA_HOME: "/builds/worker/fetches/jdk/jdk-17.0.9+9" + JAVA_HOME: "/builds/worker/fetches/jdk/jdk-17.0.10+7" linux64-android-sdk-linux-repack: description: "Android SDK (Linux) repack toolchain build" diff --git a/taskcluster/ci/toolchain/clang-tidy.yml b/taskcluster/ci/toolchain/clang-tidy.yml index 8d357e8c82..02d089eca3 100644 --- a/taskcluster/ci/toolchain/clang-tidy.yml +++ b/taskcluster/ci/toolchain/clang-tidy.yml @@ -103,6 +103,7 @@ win64-clang-tidy: toolchain: - linux64-clang-17 - vs-toolchain + - win64-libxml2 linux64-clang-tidy-external: attributes: diff --git a/taskcluster/ci/toolchain/clang.yml b/taskcluster/ci/toolchain/clang.yml index 1223afd4e5..a9cf04e0b9 100644 --- a/taskcluster/ci/toolchain/clang.yml +++ b/taskcluster/ci/toolchain/clang.yml @@ -437,6 +437,7 @@ win64-clang-17-stage1: toolchain: - linux64-clang-17-stage1 - vs-toolchain + - win64-libxml2 win64-clang-17-raw: description: "Clang-cl 17 toolchain build" @@ -467,6 +468,7 @@ win64-clang-17-raw: - win64-clang-17-stage1 - win64-compiler-rt-17 - vs-toolchain + - win64-libxml2 win64-clang-17: description: "Clang-cl 17 toolchain build" @@ -801,6 +803,7 @@ win64-clang-18-stage1: toolchain: - linux64-clang-18-stage1 - vs-toolchain + - win64-libxml2 win64-clang-18-raw: description: "Clang-cl 18 toolchain build" @@ -831,6 +834,7 @@ win64-clang-18-raw: - win64-clang-18-stage1 - win64-compiler-rt-18 - vs-toolchain + - win64-libxml2 win64-clang-18: description: "Clang-cl 18 toolchain build" @@ -1207,6 +1211,7 @@ win64-clang-trunk-stage1: toolchain: - linux64-clang-trunk-stage1 - vs-toolchain + - win64-libxml2 win64-clang-trunk-raw: description: "Clang-cl trunk toolchain build" @@ -1239,6 +1244,7 @@ win64-clang-trunk-raw: - win64-clang-trunk-stage1 - win64-compiler-rt-trunk - vs-toolchain + - win64-libxml2 win64-clang-trunk: description: "Clang-cl trunk toolchain build" diff --git a/taskcluster/ci/toolchain/macos-sdk.yml b/taskcluster/ci/toolchain/macos-sdk.yml index d3e2bc6493..60e4c231fd 100644 --- a/taskcluster/ci/toolchain/macos-sdk.yml +++ b/taskcluster/ci/toolchain/macos-sdk.yml @@ -46,3 +46,39 @@ macosx64-sdk-14.2: - macosx64-sdk - macosx64-sdk-toolchain - MacOSX14.2.sdk + +ios-sdk-17.4: + description: "iPhoneOS17.4 SDK" + treeherder: + symbol: TM(ios17.4) + # Because it's using an internal tooltool artifact, it can't be used as a local-toolchain for + # bootstrap. But we still want to use the same script as local-toolchains. + attributes: + local-toolchain: false + run: + script: private_local_toolchain.sh + arguments: + - unpack-sdk.py + - http://taskcluster/tooltool.mozilla-releng.net/sha512/7d9f1f1e0d4963fbafc8ec20ee604665e99d66699db2515f388b3b061aaab83349815e1ef862922f4f46ffbe1ac2d86fb00701802c7a9994c74f6e1d32894c34 + - 7d9f1f1e0d4963fbafc8ec20ee604665e99d66699db2515f388b3b061aaab83349815e1ef862922f4f46ffbe1ac2d86fb00701802c7a9994c74f6e1d32894c34 + - Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk + toolchain-artifact: project/gecko/mac-sdk/iPhoneOS17.4.sdk.tar.zst + toolchain-alias: ios-sdk + +ios-sim-sdk-17.4: + description: "iPhoneSimulator17.4 SDK" + treeherder: + symbol: TM(ios-sim-17.4) + # Because it's using an internal tooltool artifact, it can't be used as a local-toolchain for + # bootstrap. But we still want to use the same script as local-toolchains. + attributes: + local-toolchain: false + run: + script: private_local_toolchain.sh + arguments: + - unpack-sdk.py + - http://taskcluster/tooltool.mozilla-releng.net/sha512/7d9f1f1e0d4963fbafc8ec20ee604665e99d66699db2515f388b3b061aaab83349815e1ef862922f4f46ffbe1ac2d86fb00701802c7a9994c74f6e1d32894c34 + - 7d9f1f1e0d4963fbafc8ec20ee604665e99d66699db2515f388b3b061aaab83349815e1ef862922f4f46ffbe1ac2d86fb00701802c7a9994c74f6e1d32894c34 + - Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk + toolchain-artifact: project/gecko/mac-sdk/iPhoneSimulator17.4.sdk.tar.zst + toolchain-alias: ios-sim-sdk diff --git a/taskcluster/ci/toolchain/misc.yml b/taskcluster/ci/toolchain/misc.yml index f19201bc1e..4424cf6e0b 100644 --- a/taskcluster/ci/toolchain/misc.yml +++ b/taskcluster/ci/toolchain/misc.yml @@ -387,7 +387,7 @@ browsertime: toolchain-artifact: public/build/browsertime.tar.zst fetches: toolchain: - - linux64-node-16 + - linux64-node wrench-deps: description: "Downloads all the crates needed for building wrench" @@ -524,9 +524,6 @@ win64-vs2019: resources: - build/vs/vs2019.yaml toolchain-artifact: project/gecko/vs/vs.tar.zst - toolchain-alias: - - vs - - vs-toolchain win64-vs2022: description: "VS 2022" @@ -544,6 +541,9 @@ win64-vs2022: resources: - build/vs/vs2022.yaml toolchain-artifact: project/gecko/vs/vs.tar.zst + toolchain-alias: + - vs + - vs-toolchain linux64-afl-instrumentation-4.0: description: "AFL++ instrumentation toolchain build" @@ -603,3 +603,17 @@ windows-rs: - artifact: windows-rs.tar.zst extract: false dest: ../artifacts + +win64-libxml2: + description: "libxml2 toolchain build" + treeherder: + symbol: TW64(libxml2) + run: + script: build-libxml2.sh + toolchain-artifact: public/build/libxml2.tar.zst + fetches: + fetch: + - libxml2 + toolchain: + - linux64-clang-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/node.yml b/taskcluster/ci/toolchain/node.yml index c1bc86254d..6ab3511aab 100644 --- a/taskcluster/ci/toolchain/node.yml +++ b/taskcluster/ci/toolchain/node.yml @@ -20,16 +20,6 @@ linux64-node-12: fetch: - nodejs-12-linux64 -linux64-node-16: - treeherder: - symbol: TL(node-16) - run: - toolchain-alias: - - linux64-node - fetches: - fetch: - - nodejs-16-linux64 - linux64-node-18: # The upstream prebuilt binary requires glibc 2.28, which is too recent for Ubuntu 18.04, # so we build our own that won't have that requirement. @@ -38,6 +28,8 @@ linux64-node-18: symbol: TL(node-18) run: script: build-nodejs.sh + toolchain-alias: + - linux64-node fetches: fetch: - nodejs-18-source @@ -45,70 +37,43 @@ linux64-node-18: - linux64-gcc-11 - linux64-toolchain-sysroot -macosx64-node-16: +macosx64-node-18: treeherder: - symbol: TM(node-16) + symbol: TM(node-18) run: toolchain-alias: - macosx64-node - fetches: - fetch: - - nodejs-16-macosx64 -macosx64-node-18: - treeherder: - symbol: TM(node-18) fetches: fetch: - nodejs-18-macosx64 -macosx64-aarch64-node-16: +macosx64-aarch64-node-18: treeherder: - symbol: TM(node-16-arm64) + symbol: TM(node-18-arm64) run: toolchain-alias: - macosx64-aarch64-node fetches: fetch: - - nodejs-16-macosx64-arm64 - -macosx64-aarch64-node-18: - treeherder: - symbol: TM(node-18-arm64) - fetches: - fetch: - nodejs-18-macosx64-arm64 -win64-node-16: +win64-node-18: treeherder: - symbol: TW64(node-16) + symbol: TW64(node-18) run: toolchain-alias: - win64-node fetches: fetch: - - nodejs-16-win64 - -win64-node-18: - treeherder: - symbol: TW64(node-18) - fetches: - fetch: - nodejs-18-win64 -win32-node-16: +win32-node-18: treeherder: - symbol: TW32(node-16) + symbol: TW32(node-18) run: toolchain-alias: - win32-node fetches: fetch: - - nodejs-16-win32 - -win32-node-18: - treeherder: - symbol: TW32(node-18) - fetches: - fetch: - nodejs-18-win32 diff --git a/taskcluster/ci/toolchain/rust.yml b/taskcluster/ci/toolchain/rust.yml index 050e581247..62f4fd4747 100644 --- a/taskcluster/ci/toolchain/rust.yml +++ b/taskcluster/ci/toolchain/rust.yml @@ -11,12 +11,12 @@ job-defaults: script: repack_rust.py toolchain-artifact: public/build/rustc.tar.zst -linux64-rust-1.70: +linux64-rust-1.74: treeherder: - symbol: TL(rust-1.70) + symbol: TL(rust-1.74) run: arguments: [ - '--channel', '1.70.0', + '--channel', '1.74.0', '--host', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-unknown-linux-gnu', '--target', 'i686-unknown-linux-gnu', @@ -134,6 +134,20 @@ linux64-rust-macos-1.76: ] toolchain-alias: linux64-rust-macos +linux64-rust-ios-1.76: + description: "rust repack with ios-cross support" + treeherder: + symbol: TL(rust-ios) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'aarch64-apple-ios', + '--target', 'aarch64-apple-ios-sim', + ] + toolchain-alias: linux64-rust-ios + linux64-rust-android-1.76: description: "rust repack with android-cross support" treeherder: diff --git a/taskcluster/ci/webrender/kind.yml b/taskcluster/ci/webrender/kind.yml index 84a938cf67..099fc2bc7d 100644 --- a/taskcluster/ci/webrender/kind.yml +++ b/taskcluster/ci/webrender/kind.yml @@ -237,7 +237,7 @@ jobs: symbol: WR(wrench) when: files-changed: - - 'build/win64/mozconfig.vs2019' + - 'build/win64/mozconfig.vs2022' - 'gfx/wr/**' - 'taskcluster/scripts/misc/wrench-windows-tests.sh' diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py index 84fd2fbd0b..0e72da46fc 100644 --- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -13,7 +13,6 @@ import shutil import tempfile import time from contextlib import AsyncExitStack -from distutils.util import strtobool from pathlib import Path import aiohttp @@ -51,6 +50,22 @@ BCJ_OPTIONS = { } +def strtobool(value: str): + # Copied from `mach.util` since this script runs outside of a mach environment + # Reimplementation of distutils.util.strtobool + # https://docs.python.org/3.9/distutils/apiref.html#distutils.util.strtobool + true_vals = ("y", "yes", "t", "true", "on", "1") + false_vals = ("n", "no", "f", "false", "off", "0") + + value = value.lower() + if value in true_vals: + return 1 + if value in false_vals: + return 0 + + raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') + + def verify_signature(mar, cert): log.info("Checking %s signature", mar) with open(mar, "rb") as mar_fh: diff --git a/taskcluster/docker/periodic-updates/.eslintrc.js b/taskcluster/docker/periodic-updates/.eslintrc.js index ecaf80d033..508b1cfbce 100644 --- a/taskcluster/docker/periodic-updates/.eslintrc.js +++ b/taskcluster/docker/periodic-updates/.eslintrc.js @@ -14,9 +14,6 @@ module.exports = { // Enforce return statements in callbacks of array methods. "array-callback-return": "error", - // Verify calls of super() in constructors. - "constructor-super": "error", - // Require default case in switch statements. "default-case": "error", @@ -27,9 +24,6 @@ module.exports = { // case/default clauses. "no-case-declarations": "error", - // Disallow use of the console API. - "no-console": "error", - // Disallow constant expressions in conditions (except for loops). "no-constant-condition": ["error", { checkLoops: false }], @@ -48,9 +42,6 @@ module.exports = { // No expressions where a statement is expected "no-unused-expressions": "error", - // Disallow unnecessary escape usage in strings and regular expressions. - "no-useless-escape": "error", - // Require "use strict" to be defined globally in the script. strict: ["error", "global"], diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh index c5b9c78f6f..f01f5cd9c3 100755 --- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh +++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh @@ -322,7 +322,7 @@ function compare_remote_settings_files { # 5. Download attachments if needed. if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then - # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm. + # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.sys.mjs. # The server should return one "bloomfilter-base" record, but in case it returns multiple, # return the most recent one. The server may send multiple entries if we ever decide to use # the "filter_expression" feature of Remote Settings to send different records to specific @@ -364,7 +364,7 @@ function update_remote_settings_attachment() { # $4 is a jq filter on the arrays that should return one record with the attachment local jq_attachment_selector=".data | map(select(.attachment)) | $4" - # These paths match _readAttachmentDump in services/settings/Attachments.jsm. + # These paths match _readAttachmentDump in services/settings/Attachments.sys.mjs. local path_to_attachment="${bucket}/${collection}/${attachment_id}" local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json" local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}" diff --git a/taskcluster/docker/snap-coreXX-build/run.sh b/taskcluster/docker/snap-coreXX-build/run.sh index 5b3efbb756..234fc98519 100755 --- a/taskcluster/docker/snap-coreXX-build/run.sh +++ b/taskcluster/docker/snap-coreXX-build/run.sh @@ -19,7 +19,7 @@ unset MOZ_AUTOMATION MOZCONFIG=mozconfig.in -USE_SNAP_FROM_STORE=${USE_SNAP_FROM_STORE:-0} +USE_SNAP_FROM_STORE_OR_MC=${USE_SNAP_FROM_STORE_OR_MC:-0} TRY=0 if [ "${BRANCH}" = "try" ]; then @@ -27,7 +27,7 @@ if [ "${BRANCH}" = "try" ]; then TRY=1 fi -if [ "${USE_SNAP_FROM_STORE}" = "0" ]; then +if [ "${USE_SNAP_FROM_STORE_OR_MC}" = "0" ]; then # ESR currently still has a hard dependency against zstandard==0.17.0 so # install this specific version here if [ "${BRANCH}" = "esr" ]; then @@ -91,7 +91,7 @@ if [ "${USE_SNAP_FROM_STORE}" = "0" ]; then SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \ CRAFT_PARTS_PACKAGE_REFRESH=0 \ snapcraft --destructive-mode --verbose -else +elif [ "${USE_SNAP_FROM_STORE_OR_MC}" = "store" ]; then mkdir from-snap-store && cd from-snap-store CHANNEL="${BRANCH}" @@ -102,6 +102,45 @@ else snap download --channel="${CHANNEL}" firefox SNAP_DEBUG_NAME=$(find . -maxdepth 1 -type f -name "firefox*.snap" | sed -e 's/\.snap$/.debug/g') touch "${SNAP_DEBUG_NAME}" +else + mkdir from-mc && cd from-mc + + # index.gecko.v2.mozilla-central.latest.firefox.amd64-esr-debug + # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.amd64-esr-debug/artifacts/public%2Fbuild%2Ffirefox.snap + # index.gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.amd64-esr-debug + # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.amd64-esr-debug/artifacts/public%2Fbuild%2Ffirefox.snap + # index.gecko.v2.mozilla-central.latest.firefox.amd64-nightly + # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.amd64-nightly/artifacts/public%2Fbuild%2Ffirefox.snap + # index.gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.amd64-nightly + # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.amd64-nightly/artifacts/public%2Fbuild%2Ffirefox.snap + + INDEX_NAME="${BRANCH}" + if [ "${INDEX_NAME}" = "try" ]; then + INDEX_NAME=nightly + fi; + + if [ "${DEBUG}" = "1" ]; then + INDEX_NAME="${INDEX_NAME}-debug" + fi; + + TASKCLUSTER_API_ROOT="https://firefox-ci-tc.services.mozilla.com/api" + + URL_TASK="${TASKCLUSTER_API_ROOT}/index/v1/task/gecko.v2.mozilla-central.${USE_SNAP_FROM_STORE_OR_MC}.firefox.amd64-${INDEX_NAME}" + PKGS_TASK_ID=$(curl "${URL_TASK}" | jq -r '.taskId') + + if [ -z "${PKGS_TASK_ID}" ]; then + echo "Failure to find matching taskId for ${USE_SNAP_FROM_STORE_OR_MC} + ${INDEX_NAME}" + exit 1 + fi + + PKGS_URL="${TASKCLUSTER_API_ROOT}/queue/v1/task/${PKGS_TASK_ID}/artifacts" + for pkg in $(curl "${PKGS_URL}" | jq -r '.artifacts | . [] | select(.name | contains("public/build/firefox_")) | .name'); + do + url="${TASKCLUSTER_API_ROOT}/queue/v1/task/${PKGS_TASK_ID}/artifacts/${pkg}" + target_name="$(basename "${pkg}")" + echo "$url => $target_name" + curl -SL "${url}" -o "${target_name}" + done; fi cp ./*.snap ./*.debug /builds/worker/artifacts/ diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py index fb8841817b..87320d5638 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py @@ -13,12 +13,13 @@ import time import traceback from mozlog import formatters, handlers, structuredlog -from PIL import Image, ImageChops +from PIL import Image, ImageChops, ImageDraw from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.service import Service +from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait @@ -128,6 +129,9 @@ class SnapTestsBase: else: return 5 + def is_debug_build(self): + return "BUILD_IS_DEBUG" in os.environ.keys() + def maybe_collect_reference(self): return "TEST_COLLECT_REFERENCE" in os.environ.keys() @@ -149,9 +153,11 @@ class SnapTestsBase: element_or_driver.screenshot_as_png if isinstance(element_or_driver, WebElement) else element_or_driver.get_screenshot_as_png() + if isinstance(element_or_driver, WebDriver) + else base64.b64decode(element_or_driver) ) svg_png = Image.open(io.BytesIO(png_bytes)).convert("RGB") - svg_png_cropped = svg_png.crop((0, 0, svg_png.width - 20, svg_png.height - 20)) + svg_png_cropped = svg_png.crop((0, 35, svg_png.width - 20, svg_png.height - 10)) if self.maybe_collect_reference(): new_ref = "new_{}".format(exp["reference"]) @@ -169,8 +175,28 @@ class SnapTestsBase: svg_ref = Image.open(os.path.join(self._dir, exp["reference"])).convert("RGB") diff = ImageChops.difference(svg_ref, svg_png_cropped) + bbox = diff.getbbox() + + if bbox is not None: + (diff_r, diff_g, diff_b) = diff.getextrema() + + min_is_black = (diff_r[0] == 0) and (diff_g[0] == 0) and (diff_b[0] == 0) + max_is_low_enough = ( + (diff_r[1] <= 15) and (diff_g[1] <= 15) and (diff_b[1] <= 15) + ) + + draw_ref = ImageDraw.Draw(svg_ref) + draw_ref.rectangle(bbox, outline="red") + + draw_rend = ImageDraw.Draw(svg_png_cropped) + draw_rend.rectangle(bbox, outline="red") + + draw_diff = ImageDraw.Draw(diff) + draw_diff.rectangle(bbox, outline="red") + + # Some differences have been found, let's verify + self._logger.info("Non empty differences bbox: {}".format(bbox)) - if diff.getbbox() is not None: buffered = io.BytesIO() diff.save(buffered, format="PNG") @@ -190,8 +216,18 @@ class SnapTestsBase: ) as current_screenshot: svg_png_cropped.save(current_screenshot) - assert diff.getbbox() is None, "Mismatching screenshots for {}".format( - exp["reference"] + with open( + self.get_screenshot_destination("reference_rendering.png"), "wb" + ) as current_screenshot: + svg_ref.save(current_screenshot) + + # Assume a difference is mismatching if the minimum pixel value in + # image difference is NOT black or if the maximum pixel value is + # not low enough + assert ( + min_is_black and max_is_low_enough + ), "Mismatching screenshots for {} with extremas: {}".format( + exp["reference"], (diff_r, diff_g, diff_b) ) @@ -255,7 +291,7 @@ class SnapTests(SnapTestsBase): return True def test_youtube(self, exp): - self.open_tab("https://www.youtube.com") + self.open_tab("https://www.youtube.com/channel/UCYfdidRxbB8Qhf0Nx7ioOYw") # Wait for the consent dialog and accept it self._logger.info("Wait for consent form") @@ -268,25 +304,86 @@ class SnapTests(SnapTestsBase): except TimeoutException: self._logger.info("Wait for consent form: timed out, maybe it is not here") - try: - # Find first video and click it - self._logger.info("Wait for one video") + # Find first video and click it + self._logger.info("Wait for one video") + self._wait.until( + EC.visibility_of_element_located((By.ID, "video-title-link")) + ).click() + + # Wait for duration to be set to something + self._logger.info("Wait for video to start") + video = self._wait.until( + EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video")) + ) + self._wait.until(lambda d: type(video.get_property("duration")) == float) + self._logger.info("video duration: {}".format(video.get_property("duration"))) + assert ( + video.get_property("duration") > exp["duration"] + ), "youtube video should have duration" + + self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) + self._logger.info("video played: {}".format(video.get_property("currentTime"))) + assert ( + video.get_property("currentTime") > exp["playback"] + ), "youtube video should perform playback" + + return True + + def wait_for_enable_drm(self): + rv = True + self._driver.set_context("chrome") + self._driver.execute_script( + "Services.prefs.setBoolPref('media.gmp-manager.updateEnabled', true);" + ) + + channel = self._driver.execute_script( + "return Services.prefs.getCharPref('app.update.channel');" + ) + if channel == "esr": + rv = False + else: + enable_drm_button = self._wait.until( + EC.visibility_of_element_located( + (By.CSS_SELECTOR, ".notification-button[label='Enable DRM']") + ) + ) + self._logger.info("Enabling DRMs") + enable_drm_button.click() self._wait.until( - EC.visibility_of_element_located((By.ID, "video-title-link")) - ).click() - except TimeoutException: - # We might have got the "try searching to get started" - # link to News channel - self._driver.get("https://www.youtube.com/channel/UCYfdidRxbB8Qhf0Nx7ioOYw") - self._logger.info("Wait again for one video") + EC.invisibility_of_element_located( + (By.CSS_SELECTOR, ".notification-button[label='Enable DRM']") + ) + ) + + self._logger.info("Installing DRMs") self._wait.until( - EC.visibility_of_element_located((By.ID, "video-title-link")) - ).click() + EC.visibility_of_element_located( + (By.CSS_SELECTOR, ".infobar[value='drmContentCDMInstalling']") + ) + ) + + self._logger.info("Waiting for DRMs installation to complete") + self._longwait.until( + EC.invisibility_of_element_located( + (By.CSS_SELECTOR, ".infobar[value='drmContentCDMInstalling']") + ) + ) + + self._driver.set_context("content") + return rv + + def test_youtube_film(self, exp): + self.open_tab("https://www.youtube.com/watch?v=i4FSx9LXVSE") + if not self.wait_for_enable_drm(): + self._logger.info("Skipped on ESR because cannot enable DRM") + return True # Wait for duration to be set to something self._logger.info("Wait for video to start") video = self._wait.until( - EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video")) + EC.visibility_of_element_located( + (By.CSS_SELECTOR, "video.html5-main-video") + ) ) self._wait.until(lambda d: type(video.get_property("duration")) == float) self._logger.info("video duration: {}".format(video.get_property("duration"))) @@ -294,6 +391,9 @@ class SnapTests(SnapTestsBase): video.get_property("duration") > exp["duration"] ), "youtube video should have duration" + self._driver.execute_script("arguments[0].click();", video) + video.send_keys("k") + self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) self._logger.info("video played: {}".format(video.get_property("currentTime"))) assert ( diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in index 92e26dd699..32cb20c88c 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in @@ -10,5 +10,9 @@ "test_youtube": { "duration": 1, "playback": 2 + }, + "test_youtube_film": { + "duration": 1, + "playback": 2 } } diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py index 781000981e..7788dabe34 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py @@ -10,7 +10,7 @@ import tempfile import time from basic_tests import SnapTestsBase -from selenium.common.exceptions import StaleElementReferenceException, TimeoutException +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys @@ -35,7 +35,9 @@ class QATests(SnapTestsBase): if iframe_selector: self._logger.info("find iframe") - iframe = self._driver.find_element(By.CSS_SELECTOR, iframe_selector) + iframe = self._wait.until( + EC.visibility_of_element_located((By.CSS_SELECTOR, iframe_selector)) + ) self._driver.switch_to.frame(iframe) self._logger.info("find video") @@ -259,6 +261,15 @@ class QATests(SnapTestsBase): self._wait.until(lambda d: pdf_div.is_displayed() is True) return pdf_div + def is_esr(self): + self._driver.set_context("chrome") + update_channel = self._driver.execute_script( + "return Services.prefs.getStringPref('app.update.channel');" + ) + self._logger.info("Update channel: {}".format(update_channel)) + self._driver.set_context("content") + return update_channel == "esr" + def pdf_get_page(self, page, long=False): waiter = self._longwait if long is True else self._wait page = waiter.until( @@ -267,22 +278,26 @@ class QATests(SnapTestsBase): ) ) - try: - self._wait.until(lambda d: page.is_displayed() is True) - except StaleElementReferenceException as ex: - self._logger.info("Stale element but who cares?: {}".format(ex)) - time.sleep(2) - - # self._wait.until( - # lambda d: d.execute_script( - # 'return window.getComputedStyle(document.querySelector(".loadingInput.start"), "::after").getPropertyValue("visibility");' - # ) - # == "hidden" - # ) - - # PDF.js can take time to settle and we don't have a nice way to wait - # for an event on it - time.sleep(2) + if not self.is_esr(): + self._wait.until( + lambda d: d.execute_script( + 'return window.getComputedStyle(document.querySelector(".loadingInput.start"), "::after").getPropertyValue("visibility");' + ) + != "visible" + ) + # PDF.js can take time to settle and we don't have a nice way to wait + # for an event on it + time.sleep(1) + else: + self._logger.info("Running against ESR, just wait too much.") + # Big but let's be safe, this is only for ESR because its PDF.js + # does not have "<span class='loadingInput start'>" + time.sleep(10) + + # Rendering can be slower on debug build so give more time to settle + if self.is_debug_build(): + time.sleep(3) + return page def pdf_go_to_page(self, page): @@ -302,8 +317,8 @@ class QATests(SnapTestsBase): # Test basic rendering self.pdf_wait_div() self.pdf_select_zoom("1") - page_1 = self.pdf_get_page(1) - self.assert_rendering(exp["base"], page_1) + self.pdf_get_page(1) + self.assert_rendering(exp["base"], self._driver) # Navigating to page X, we know the PDF has 5 pages. rand_page = random.randint(1, 5) @@ -396,10 +411,37 @@ class QATests(SnapTestsBase): if menu_id == "pageRotateCw" or menu_id == "pageRotateCcw": secondary_menu.click() - time.sleep(0.2) + time.sleep(0.75) self._logger.info("assert {}".format(menu_id)) - self.assert_rendering(exp[menu_id], self._driver) + if self.is_esr() and menu_id == "documentProperties": + # on ESR pdf.js misreports in mm instead of inches + title = self._wait.until( + EC.visibility_of_element_located((By.ID, "titleField")) + ) + author = self._wait.until( + EC.visibility_of_element_located((By.ID, "authorField")) + ) + subject = self._wait.until( + EC.visibility_of_element_located((By.ID, "subjectField")) + ) + version = self._wait.until( + EC.visibility_of_element_located((By.ID, "versionField")) + ) + assert title.text == "PDF", "Incorrect PDF title reported: {}".format( + title + ) + assert ( + author.text == "Software 995" + ), "Incorrect PDF author reported: {}".format(author) + assert ( + subject.text == "Create PDF with Pdf 995" + ), "Incorrect PDF subject reported: {}".format(subject) + assert ( + version.text == "1.3" + ), "Incorrect PDF version reported: {}".format(version) + else: + self.assert_rendering(exp[menu_id], self._driver) if menu_id == "documentProperties": close = self._wait.until( @@ -423,6 +465,7 @@ class QATests(SnapTestsBase): ) ) action.drag_and_drop_by_offset(paragraph, 50, 10).perform() + time.sleep(0.75) self.assert_rendering(exp["select_text"], self._driver) # release select selection @@ -464,7 +507,7 @@ class QATests(SnapTestsBase): for zoom, page, ref in zoom_levels: self.pdf_select_zoom(zoom) - self.pdf_get_page(page) + self.pdf_get_page(page, long=True) self._logger.info("assert {}".format(ref)) self.assert_rendering(exp[ref], self._driver) @@ -704,17 +747,8 @@ class QATests(SnapTestsBase): download_dir_pref == new ), "download directory from pref should match new directory" - def open_thinkbroadband(self): - download_site = self.open_tab("https://www.thinkbroadband.com/download") - try: - consent = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".t-acceptAllButton") - ) - ) - consent.click() - except TimeoutException: - self._logger.info("Wait for consent form: timed out, maybe it is not here") + def open_lafibre(self): + download_site = self.open_tab("https://ip.lafibre.info/test-debit.php") return download_site def test_download_folder_change(self, exp): @@ -722,12 +756,12 @@ class QATests(SnapTestsBase): C1756713 """ - download_site = self.open_thinkbroadband() + download_site = self.open_lafibre() extra_small = self._wait.until( EC.presence_of_element_located( ( By.CSS_SELECTOR, - "div.module:nth-child(8) > p:nth-child(1) > a:nth-child(1)", + ".tableau > tbody:nth-child(1) > tr:nth-child(6) > td:nth-child(2) > a:nth-child(1)", ) ) ) @@ -775,12 +809,12 @@ class QATests(SnapTestsBase): C1756715 """ - download_site = self.open_thinkbroadband() + download_site = self.open_lafibre() extra_small = self._wait.until( EC.presence_of_element_located( ( By.CSS_SELECTOR, - "div.module:nth-child(8) > p:nth-child(1) > a:nth-child(1)", + ".tableau > tbody:nth-child(1) > tr:nth-child(6) > td:nth-child(2) > a:nth-child(1)", ) ) ) diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png Binary files differindex 02263329e2..03a4cfeb3d 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png Binary files differindex 98bf604c4b..da5824e130 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png Binary files differindex 71046ad3c3..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png Binary files differindex 6eeeb95f2b..f0f4ea15f5 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png Binary files differindex c3818c9726..84fcdd160d 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png Binary files differindex 86bf1f158d..bd18d5e083 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png Binary files differindex 1b33d8374e..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png Binary files differindex d84c9158d9..368dbab60b 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png Binary files differindex 1b33d8374e..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png Binary files differindex 86bf1f158d..bd18d5e083 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png Binary files differindex aa36bda00c..6375855c02 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png Binary files differindex 21e930607c..c43825e7be 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png Binary files differindex 19062697d9..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png Binary files differindex 176ab1dcff..484df8e2f0 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png Binary files differindex db05b7feb2..6523031e0e 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png Binary files differindex 8df2181704..628bd1d25f 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png Binary files differindex 19c727c5e0..f017be89ba 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png Binary files differindex be6d48cc30..22e4d3a33d 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png Binary files differindex 95fa211e1b..0e24a1ac74 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png Binary files differindex cd5067574c..f017be89ba 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png Binary files differindex 1b33d8374e..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png Binary files differindex add7d3a9bb..30602dc1ee 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png Binary files differindex 58f832c3e5..3ebd4191fd 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png Binary files differindex 310bb889be..f860c9d640 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png Binary files differindex 5a21ade0c0..fd136334d2 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png Binary files differindex d047a4fcbf..99e2deee95 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png Binary files differindex b721fd3296..4e1dc3066a 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png Binary files differindex 061eafda7f..c77f23edbd 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png diff --git a/taskcluster/gecko_taskgraph/morph.py b/taskcluster/gecko_taskgraph/morph.py index 1d03ddaab6..42fe4597fa 100644 --- a/taskcluster/gecko_taskgraph/morph.py +++ b/taskcluster/gecko_taskgraph/morph.py @@ -254,10 +254,31 @@ def add_eager_cache_index_tasks(taskgraph, label_to_taskid, parameters, graph_co @register_morph def add_try_task_duplicates(taskgraph, label_to_taskid, parameters, graph_config): - try_config = parameters["try_task_config"] + return _add_try_task_duplicates( + taskgraph, label_to_taskid, parameters, graph_config + ) + + +# this shim function exists so we can call it from the unittests. +# this works around an issue with +# third_party/python/taskcluster_taskgraph/taskgraph/morph.py#40 +def _add_try_task_duplicates(taskgraph, label_to_taskid, parameters, graph_config): + try_config = parameters.get("try_task_config", {}) + tasks = try_config.get("tasks", []) + glob_tasks = {x.strip("-*") for x in tasks if x.endswith("-*")} + tasks = set(tasks) - glob_tasks + rebuild = try_config.get("rebuild") if rebuild: for task in taskgraph.tasks.values(): - if task.label in try_config.get("tasks", []): + chunk_index = -1 + if task.label.endswith("-cf"): + chunk_index = -2 + label_parts = task.label.split("-") + label_no_chunk = "-".join(label_parts[:chunk_index]) + + if label_parts[chunk_index].isnumeric() and label_no_chunk in glob_tasks: + task.attributes["task_duplicates"] = rebuild + elif task.label in tasks: task.attributes["task_duplicates"] = rebuild return taskgraph, label_to_taskid diff --git a/taskcluster/gecko_taskgraph/parameters.py b/taskcluster/gecko_taskgraph/parameters.py index 2a61a71b96..7e3de1372f 100644 --- a/taskcluster/gecko_taskgraph/parameters.py +++ b/taskcluster/gecko_taskgraph/parameters.py @@ -74,6 +74,10 @@ gecko_parameters_schema = { "worker-overrides", description="Mapping of worker alias to worker pools to use for those aliases.", ): {str: str}, + Optional( + "worker-types", + description="List of worker types that we will use to run tasks on.", + ): [str], Optional("routes"): [str], }, Required("version"): str, diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index 2f445d3f95..fcbfab4e17 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -4,6 +4,7 @@ import itertools +import logging import os import re from datetime import datetime, timedelta @@ -21,6 +22,9 @@ from gecko_taskgraph.util.attributes import ( from gecko_taskgraph.util.hg import find_hg_revision_push_info, get_hg_commit_message from gecko_taskgraph.util.platforms import platform_family +logger = logging.getLogger(__name__) + + # Some tasks show up in the target task set, but are possibly special cases, # uncommon tasks, or tasks running against limited hardware set that they # should only be selectable with --full. @@ -247,9 +251,9 @@ def accept_raptor_android_build(platform): if "p5" in platform and "aarch64" in platform: return False if "p6" in platform and "aarch64" in platform: - return False + return True if "s21" in platform and "aarch64" in platform: - return False + return True if "a51" in platform: return True return False @@ -299,16 +303,30 @@ def _try_task_config(full_task_graph, parameters, graph_config): pattern_tasks = [x for x in requested_tasks if x.endswith("-*")] tasks = list(set(requested_tasks) - set(pattern_tasks)) matched_tasks = [] + missing = set() for pattern in pattern_tasks: - matched_tasks.extend( - [ - t - for t in full_task_graph.graph.nodes - if t.split(pattern.replace("*", ""))[-1].isnumeric() - ] - ) + found = [ + t + for t in full_task_graph.graph.nodes + if t.split(pattern.replace("*", ""))[-1].isnumeric() + ] + if found: + matched_tasks.extend(found) + else: + missing.add(pattern) + + if "MOZHARNESS_TEST_PATHS" in parameters["try_task_config"].get("env", {}): + matched_tasks = [x for x in matched_tasks if x.endswith("-1")] - return list(set(tasks) | set(matched_tasks)) + selected_tasks = set(tasks) | set(matched_tasks) + missing.update(selected_tasks - set(full_task_graph.tasks)) + + if missing: + missing_str = "\n ".join(sorted(missing)) + logger.warning( + f"The following tasks were requested but do not exist in the full task graph and will be skipped:\n {missing_str}" + ) + return list(selected_tasks - missing) def _try_option_syntax(full_task_graph, parameters, graph_config): @@ -731,6 +749,7 @@ def target_tasks_larch(full_task_graph, parameters, graph_config): "l10n" in task.kind or "msix" in task.kind or "android" in task.attributes.get("build_platform", "") + or (task.kind == "test" and "msix" in task.label) ): return False # otherwise reduce tests only @@ -800,6 +819,8 @@ def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_conf if "browsertime" in try_name and ( "custom-car" in try_name or "cstm-car-m" in try_name ): + if "hw-s21" in platform and "speedometer3" not in try_name: + return False return True return False @@ -853,6 +874,8 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config) return True # Android selection elif accept_raptor_android_build(platform): + if "hw-s21" in platform and "speedometer3" not in try_name: + return False if "chrome-m" in try_name and ( ("ebay" in try_name and "live" not in try_name) or ( @@ -935,6 +958,8 @@ def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config): platform ): try_name = attributes.get("raptor_try_name") + if "hw-s21" in platform and "speedometer3" not in try_name: + return False if ( "browsertime" in try_name and "speedometer" in try_name @@ -950,7 +975,9 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux. The nightly build process involves a pipeline of builds, signing, and, eventually, uploading the tasks to balrog.""" - filter = make_desktop_nightly_filter({"linux64-shippable", "linux-shippable"}) + filter = make_desktop_nightly_filter( + {"linux64-shippable", "linux-shippable", "linux-aarch64-shippable"} + ) return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] @@ -1060,6 +1087,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config): "searchfox-macosx64-searchfox/debug", "searchfox-win64-searchfox/debug", "searchfox-android-armv7-searchfox/debug", + "searchfox-ios-searchfox/debug", "source-test-file-metadata-bugzilla-components", "source-test-file-metadata-test-info-all", "source-test-wpt-metadata-summary", diff --git a/taskcluster/gecko_taskgraph/test/test_morph.py b/taskcluster/gecko_taskgraph/test/test_morph.py index c29fb58207..c3c499769c 100644 --- a/taskcluster/gecko_taskgraph/test/test_morph.py +++ b/taskcluster/gecko_taskgraph/test/test_morph.py @@ -26,6 +26,102 @@ def make_taskgraph(): return inner +@pytest.mark.parametrize( + "params,expected", + ( + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "rebuild": 10, + "tasks": ["b"], + }, + "project": "try", + }, + {"b": 10}, + id="duplicates no chunks", + ), + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "rebuild": 10, + "tasks": ["a-*"], + }, + "project": "try", + }, + {"a-1": 10, "a-2": 10}, + id="duplicates with chunks", + ), + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "rebuild": 10, + "tasks": ["a-*", "b"], + }, + "project": "try", + }, + {"a-1": 10, "a-2": 10, "b": 10}, + id="duplicates with and without chunks", + ), + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "tasks": ["a-*"], + }, + "project": "try", + }, + {"a-1": -1, "a-2": -1}, + id="no rebuild, no duplicates", + ), + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "rebuild": 0, + "tasks": ["a-*"], + }, + "project": "try", + }, + {"a-1": -1, "a-2": -1}, + id="rebuild of zero", + ), + pytest.param( + { + "try_mode": "try_task_config", + "try_task_config": { + "rebuild": 100, + "tasks": ["a-*"], + }, + "project": "try", + }, + {"a-1": 100, "a-2": 100}, + id="rebuild 100", + ), + ), +) +def test_try_task_duplicates(make_taskgraph, graph_config, params, expected): + taskb = Task(kind="test", label="b", attributes={}, task={}) + task1 = Task(kind="test", label="a-1", attributes={}, task={}) + task2 = Task(kind="test", label="a-2", attributes={}, task={}) + taskgraph, label_to_taskid = make_taskgraph( + { + taskb.label: taskb, + task1.label: task1, + task2.label: task2, + } + ) + + taskgraph, label_to_taskid = morph._add_try_task_duplicates( + taskgraph, label_to_taskid, params, graph_config + ) + for label in expected: + task = taskgraph.tasks[label] + assert task.attributes.get("task_duplicates", -1) == expected[label] + + def test_make_index_tasks(make_taskgraph, graph_config): task_def = { "routes": [ diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py index 2bbc57fcf3..22582f9040 100644 --- a/taskcluster/gecko_taskgraph/test/test_target_tasks.py +++ b/taskcluster/gecko_taskgraph/test/test_target_tasks.py @@ -210,6 +210,21 @@ class TestTargetTasks(unittest.TestCase): } self.assertEqual(sorted(method(tg, params, {})), ["ddd-1", "ddd-2"]) + def test_try_task_config_regex_with_paths(self): + "try_mode = try_task_config uses the try config with regex instead of chunk numbers" + tg = self.make_task_graph() + method = target_tasks.get_method("try_tasks") + params = { + "try_mode": "try_task_config", + "try_task_config": { + "new-test-config": True, + "tasks": ["ddd-*"], + "env": {"MOZHARNESS_TEST_PATHS": "foo/bar"}, + }, + "project": "try", + } + self.assertEqual(sorted(method(tg, params, {})), ["ddd-1"]) + def test_try_task_config_absolute(self): "try_mode = try_task_config uses the try config with full task labels" tg = self.make_task_graph() diff --git a/taskcluster/gecko_taskgraph/test/test_util_partials.py b/taskcluster/gecko_taskgraph/test/test_util_partials.py index 3630d7b0ec..4f8bea8662 100644 --- a/taskcluster/gecko_taskgraph/test/test_util_partials.py +++ b/taskcluster/gecko_taskgraph/test/test_util_partials.py @@ -30,18 +30,34 @@ release_blob = { def nightly_blob(release): - return { - "platforms": { - "WINNT_x86_64-msvc": { - "locales": { - "en-US": { - "buildID": release[-14:], - "completes": [{"fileUrl": release}], + # Added for bug 1883046, where we identified a case where a Balrog release + # that does not contain completes will throw an unnecessary exception. + if release == "Firefox-mozilla-central-nightly-20211001214601": + return { + "platforms": { + "WINNT_x86_64-msvc": { + "locales": { + "en-US": { + "buildID": release[-14:], + "partials": [{"fileUrl": release}], + } + } + } + } + } + else: + return { + "platforms": { + "WINNT_x86_64-msvc": { + "locales": { + "en-US": { + "buildID": release[-14:], + "completes": [{"fileUrl": release}], + } } } } } - } class TestReleaseHistory(unittest.TestCase): diff --git a/taskcluster/gecko_taskgraph/transforms/artifact.py b/taskcluster/gecko_taskgraph/transforms/artifact.py index 559148f7b4..b14f723d4a 100644 --- a/taskcluster/gecko_taskgraph/transforms/artifact.py +++ b/taskcluster/gecko_taskgraph/transforms/artifact.py @@ -85,9 +85,11 @@ def set_artifact_expiration(config, jobs): art_dict = manifest["macos"] elif plat.startswith("android"): art_dict = manifest["android"] + elif plat.startswith("ios"): + art_dict = manifest["ios"] else: print( - 'The platform name "{plat}" didn\'t start with', + f'The platform name "{plat}" didn\'t start with', '"win", "mac", "android", or "linux".', file=sys.stderr, ) diff --git a/taskcluster/gecko_taskgraph/transforms/artifacts.yml b/taskcluster/gecko_taskgraph/transforms/artifacts.yml index 26f06640ad..efcbc70f15 100644 --- a/taskcluster/gecko_taskgraph/transforms/artifacts.yml +++ b/taskcluster/gecko_taskgraph/transforms/artifacts.yml @@ -18,3 +18,7 @@ android: target.crashreporter-symbols-full.tar.zst: shortest sccache.log: shortest sccache-stats.json: shortest + +ios: + sccache.log: shortest + sccache-stats.json: shortest diff --git a/taskcluster/gecko_taskgraph/transforms/bootstrap.py b/taskcluster/gecko_taskgraph/transforms/bootstrap.py index e4537cab01..9c02fc6819 100644 --- a/taskcluster/gecko_taskgraph/transforms/bootstrap.py +++ b/taskcluster/gecko_taskgraph/transforms/bootstrap.py @@ -59,7 +59,7 @@ def bootstrap_tasks(config, tasks): f"python3 bootstrap.py --no-interactive --application-choice {app}", "cd mozilla-unified", # After bootstrap, configure should go through without its own auto-bootstrap. - "./mach configure --disable-bootstrap", + "./mach configure --enable-bootstrap=no-update", # Then a build should go through too. "./mach build", ] diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py index eb4aea609f..060f37c9c2 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py @@ -15,6 +15,7 @@ from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_u from gecko_taskgraph.transforms.job.common import get_expiration, support_vcs_checkout from gecko_taskgraph.transforms.test import normpath, test_description_schema from gecko_taskgraph.util.attributes import is_try +from gecko_taskgraph.util.perftest import is_external_browser VARIANTS = [ "shippable", @@ -63,9 +64,13 @@ def test_packages_url(taskdesc): ) # for android shippable we need to add 'en-US' to the artifact url test = taskdesc["run"]["test"] - if "android" in test["test-platform"] and ( - get_variant(test["test-platform"]) - in ("shippable", "shippable-qr", "shippable-lite", "shippable-lite-qr") + if ( + "android" in test["test-platform"] + and ( + get_variant(test["test-platform"]) + in ("shippable", "shippable-qr", "shippable-lite", "shippable-lite-qr") + ) + and not is_external_browser(test.get("try-name", "")) ): head, tail = os.path.split(artifact_url) artifact_url = os.path.join(head, "en-US", tail) diff --git a/taskcluster/gecko_taskgraph/transforms/snap_test.py b/taskcluster/gecko_taskgraph/transforms/snap_test.py index e6d879f225..d49276843d 100644 --- a/taskcluster/gecko_taskgraph/transforms/snap_test.py +++ b/taskcluster/gecko_taskgraph/transforms/snap_test.py @@ -43,6 +43,8 @@ def fill_template(config, tasks): timeout = 10 if collection != "opt": timeout = 60 + task["task"]["payload"]["env"]["BUILD_IS_DEBUG"] = "1" + task["task"]["payload"]["env"]["TEST_TIMEOUT"] = "{}".format(timeout) yield task diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py index 5c561e8114..e3eeb7c819 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_test.py +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -10,7 +10,6 @@ treeherder configuration and attributes for that platform. import copy import os -import taskgraph from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import keymatch from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by @@ -18,7 +17,6 @@ from taskgraph.util.treeherder import join_symbol, split_symbol from voluptuous import Any, Extra, Optional, Required from gecko_taskgraph.transforms.job import job_description_schema -from gecko_taskgraph.util.hg import get_json_automationrelevance source_test_description_schema = Schema( { @@ -239,33 +237,6 @@ def set_code_review_env(config, jobs): @transforms.add -def set_base_revision_in_tgdiff(config, jobs): - # Don't attempt to download 'json-automation' locally as the revision may - # not exist in the repository. - if not os.environ.get("MOZ_AUTOMATION") or taskgraph.fast: - yield from jobs - return - - data = get_json_automationrelevance( - config.params["head_repository"], config.params["head_rev"] - ) - for job in jobs: - if job["name"] != "taskgraph-diff": - yield job - continue - - job["task-context"] = { - "from-object": { - "base_rev": data["changesets"][0]["parents"][0], - }, - "substitution-fields": [ - "run.command", - ], - } - yield job - - -@transforms.add def set_worker_exit_code(config, jobs): for job in jobs: worker = job["worker"] diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py index dc258ef97a..b8cb95cff7 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/other.py +++ b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -16,6 +16,7 @@ from taskgraph.util.taskcluster import get_artifact_path, get_index_url from voluptuous import Any, Optional, Required from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS +from gecko_taskgraph.util.perftest import is_external_browser from gecko_taskgraph.util.platforms import platform_family from gecko_taskgraph.util.templates import merge @@ -260,6 +261,20 @@ def handle_keyed_by(config, tasks): @transforms.add +def setup_raptor_external_browser_platforms(config, tasks): + for task in tasks: + if task["suite"] != "raptor": + yield task + continue + + if is_external_browser(task["try-name"]): + task["build-platform"] = "linux64/opt" + task["build-label"] = "build-linux64/opt" + + yield task + + +@transforms.add def set_target(config, tasks): for task in tasks: build_platform = task["build-platform"] @@ -308,25 +323,25 @@ def setup_browsertime(config, tasks): ts = { "by-test-platform": { - "android.*": ["browsertime", "linux64-geckodriver", "linux64-node-16"], - "linux.*": ["browsertime", "linux64-geckodriver", "linux64-node-16"], + "android.*": ["browsertime", "linux64-geckodriver", "linux64-node"], + "linux.*": ["browsertime", "linux64-geckodriver", "linux64-node"], "macosx1015.*": [ "browsertime", "macosx64-geckodriver", - "macosx64-node-16", + "macosx64-node", ], "macosx1400.*": [ "browsertime", "macosx64-aarch64-geckodriver", - "macosx64-aarch64-node-16", + "macosx64-aarch64-node", ], "windows.*aarch64.*": [ "browsertime", "win32-geckodriver", - "win32-node-16", + "win32-node", ], - "windows.*-32.*": ["browsertime", "win32-geckodriver", "win32-node-16"], - "windows.*-64.*": ["browsertime", "win64-geckodriver", "win64-node-16"], + "windows.*-32.*": ["browsertime", "win32-geckodriver", "win32-node"], + "windows.*-64.*": ["browsertime", "win64-geckodriver", "win64-node"], }, } diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py index 0667d22bb2..18e21e6a1e 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -10,6 +10,7 @@ from voluptuous import Extra, Optional, Required from gecko_taskgraph.transforms.test import test_description_schema from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.perftest import is_external_browser transforms = TransformSequence() task_transforms = TransformSequence() @@ -316,6 +317,28 @@ def add_extra_options(config, tests): yield test +@transforms.add +def modify_mozharness_configs(config, tests): + for test in tests: + if not is_external_browser(test["app"]): + yield test + continue + + test_platform = test["test-platform"] + mozharness = test.setdefault("mozharness", {}) + if "mac" in test_platform: + mozharness["config"] = ["raptor/mac_external_browser_config.py"] + elif "windows" in test_platform: + mozharness["config"] = ["raptor/windows_external_browser_config.py"] + elif "linux" in test_platform: + mozharness["config"] = ["raptor/linux_external_browser_config.py"] + elif "android" in test_platform: + test["target"] = "target.tar.bz2" + mozharness["config"] = ["raptor/android_hw_external_browser_config.py"] + + yield test + + @task_transforms.add def add_scopes_and_proxy(config, tasks): for task in tasks: diff --git a/taskcluster/gecko_taskgraph/util/partials.py b/taskcluster/gecko_taskgraph/util/partials.py index 1a3affcc42..b04fc64e17 100644 --- a/taskcluster/gecko_taskgraph/util/partials.py +++ b/taskcluster/gecko_taskgraph/util/partials.py @@ -251,7 +251,17 @@ def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10): builds[platform][locale] = dict() if len(builds[platform][locale]) >= maxbuilds: continue + if "buildID" not in history["platforms"][platform]["locales"][locale]: + continue buildid = history["platforms"][platform]["locales"][locale]["buildID"] + if ( + "completes" not in history["platforms"][platform]["locales"][locale] + or len( + history["platforms"][platform]["locales"][locale]["completes"] + ) + == 0 + ): + continue url = history["platforms"][platform]["locales"][locale]["completes"][0][ "fileUrl" ] diff --git a/taskcluster/gecko_taskgraph/util/perftest.py b/taskcluster/gecko_taskgraph/util/perftest.py new file mode 100644 index 0000000000..01b153be37 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/perftest.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def is_external_browser(label): + if any( + external_browser in label + for external_browser in ( + "safari", + "chrome", + "custom-car", + "chrome-m", + "cstm-car-m", + ) + ): + return True + return False diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py index 78099f6eea..73e77fce66 100644 --- a/taskcluster/mach_commands.py +++ b/taskcluster/mach_commands.py @@ -17,23 +17,11 @@ from functools import partial import gecko_taskgraph.main from gecko_taskgraph.main import commands as taskgraph_commands from mach.decorators import Command, CommandArgument, SubCommand +from mach.util import strtobool logger = logging.getLogger("taskcluster") -def strtobool(value): - """Convert string to boolean. - - Wraps "distutils.util.strtobool", deferring the import of the package - in case it's not installed. Otherwise, we have a "chicken and egg problem" where - |mach bootstrap| would install the required package to enable "distutils.util", but - it can't because mach fails to interpret this file. - """ - from distutils.util import strtobool - - return bool(strtobool(value)) - - def get_taskgraph_command_parser(name): """Given a command name, obtain its argument parser. @@ -59,7 +47,7 @@ def get_taskgraph_decision_parser(): ( ["--optimize-target-tasks"], { - "type": lambda flag: strtobool(flag), + "type": lambda flag: bool(strtobool(flag)), "nargs": "?", "const": "true", "help": "If specified, this indicates whether the target " diff --git a/taskcluster/scripts/misc/afl-nyx.patch b/taskcluster/scripts/misc/afl-nyx.patch index 67b460900c..94a1bbb31d 100644 --- a/taskcluster/scripts/misc/afl-nyx.patch +++ b/taskcluster/scripts/misc/afl-nyx.patch @@ -16,3 +16,38 @@ index 988e536e..5e9870c0 100644 +#define MAP_SIZE_POW2 23 /* Do not change this unless you really know what you are doing. */ + +diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c +index caa3c3a8..76ceb451 100644 +--- a/instrumentation/afl-compiler-rt.o.c ++++ b/instrumentation/afl-compiler-rt.o.c +@@ -1837,7 +1837,7 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + + } + +- if (pc_filter) { ++ if (pc_filter && !mod_info->next) { + + char PcDescr[1024]; + // This function is a part of the sanitizer run-time. +@@ -1864,7 +1864,7 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + + } + +- if (__afl_filter_pcs && strstr(mod_info->name, __afl_filter_pcs_module)) { ++ if (__afl_filter_pcs && !mod_info->next && strstr(mod_info->name, __afl_filter_pcs_module)) { + + u32 result_index; + if (locate_in_pcs(PC, &result_index)) { +@@ -1889,7 +1889,11 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + + } + +- mod_info->mapped = 1; ++ if (__afl_pcmap_ptr) { ++ ++ mod_info->mapped = 1; ++ ++ } + + if (__afl_debug) { diff --git a/taskcluster/scripts/misc/are-we-esmified-yet.py b/taskcluster/scripts/misc/are-we-esmified-yet.py index ff888b80c7..9723565dfc 100644 --- a/taskcluster/scripts/misc/are-we-esmified-yet.py +++ b/taskcluster/scripts/misc/are-we-esmified-yet.py @@ -30,7 +30,6 @@ excluded_files = [ "js/xpconnect/tests/unit/not-esmified-not-exported.jsm", "js/xpconnect/tests/unit/recursive_importA.jsm", "js/xpconnect/tests/unit/recursive_importB.jsm", - "js/xpconnect/tests/unit/ReturnCodeChild.jsm", "js/xpconnect/tests/unit/syntax_error.jsm", "js/xpconnect/tests/unit/TestBlob.jsm", "js/xpconnect/tests/unit/TestFile.jsm", @@ -112,12 +111,12 @@ def collect_jsm(files): # js files with EXPORTED_SYMBOLS if mode == "hg": - cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"] + cmd = ["hg", "files", r"set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"] for line in run(cmd): put_file(files, kind, pathlib.Path(line)) else: handled = {} - cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", "*.js"] + cmd = ["git", "grep", r"EXPORTED_SYMBOLS = \[", "*.js"] for line in run(cmd): m = re.search("^([^:]+):", line) if not m: diff --git a/taskcluster/scripts/misc/build-custom-car.sh b/taskcluster/scripts/misc/build-custom-car.sh index d95538cb0e..62c135c83e 100755 --- a/taskcluster/scripts/misc/build-custom-car.sh +++ b/taskcluster/scripts/misc/build-custom-car.sh @@ -106,7 +106,7 @@ if [[ $(uname -o) == "Msys" ]]; then pushd "$WINDOWSSDKDIR" mkdir -p Debuggers/x64/ popd - mv $MOZ_FETCHES_DIR/VS/VC/Redist/MSVC/14.38.33130/x64/Microsoft.VC143.CRT/* chrome_dll/system32/ + mv $MOZ_FETCHES_DIR/VS/VC/Redist/MSVC/14.38.33135/x64/Microsoft.VC143.CRT/* chrome_dll/system32/ mv "$WINDOWSSDKDIR/App Certification Kit/"* "$WINDOWSSDKDIR"/Debuggers/x64/ export WINDIR="$PWD/chrome_dll" diff --git a/taskcluster/scripts/misc/build-libxml2.sh b/taskcluster/scripts/misc/build-libxml2.sh new file mode 100755 index 0000000000..0770ba0e43 --- /dev/null +++ b/taskcluster/scripts/misc/build-libxml2.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e -v -x + +mkdir -p $UPLOAD_DIR + +cd $MOZ_FETCHES_DIR/libxml2 + +export PATH="$MOZ_FETCHES_DIR/clang/bin:$PATH" + +# Building the .rc file requires some extra work, but we don't +# actually need it, so just remove its creation. +sed -i /libxml2\\.rc/d CMakeLists.txt + +cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=Off \ + -DCMAKE_C_COMPILER=clang-cl \ + -DCMAKE_LINKER=lld-link \ + -DCMAKE_MT=llvm-mt \ + -DCMAKE_C_FLAGS="-fuse-ld=lld -Xclang -ivfsoverlay -Xclang $MOZ_FETCHES_DIR/vs/overlay.yaml -winsysroot $MOZ_FETCHES_DIR/vs" \ + -DCMAKE_EXE_LINKER_FLAGS="-winsysroot:$MOZ_FETCHES_DIR/vs" \ + -DCMAKE_SYSTEM_NAME=Windows \ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded \ + -DLIBXML2_WITH_C14N=OFF \ + -DLIBXML2_WITH_CATALOG=OFF \ + -DLIBXML2_WITH_DEBUG=OFF \ + -DLIBXML2_WITH_DOCB=OFF \ + -DLIBXML2_WITH_FTP=OFF \ + -DLIBXML2_WITH_HTML=OFF \ + -DLIBXML2_WITH_HTTP=OFF \ + -DLIBXML2_WITH_ICONV=OFF \ + -DLIBXML2_WITH_ICU=OFF \ + -DLIBXML2_WITH_ISO8859X=OFF \ + -DLIBXML2_WITH_LEGACY=OFF \ + -DLIBXML2_WITH_LZMA=OFF \ + -DLIBXML2_WITH_MEM_DEBUG=OFF \ + -DLIBXML2_WITH_MODULES=OFF \ + -DLIBXML2_WITH_PROGRAMS=OFF \ + -DLIBXML2_WITH_PUSH=OFF \ + -DLIBXML2_WITH_PYTHON=OFF \ + -DLIBXML2_WITH_READER=OFF \ + -DLIBXML2_WITH_RUN_DEBUG=OFF \ + -DLIBXML2_WITH_SCHEMATRON=OFF \ + -DLIBXML2_WITH_TESTS=OFF \ + -DLIBXML2_WITH_THREAD_ALLOC=OFF \ + -DLIBXML2_WITH_VALID=OFF \ + -DLIBXML2_WITH_WRITER=OFF \ + -DLIBXML2_WITH_XINCLUDE=OFF \ + -DLIBXML2_WITH_XPATH=OFF \ + -DLIBXML2_WITH_XPTR=OFF \ + -DLIBXML2_WITH_ZLIB=OFF \ + -DCMAKE_INSTALL_PREFIX=$PWD/libxml2 \ + -B build + +ninja -C build -v install + +tar -caf libxml2.tar.zst libxml2/ +cp libxml2.tar.zst $UPLOAD_DIR/ diff --git a/taskcluster/scripts/misc/build-llvm-common.sh b/taskcluster/scripts/misc/build-llvm-common.sh index 2dbf68d3ec..73efc5093d 100755 --- a/taskcluster/scripts/misc/build-llvm-common.sh +++ b/taskcluster/scripts/misc/build-llvm-common.sh @@ -130,9 +130,9 @@ case "$target" in ln -s "windows kits/10" $MOZ_FETCHES_DIR/vs/sdk EXTRA_CMAKE_FLAGS=" $EXTRA_CMAKE_FLAGS - -DMSVC_BASE=$MOZ_FETCHES_DIR/vs/vc/tools/msvc/14.29.30133 + -DMSVC_BASE=$MOZ_FETCHES_DIR/vs/vc/tools/msvc/14.39.33519 -DWINSDK_BASE=$MOZ_FETCHES_DIR/vs/sdk - -DWINSDK_VER=10.0.19041.0 + -DWINSDK_VER=10.0.22621.0 " fi ;; diff --git a/taskcluster/scripts/misc/fetch-content b/taskcluster/scripts/misc/fetch-content index 9e6a1f4ef0..6e7b625dce 100755 --- a/taskcluster/scripts/misc/fetch-content +++ b/taskcluster/scripts/misc/fetch-content @@ -188,6 +188,25 @@ def stream_download(url, sha256=None, size=None): ) if certifi else urllib.request.urlopen(url, timeout=60) as fh: if not url.endswith(".gz") and fh.info().get("Content-Encoding") == "gzip": fh = gzip.GzipFile(fileobj=fh) + else: + # when using gzip we can't compare size or length (inflated) against content-length (compressed) + content_length = fh.getheader("content-length") + if content_length: + try: + content_length = int(content_length) + except ValueError: + raise IntegrityError( + "content-length header for %s is not an integer; got %s" + % (url, content_length) + ) + if size: + if size != content_length: + raise IntegrityError( + "size mismatch on %s: wanted %d; content-length is %d" + % (url, size, content_length) + ) + else: + size = content_length while True: chunk = fh.read(65536) diff --git a/taskcluster/scripts/misc/repack-android-emulator-linux.sh b/taskcluster/scripts/misc/repack-android-emulator-linux.sh index a9745d7a1e..277e6dff09 100755 --- a/taskcluster/scripts/misc/repack-android-emulator-linux.sh +++ b/taskcluster/scripts/misc/repack-android-emulator-linux.sh @@ -11,12 +11,19 @@ cd $GECKO_PATH ./mach python python/mozboot/mozboot/android.py --emulator-only --no-interactive --list-packages # Bug 1868944: override emulator to a known working version -curl -L http://dl.google.com/android/repository/emulator-linux_x64-10696886.zip > /tmp/emulator.zip +curl --fail -L http://dl.google.com/android/repository/emulator-linux_x64-10696886.zip > /tmp/emulator.zip cd /builds/worker/.mozbuild/android-sdk-linux rm -rf emulator unzip /tmp/emulator.zip cd $GECKO_PATH +# Bug 1880925: override platform-tools to a known working version +curl --fail -L https://dl.google.com/android/repository/platform-tools_r34.0.5-linux.zip > /tmp/platform-tools.zip +cd /builds/worker/.mozbuild/android-sdk-linux +rm -rf platform-tools +unzip /tmp/platform-tools.zip +cd $GECKO_PATH + # Remove extra files we don't need rm -rfv /builds/worker/.mozbuild/android-sdk-linux/tools mkdir /builds/worker/.mozbuild/android-sdk-linux/system-images diff --git a/taskcluster/scripts/misc/repack-android-sdk-linux.sh b/taskcluster/scripts/misc/repack-android-sdk-linux.sh index 0042d17472..d2fe85a9b5 100755 --- a/taskcluster/scripts/misc/repack-android-sdk-linux.sh +++ b/taskcluster/scripts/misc/repack-android-sdk-linux.sh @@ -11,12 +11,19 @@ cd $GECKO_PATH ./mach python python/mozboot/mozboot/android.py --artifact-mode --no-interactive --list-packages # Bug 1869407: override emulator to a known working version -curl -L http://dl.google.com/android/repository/emulator-linux_x64-10696886.zip > /tmp/emulator.zip +curl --fail -L http://dl.google.com/android/repository/emulator-linux_x64-10696886.zip > /tmp/emulator.zip cd /builds/worker/.mozbuild/android-sdk-linux rm -rf emulator unzip /tmp/emulator.zip cd $GECKO_PATH +# Bug 1880925: override platform-tools to a known working version +curl --fail -L https://dl.google.com/android/repository/platform-tools_r34.0.5-linux.zip > /tmp/platform-tools.zip +cd /builds/worker/.mozbuild/android-sdk-linux +rm -rf platform-tools +unzip /tmp/platform-tools.zip +cd $GECKO_PATH + tar cavf $UPLOAD_DIR/android-sdk-linux.tar.zst -C /builds/worker/.mozbuild android-sdk-linux bundletool.jar ls -al $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/vs-setup.sh b/taskcluster/scripts/misc/vs-setup.sh index 05eb219794..bef579b1e2 100644 --- a/taskcluster/scripts/misc/vs-setup.sh +++ b/taskcluster/scripts/misc/vs-setup.sh @@ -6,7 +6,7 @@ if [ ! -d "${VSPATH}/${VCDIR}" ]; then VCDIR=VC/Tools/MSVC/14.29.30133 fi if [ ! -d "${VSPATH}/${VCDIR}" ]; then - VCDIR=VC/Tools/MSVC/14.38.33130 + VCDIR=VC/Tools/MSVC/14.39.33519 fi SDKDIR="Windows Kits/10" SDK_VERSION=10.0.17134.0 diff --git a/taskcluster/scripts/misc/wasi-sdk.patch b/taskcluster/scripts/misc/wasi-sdk.patch index 9f7b6ad908..b9e26074d1 100644 --- a/taskcluster/scripts/misc/wasi-sdk.patch +++ b/taskcluster/scripts/misc/wasi-sdk.patch @@ -1,3 +1,13 @@ +diff --git a/llvm_version_major.sh b/llvm_version_major.sh +index 2e5d973..93fe9b0 100755 +--- a/llvm_version_major.sh ++++ b/llvm_version_major.sh +@@ -1,4 +1,4 @@ + #/bin/bash + LLVM_PROJ_DIR=${1:-./src/llvm-project} +-MAJOR=`grep "set(LLVM_VERSION_MAJOR" $LLVM_PROJ_DIR/llvm/CMakeLists.txt | awk '{print substr($2, 1, length($2) - 1)}'` ++MAJOR=`(grep "set(LLVM_VERSION_MAJOR" $LLVM_PROJ_DIR/llvm/CMakeLists.txt || grep "set(LLVM_VERSION_MAJOR" $LLVM_PROJ_DIR/cmake/Modules/LLVMVersion.cmake) | awk '{print substr($2, 1, length($2) - 1)}'` + echo $MAJOR diff --git a/Makefile b/Makefile index d2ccff6..95f395b 100644 --- a/Makefile diff --git a/taskcluster/test/params/try-config.yml b/taskcluster/test/params/try-config.yml new file mode 100644 index 0000000000..4eee79df5e --- /dev/null +++ b/taskcluster/test/params/try-config.yml @@ -0,0 +1,51 @@ +--- +base_repository: https://hg.mozilla.org/mozilla-central +build_date: 1509479809 +build_number: 1 +app_version: 60.0a1 +version: 60.0a1 +next_version: null +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +filters: + - target_tasks_method +head_ref: 482ccd66d49e0e4a0d260ca872f770df4983ccea +head_repository: https://hg.mozilla.org/try +head_rev: 482ccd66d49e0e4a0d260ca872f770df4983ccea +hg_branch: default +level: "1" +message: "Fuzzy query='foobar" +moz_build_date: "20171031195649" +optimize_target_tasks: false +owner: user@example.com +project: try +pushdate: 1509479809 +pushlog_id: "232079" +release_history: {} +release_eta: "" +release_enable_partners: false +release_partners: [] +release_partner_build_number: 1 +release_partner_config: null +release_enable_emefree: false +target_tasks_method: try_tasks +try_mode: try_task_config +try_task_config: + disable-pgo: true + env: + TRY_SELECTOR: "fuzzy" + tasks: + - test-linux1804-64-qr/opt-mochitest-plain-* + - test-linux1804-64-shippable-qr/opt-mochitest-plain-headless-1 + - test-linux1804-64-tsan-qr/opt-mochitest-plain-1 + - missing-task-label-1 + - missing-task-label-2 + - missing-task-label-* + use-artifact-builds: true +release_type: "nightly" +release_product: null +required_signoffs: [] +signoff_urls: {} +phabricator_diff: +tasks_for: hg-push diff --git a/taskcluster/test/params/try.yml b/taskcluster/test/params/try-syntax.yml index 7fddf517f3..7781ce927d 100644 --- a/taskcluster/test/params/try.yml +++ b/taskcluster/test/params/try-syntax.yml @@ -18,7 +18,7 @@ level: "1" message: "try: -b do -p linux64 -u none" moz_build_date: "20171031195649" optimize_target_tasks: false -owner: sfink@mozilla.com +owner: user@example.com project: try pushdate: 1509479809 pushlog_id: "232079" diff --git a/taskcluster/test/test_new_config.py b/taskcluster/test/test_new_config.py index a233dce97b..2410bc01a4 100644 --- a/taskcluster/test/test_new_config.py +++ b/taskcluster/test/test_new_config.py @@ -48,8 +48,9 @@ PARAMS_NEW_CONFIG.update( t.kind == "test" and t.attributes["unittest_suite"] == "mochitest-browser-chrome" and t.attributes["test_platform"] == "linux1804-64-qr/opt" + and ("spi-nw" not in t.label and "a11y-checks" not in t.label) ), - 64, + 32, id="mochitest-browser-chrome", ), ), |
