From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- taskcluster/.yamllint | 16 + taskcluster/ci/addon/kind.yml | 44 + taskcluster/ci/are-we-esmified-yet/kind.yml | 42 + taskcluster/ci/artifact-build/kind.yml | 105 + taskcluster/ci/attribution-l10n/kind.yml | 187 + taskcluster/ci/attribution/kind.yml | 176 + taskcluster/ci/balrog/kind.yml | 35 + taskcluster/ci/beetmover-apt/kind.yml | 29 + taskcluster/ci/beetmover-checksums/kind.yml | 30 + taskcluster/ci/beetmover-geckoview/kind.yml | 68 + taskcluster/ci/beetmover-repackage/kind.yml | 65 + taskcluster/ci/beetmover-source/kind.yml | 23 + taskcluster/ci/bootstrap/kind.yml | 103 + taskcluster/ci/bouncer-locations/kind.yml | 52 + taskcluster/ci/build-fat-aar/kind.yml | 200 + taskcluster/ci/build-mac-notarization/kind.yml | 34 + taskcluster/ci/build-mac-signing/kind.yml | 34 + taskcluster/ci/build-signing/kind.yml | 34 + taskcluster/ci/build/android-asan.yml | 75 + taskcluster/ci/build/android-stuff.yml | 53 + taskcluster/ci/build/android.yml | 644 + taskcluster/ci/build/kind.yml | 91 + taskcluster/ci/build/linux-base-toolchains.yml | 173 + taskcluster/ci/build/linux.yml | 1799 ++ taskcluster/ci/build/macosx-native.yml | 145 + taskcluster/ci/build/macosx.yml | 943 + taskcluster/ci/build/windows-mingw.yml | 173 + taskcluster/ci/build/windows.yml | 1583 ++ taskcluster/ci/code-review/kind.yml | 45 + taskcluster/ci/condprof/kind.yml | 136 + taskcluster/ci/config.yml | 879 + taskcluster/ci/cron-bouncer-check/kind.yml | 82 + taskcluster/ci/diffoscope/artifacts.yml | 52 + taskcluster/ci/diffoscope/kind.yml | 61 + taskcluster/ci/diffoscope/reproducible.yml | 29 + taskcluster/ci/docker-image/kind.yml | 247 + taskcluster/ci/fetch/benchmarks.yml | 19 + taskcluster/ci/fetch/browsertime.yml | 180 + taskcluster/ci/fetch/chromium-fetch.yml | 38 + taskcluster/ci/fetch/kind.yml | 20 + taskcluster/ci/fetch/moz.build | 8 + taskcluster/ci/fetch/resource-monitor.yml | 10 + .../ci/fetch/toolchain-clang-tidy-external.yml | 13 + taskcluster/ci/fetch/toolchains.yml | 714 + taskcluster/ci/fetch/updatebot.yml | 33 + taskcluster/ci/fuzzing/kind.yml | 69 + taskcluster/ci/fxrecord/kind.yml | 46 + .../ci/geckodriver-mac-notarization/kind.yml | 37 + taskcluster/ci/geckodriver-signing/kind.yml | 20 + taskcluster/ci/generate-profile/kind.yml | 324 + taskcluster/ci/github-sync/kind.yml | 53 + taskcluster/ci/hazard/kind.yml | 83 + taskcluster/ci/instrumented-build/kind.yml | 409 + taskcluster/ci/l10n-bump/kind.yml | 131 + taskcluster/ci/l10n-cross-channel/kind.yml | 41 + taskcluster/ci/l10n/kind.yml | 145 + .../ci/mar-signing-autograph-stage/kind.yml | 34 + taskcluster/ci/mar-signing-l10n/kind.yml | 39 + taskcluster/ci/mar-signing/kind.yml | 40 + taskcluster/ci/maybe-release/kind.yml | 45 + taskcluster/ci/merge-automation/kind.yml | 72 + taskcluster/ci/openh264-plugin/kind.yml | 226 + taskcluster/ci/openh264-signing/kind.yml | 19 + taskcluster/ci/packages/debian.yml | 126 + taskcluster/ci/packages/kind.yml | 25 + taskcluster/ci/packages/ubuntu.yml | 156 + taskcluster/ci/partials-signing/kind.yml | 24 + taskcluster/ci/partials/kind.yml | 39 + taskcluster/ci/perftest/android.yml | 338 + taskcluster/ci/perftest/kind.yml | 48 + taskcluster/ci/perftest/linux.yml | 384 + taskcluster/ci/perftest/macosx.yml | 287 + taskcluster/ci/perftest/windows.yml | 235 + taskcluster/ci/post-balrog-dummy/kind.yml | 22 + .../ci/post-beetmover-checksums-dummy/kind.yml | 31 + taskcluster/ci/post-beetmover-dummy/kind.yml | 23 + taskcluster/ci/post-langpack-dummy/kind.yml | 29 + taskcluster/ci/post-update-verify-dummy/kind.yml | 22 + taskcluster/ci/release-balrog-scheduling/kind.yml | 92 + .../ci/release-balrog-submit-toplevel/kind.yml | 79 + .../ci/release-beetmover-push-to-release/kind.yml | 33 + .../kind.yml | 23 + .../ci/release-beetmover-signed-langpacks/kind.yml | 40 + .../ci/release-beetmover-source-checksums/kind.yml | 23 + taskcluster/ci/release-bouncer-aliases/kind.yml | 95 + taskcluster/ci/release-bouncer-check/kind.yml | 76 + taskcluster/ci/release-bouncer-sub/kind.yml | 58 + taskcluster/ci/release-early-tagging/kind.yml | 46 + .../kind.yml | 24 + .../ci/release-eme-free-repack-beetmover/kind.yml | 26 + .../kind.yml | 29 + .../release-eme-free-repack-mac-signing/kind.yml | 28 + .../kind.yml | 24 + .../ci/release-eme-free-repack-repackage/kind.yml | 60 + taskcluster/ci/release-eme-free-repack/kind.yml | 81 + taskcluster/ci/release-final-verify/kind.yml | 47 + taskcluster/ci/release-flatpak-push/kind.yml | 42 + taskcluster/ci/release-flatpak-repackage/kind.yml | 66 + .../release-generate-checksums-beetmover/kind.yml | 21 + .../ci/release-generate-checksums-signing/kind.yml | 18 + taskcluster/ci/release-generate-checksums/kind.yml | 86 + taskcluster/ci/release-mark-as-shipped/kind.yml | 40 + taskcluster/ci/release-msix-push/kind.yml | 45 + taskcluster/ci/release-notify-av-announce/kind.yml | 40 + taskcluster/ci/release-notify-promote/kind.yml | 36 + taskcluster/ci/release-notify-push/kind.yml | 34 + taskcluster/ci/release-notify-ship/kind.yml | 41 + taskcluster/ci/release-notify-started/kind.yml | 32 + .../release-partner-attribution-beetmover/kind.yml | 22 + .../ci/release-partner-attribution/kind.yml | 40 + .../ci/release-partner-repack-beetmover/kind.yml | 30 + .../ci/release-partner-repack-bouncer-sub/kind.yml | 41 + .../release-partner-repack-chunking-dummy/kind.yml | 30 + .../kind.yml | 29 + .../ci/release-partner-repack-mac-signing/kind.yml | 29 + .../kind.yml | 29 + .../ci/release-partner-repack-repackage/kind.yml | 61 + taskcluster/ci/release-partner-repack/kind.yml | 87 + taskcluster/ci/release-push-langpacks/kind.yml | 52 + .../release-secondary-balrog-scheduling/kind.yml | 41 + .../kind.yml | 46 + .../ci/release-secondary-final-verify/kind.yml | 38 + .../ci/release-secondary-flatpak-push/kind.yml | 38 + .../ci/release-secondary-notify-ship/kind.yml | 34 + .../kind.yml | 124 + .../ci/release-secondary-update-verify/kind.yml | 77 + taskcluster/ci/release-snap-repackage/kind.yml | 60 + .../ci/release-source-checksums-signing/kind.yml | 20 + taskcluster/ci/release-source-signing/kind.yml | 21 + taskcluster/ci/release-source/kind.yml | 54 + .../ci/release-update-verify-config-next/kind.yml | 98 + .../ci/release-update-verify-config/kind.yml | 292 + taskcluster/ci/release-update-verify-next/kind.yml | 73 + taskcluster/ci/release-update-verify/kind.yml | 115 + taskcluster/ci/release-version-bump/kind.yml | 59 + taskcluster/ci/repackage-deb-l10n/kind.yml | 43 + taskcluster/ci/repackage-deb/kind.yml | 53 + taskcluster/ci/repackage-l10n/kind.yml | 86 + taskcluster/ci/repackage-msi/kind.yml | 49 + taskcluster/ci/repackage-msix/kind.yml | 107 + .../ci/repackage-shippable-l10n-msix/kind.yml | 109 + taskcluster/ci/repackage-signing-l10n/kind.yml | 31 + taskcluster/ci/repackage-signing-msi/kind.yml | 27 + taskcluster/ci/repackage-signing-msix/kind.yml | 19 + .../repackage-signing-shippable-l10n-msix/kind.yml | 18 + taskcluster/ci/repackage-signing/kind.yml | 29 + taskcluster/ci/repackage/kind.yml | 99 + taskcluster/ci/repo-update/kind.yml | 77 + taskcluster/ci/scriptworker-canary/kind.yml | 33 + taskcluster/ci/searchfox/kind.yml | 183 + taskcluster/ci/sentry/kind.yml | 34 + .../ci/shippable-l10n-mac-notarization/kind.yml | 29 + taskcluster/ci/shippable-l10n-mac-signing/kind.yml | 31 + taskcluster/ci/shippable-l10n-signing/kind.yml | 31 + taskcluster/ci/shippable-l10n/kind.yml | 199 + taskcluster/ci/snap-upstream-build/kind.yml | 218 + taskcluster/ci/snap-upstream-test/kind.yml | 73 + taskcluster/ci/source-test/clang.yml | 117 + taskcluster/ci/source-test/cram.yml | 21 + taskcluster/ci/source-test/doc.yml | 75 + taskcluster/ci/source-test/file-metadata.yml | 126 + taskcluster/ci/source-test/jsshell.yml | 87 + taskcluster/ci/source-test/kind.yml | 45 + taskcluster/ci/source-test/mozlint-android.yml | 106 + taskcluster/ci/source-test/mozlint.yml | 609 + taskcluster/ci/source-test/node.yml | 186 + taskcluster/ci/source-test/puppeteer.yml | 52 + taskcluster/ci/source-test/python.yml | 636 + taskcluster/ci/source-test/shadow-scheduler.yml | 178 + taskcluster/ci/source-test/taskgraph.yml | 37 + taskcluster/ci/source-test/vendor.yml | 44 + taskcluster/ci/source-test/webcompat.yml | 37 + taskcluster/ci/source-test/webidl.yml | 24 + taskcluster/ci/source-test/wpt-manifest.yml | 39 + taskcluster/ci/source-test/wpt-metadata.yml | 58 + taskcluster/ci/spidermonkey/kind.yml | 74 + taskcluster/ci/spidermonkey/linux.yml | 606 + taskcluster/ci/spidermonkey/windows.yml | 124 + taskcluster/ci/startup-test/kind.yml | 129 + taskcluster/ci/static-analysis-autotest/kind.yml | 101 + taskcluster/ci/system-symbols-reprocess/kind.yml | 46 + taskcluster/ci/system-symbols-upload/kind.yml | 34 + taskcluster/ci/system-symbols/kind.yml | 320 + taskcluster/ci/test/awsy.yml | 75 + taskcluster/ci/test/browsertime-desktop.yml | 918 + taskcluster/ci/test/browsertime-mobile.yml | 456 + taskcluster/ci/test/compiled.yml | 135 + taskcluster/ci/test/firefox-ui.yml | 27 + taskcluster/ci/test/kind.yml | 88 + taskcluster/ci/test/marionette.yml | 54 + taskcluster/ci/test/misc.yml | 269 + taskcluster/ci/test/mochitest.yml | 1171 ++ taskcluster/ci/test/reftest.yml | 248 + taskcluster/ci/test/talos.yml | 777 + taskcluster/ci/test/test-platforms.yml | 505 + taskcluster/ci/test/test-sets.yml | 499 + taskcluster/ci/test/variants.yml | 324 + taskcluster/ci/test/web-platform.yml | 598 + taskcluster/ci/test/xpcshell.yml | 177 + taskcluster/ci/toolchain/android.yml | 244 + taskcluster/ci/toolchain/cargo-vet.yml | 61 + taskcluster/ci/toolchain/cbindgen.yml | 65 + taskcluster/ci/toolchain/clang-tidy.yml | 127 + taskcluster/ci/toolchain/clang.yml | 1278 ++ taskcluster/ci/toolchain/compiler-rt.yml | 705 + taskcluster/ci/toolchain/dist-toolchains.yml | 40 + taskcluster/ci/toolchain/dump-syms.yml | 62 + taskcluster/ci/toolchain/fix-stacks.yml | 72 + taskcluster/ci/toolchain/gcc.yml | 79 + taskcluster/ci/toolchain/geckodriver.yml | 137 + taskcluster/ci/toolchain/gn.yml | 49 + taskcluster/ci/toolchain/grcov.yml | 48 + taskcluster/ci/toolchain/kind.yml | 54 + taskcluster/ci/toolchain/libunwind.yml | 247 + taskcluster/ci/toolchain/llvm-symbolizer.yml | 331 + taskcluster/ci/toolchain/macos-sdk.yml | 48 + taskcluster/ci/toolchain/mingw.yml | 42 + taskcluster/ci/toolchain/minidump-stackwalk.yml | 96 + taskcluster/ci/toolchain/misc.yml | 605 + taskcluster/ci/toolchain/nasm.yml | 87 + taskcluster/ci/toolchain/node.yml | 114 + taskcluster/ci/toolchain/pkgconf.yml | 56 + taskcluster/ci/toolchain/python.yml | 97 + taskcluster/ci/toolchain/resourcemonitor.yml | 44 + taskcluster/ci/toolchain/rust-size.yml | 38 + taskcluster/ci/toolchain/rust.yml | 219 + taskcluster/ci/toolchain/sccache.yml | 88 + taskcluster/ci/toolchain/sysroot.yml | 206 + taskcluster/ci/trigger-comm-central/kind.yml | 43 + taskcluster/ci/updatebot/kind.yml | 92 + taskcluster/ci/updatebot/moz.build | 8 + .../tooltool-manifests/updatebot.manifest | 29 + .../ci/upload-generated-sources-dummy/kind.yml | 26 + taskcluster/ci/upload-generated-sources/kind.yml | 42 + taskcluster/ci/upload-symbols-dummy/kind.yml | 27 + taskcluster/ci/upload-symbols/kind.yml | 53 + taskcluster/ci/valgrind/kind.yml | 65 + taskcluster/ci/webrender/kind.yml | 634 + taskcluster/docker/REGISTRY | 1 + taskcluster/docker/android-build/Dockerfile | 45 + taskcluster/docker/android-build/README.md | 2 + taskcluster/docker/android-build/VERSION | 1 + taskcluster/docker/condprof/Dockerfile | 15 + taskcluster/docker/custom-car-android/Dockerfile | 25 + taskcluster/docker/custom-car-linux/Dockerfile | 24 + taskcluster/docker/custom-v8/Dockerfile | 22 + taskcluster/docker/debian-base/Dockerfile | 66 + taskcluster/docker/debian-build/Dockerfile | 53 + taskcluster/docker/debian-packages/Dockerfile | 11 + taskcluster/docker/debian-raw/Dockerfile | 70 + taskcluster/docker/debian-raw/gpgvnoexpkeysig | 52 + taskcluster/docker/debian-raw/snapshot-hack.py | 105 + taskcluster/docker/debian-raw/taskcluster-hack.sh | 7 + taskcluster/docker/debian-repackage/Dockerfile | 35 + taskcluster/docker/decision/Dockerfile | 15 + taskcluster/docker/decision/HASH | 1 + taskcluster/docker/decision/README.md | 5 + taskcluster/docker/decision/VERSION | 1 + taskcluster/docker/decision/comm-task-env | 199 + taskcluster/docker/decision/system-setup.sh | 14 + taskcluster/docker/diffoscope/Dockerfile | 34 + taskcluster/docker/diffoscope/get_and_diffoscope | 109 + taskcluster/docker/diffoscope/readelf | 13 + taskcluster/docker/diffoscope/report_error | 36 + taskcluster/docker/diffoscope/test_diffoscope | 47 + taskcluster/docker/fetch/Dockerfile | 37 + taskcluster/docker/firefox-flatpak/Dockerfile | 19 + taskcluster/docker/firefox-flatpak/close_range.c | 12 + .../extract_locales_from_l10n_json.py | 18 + .../docker/firefox-flatpak/firefox-symbolic.svg | 6 + .../docker/firefox-flatpak/launch-script.sh | 3 + .../org.mozilla.firefox.appdata.xml.in | 45 + .../firefox-flatpak/org.mozilla.firefox.desktop | 386 + taskcluster/docker/firefox-flatpak/runme.sh | 186 + taskcluster/docker/firefox-snap/Dockerfile | 71 + taskcluster/docker/firefox-snap/Makefile | 12 + .../firefox-snap/download_and_install_snap.sh | 27 + .../firefox-snap/extract_locales_from_l10n_json.py | 15 + taskcluster/docker/firefox-snap/firefox.desktop | 219 + .../docker/firefox-snap/firefox.snapcraft.yaml.in | 91 + taskcluster/docker/firefox-snap/policies.json | 5 + taskcluster/docker/firefox-snap/runme.sh | 104 + taskcluster/docker/firefox-snap/tmpdir | 6 + .../docker/funsize-update-generator/Dockerfile | 48 + .../docker/funsize-update-generator/Makefile | 9 + taskcluster/docker/funsize-update-generator/README | 7 + .../docker/funsize-update-generator/dep1.pubkey | 14 + .../docker/funsize-update-generator/nightly.pubkey | 14 + .../docker/funsize-update-generator/release.pubkey | 14 + .../funsize-update-generator/requirements.in | 7 + .../funsize-update-generator/requirements.txt | 663 + .../docker/funsize-update-generator/runme.sh | 61 + .../funsize-update-generator/scripts/funsize.py | 471 + .../scripts/mbsdiff_hook.sh | 157 + taskcluster/docker/gdb-test/Dockerfile | 11 + taskcluster/docker/github-sync/Dockerfile | 14 + taskcluster/docker/github-sync/prepare.sh | 31 + taskcluster/docker/github-sync/requirements.txt | 3 + taskcluster/docker/image_builder/Dockerfile | 99 + taskcluster/docker/image_builder/VERSION | 1 + taskcluster/docker/image_builder/apt.conf | 5 + .../docker/image_builder/build-image/Cargo.lock | 1085 + .../docker/image_builder/build-image/Cargo.toml | 23 + .../docker/image_builder/build-image/src/config.rs | 112 + .../docker/image_builder/build-image/src/main.rs | 182 + .../image_builder/build-image/src/taskcluster.rs | 55 + taskcluster/docker/image_builder/policy.json | 11 + taskcluster/docker/index-task/.eslintrc.js | 11 + taskcluster/docker/index-task/Dockerfile | 11 + taskcluster/docker/index-task/README | 36 + taskcluster/docker/index-task/insert-indexes.js | 73 + taskcluster/docker/index-task/package.json | 12 + taskcluster/docker/index-task/yarn.lock | 326 + taskcluster/docker/lint/Dockerfile | 36 + taskcluster/docker/lint/system-setup.sh | 92 + taskcluster/docker/partner-repack/Dockerfile | 19 + taskcluster/docker/partner-repack/known_hosts | 3 + taskcluster/docker/periodic-updates/.eslintrc.js | 60 + taskcluster/docker/periodic-updates/Dockerfile | 11 + taskcluster/docker/periodic-updates/README.md | 96 + taskcluster/docker/periodic-updates/runme.sh | 98 + .../periodic-updates/scripts/genHPKPStaticPins.js | 673 + .../periodic-updates/scripts/getHSTSPreloadList.js | 557 + .../scripts/periodic_file_updates.sh | 672 + taskcluster/docker/periodic-updates/setup.sh | 24 + taskcluster/docker/push-to-try/Dockerfile | 22 + taskcluster/docker/push-to-try/hgrc | 2 + taskcluster/docker/push-to-try/known_hosts | 2 + taskcluster/docker/recipes/clean_packages.sh | 9 + taskcluster/docker/recipes/common.sh | 10 + .../docker/recipes/debian-test-system-setup.sh | 83 + taskcluster/docker/recipes/dot-config/pip/pip.conf | 2 + taskcluster/docker/recipes/hgrc | 33 + taskcluster/docker/recipes/install-node.sh | 15 + taskcluster/docker/recipes/setup_packages.sh | 13 + .../recipes/ubuntu1804-test-system-setup-base.sh | 162 + .../docker/recipes/ubuntu1804-test-system-setup.sh | 24 + taskcluster/docker/recipes/xvfb.sh | 75 + taskcluster/docker/sentry/Dockerfile | 11 + taskcluster/docker/sentry/prepare.sh | 18 + taskcluster/docker/sentry/submit_sentry_release.sh | 29 + taskcluster/docker/snap-coreXX-build/Dockerfile | 80 + .../docker/snap-coreXX-build/install-snap.sh | 24 + taskcluster/docker/snap-coreXX-build/parse.py | 39 + taskcluster/docker/snap-coreXX-build/run.sh | 114 + .../docker/snap-coreXX-build/snap-tests/README.md | 24 + .../snap-coreXX-build/snap-tests/basic_tests.py | 307 + .../snap-tests/basic_tests/expectations.json.in | 14 + .../snap-coreXX-build/snap-tests/qa_tests.py | 860 + .../snap-tests/qa_tests/qa_expectations.json | 98 + .../snap-tests/qa_tests/test_custom_fonts_ref.png | Bin 0 -> 51060 bytes .../snap-tests/qa_tests/test_pdf_download_base.png | Bin 0 -> 127756 bytes .../qa_tests/test_pdf_navigation_base.png | Bin 0 -> 167492 bytes .../test_pdf_navigation_documentProperties.png | Bin 0 -> 177679 bytes .../qa_tests/test_pdf_navigation_down.png | Bin 0 -> 242142 bytes .../qa_tests/test_pdf_navigation_end.png | Bin 0 -> 135099 bytes .../qa_tests/test_pdf_navigation_firstPage.png | Bin 0 -> 194764 bytes .../qa_tests/test_pdf_navigation_hand_tool.png | Bin 0 -> 197844 bytes .../qa_tests/test_pdf_navigation_home.png | Bin 0 -> 194764 bytes .../qa_tests/test_pdf_navigation_lastPage.png | Bin 0 -> 135099 bytes .../qa_tests/test_pdf_navigation_left.png | Bin 0 -> 194878 bytes .../qa_tests/test_pdf_navigation_next.png | Bin 0 -> 247191 bytes .../qa_tests/test_pdf_navigation_pageRotateCcw.png | Bin 0 -> 194810 bytes .../qa_tests/test_pdf_navigation_pageRotateCw.png | Bin 0 -> 276055 bytes .../qa_tests/test_pdf_navigation_pagedown.png | Bin 0 -> 216369 bytes .../qa_tests/test_pdf_navigation_pageup.png | Bin 0 -> 171383 bytes .../qa_tests/test_pdf_navigation_previous.png | Bin 0 -> 224991 bytes .../qa_tests/test_pdf_navigation_right.png | Bin 0 -> 247234 bytes .../qa_tests/test_pdf_navigation_select_text.png | Bin 0 -> 195176 bytes .../snap-tests/qa_tests/test_pdf_navigation_up.png | Bin 0 -> 224891 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_100p.png | Bin 0 -> 194764 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_150p.png | Bin 0 -> 166658 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_400p.png | Bin 0 -> 72156 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_50p.png | Bin 0 -> 150887 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_75p.png | Bin 0 -> 155313 bytes .../qa_tests/test_pdf_zoom_p1_actual.png | Bin 0 -> 195201 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_fit.png | Bin 0 -> 162736 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_width.png | Bin 0 -> 188823 bytes .../snap-coreXX-build/snap-tests/requirements.txt | 22 + .../docker/snap-coreXX-build/snap-tests/tests.sh | 68 + .../snap-tests/update-references.sh | 22 + .../docker/static-analysis-build/Dockerfile | 56 + .../docker/system-symbols-linux-scraper/Dockerfile | 28 + .../system-symbols-linux-scraper/SHA256SUMS.txt | 8 + .../docker/system-symbols-linux-scraper/run.sh | 80 + taskcluster/docker/system-symbols-mac/Dockerfile | 37 + .../docker/system-symbols-mac/requirements.txt | 2 + taskcluster/docker/system-symbols-mac/setup.sh | 27 + .../docker/system-symbols-win-gfx/Dockerfile | 25 + taskcluster/docker/system-symbols-win-gfx/run.sh | 59 + taskcluster/docker/system-symbols-win/Dockerfile | 25 + .../docker/system-symbols-win/requirements.txt | 17 + taskcluster/docker/toolchain-build/Dockerfile | 63 + taskcluster/docker/ubuntu1804-base/Dockerfile | 70 + .../docker/ubuntu1804-build-python/Dockerfile | 22 + taskcluster/docker/ubuntu1804-test-base/Dockerfile | 25 + taskcluster/docker/ubuntu1804-test/Dockerfile | 86 + taskcluster/docker/ubuntu1804-test/apport | 1 + .../autostart/deja-dup-monitor.desktop | 19 + .../autostart/gnome-software-service.desktop | 9 + taskcluster/docker/ubuntu1804-test/dbus.sh | 7 + .../dot-files/config/user-dirs.dirs | 15 + .../dot-files/config/user-dirs.locale | 1 + .../dot-files/config/xorg/99-serverflags.conf | 3 + .../ubuntu1804-test/dot-files/pulse/client.conf | 1 + taskcluster/docker/ubuntu1804-test/fonts.conf | 5 + taskcluster/docker/ubuntu1804-test/motd | 6 + .../ubuntu1804-test/taskcluster-interactive-shell | 22 + taskcluster/docker/update-verify/Dockerfile | 32 + taskcluster/docker/update-verify/system-setup.sh | 29 + taskcluster/docker/updatebot/Dockerfile | 24 + taskcluster/docker/updatebot/VERSION | 1 + .../docker/updatebot/arcanist_patch_size.patch | 13 + .../docker/updatebot/arcanist_windows_stream.patch | 19 + taskcluster/docker/updatebot/hgrc | 11 + taskcluster/docker/updatebot/moz.build | 8 + taskcluster/docker/updatebot/privileged-setup.sh | 71 + taskcluster/docker/updatebot/run.py | 223 + taskcluster/docker/updatebot/setup.sh | 20 + taskcluster/docker/updatebot/updatebot-version.sh | 2 + taskcluster/docker/updatebot/windows-php.ini | 130 + taskcluster/docker/updatebot/windows-setup.sh | 135 + taskcluster/docker/valgrind-build/Dockerfile | 67 + taskcluster/docker/webrender/Dockerfile | 16 + taskcluster/docs/attributes.rst | 476 + taskcluster/docs/balrog.rst | 45 + taskcluster/docs/caches.rst | 98 + taskcluster/docs/config.rst | 35 + taskcluster/docs/cron.rst | 102 + taskcluster/docs/howto/index.rst | 203 + taskcluster/docs/img/enableSourceServer.png | Bin 0 -> 28002 bytes taskcluster/docs/img/windbg-srcfix.png | Bin 0 -> 17238 bytes taskcluster/docs/index.rst | 79 + taskcluster/docs/kinds.rst | 809 + taskcluster/docs/kinds/test.rst | 161 + taskcluster/docs/optimization/index.rst | 20 + taskcluster/docs/optimization/schedules.rst | 97 + taskcluster/docs/parameters.rst | 267 + taskcluster/docs/partials.rst | 123 + taskcluster/docs/partner-attribution.rst | 118 + taskcluster/docs/partner-repacks.rst | 249 + taskcluster/docs/platforms.rst | 189 + taskcluster/docs/reference.rst | 12 + taskcluster/docs/release-promotion-action.rst | 158 + taskcluster/docs/release-promotion.rst | 53 + taskcluster/docs/signing.rst | 188 + taskcluster/docs/taskgraph.rst | 140 + taskcluster/docs/transforms/index.rst | 22 + taskcluster/docs/transforms/job.rst | 42 + taskcluster/docs/transforms/signing.rst | 15 + taskcluster/docs/transforms/task.rst | 45 + taskcluster/docs/transforms/test.rst | 15 + taskcluster/docs/try.rst | 153 + .../docs/using-the-mozilla-source-server.rst | 40 + taskcluster/docs/versioncontrol.rst | 74 + taskcluster/gecko_taskgraph/.ruff.toml | 4 + taskcluster/gecko_taskgraph/__init__.py | 68 + taskcluster/gecko_taskgraph/actions/__init__.py | 16 + .../gecko_taskgraph/actions/add_new_jobs.py | 59 + taskcluster/gecko_taskgraph/actions/add_talos.py | 59 + taskcluster/gecko_taskgraph/actions/backfill.py | 441 + taskcluster/gecko_taskgraph/actions/cancel.py | 36 + taskcluster/gecko_taskgraph/actions/cancel_all.py | 60 + .../gecko_taskgraph/actions/confirm_failure.py | 268 + .../gecko_taskgraph/actions/create_interactive.py | 188 + .../gecko_taskgraph/actions/gecko_profile.py | 138 + .../gecko_taskgraph/actions/merge_automation.py | 98 + taskcluster/gecko_taskgraph/actions/openh264.py | 33 + .../gecko_taskgraph/actions/purge_caches.py | 34 + .../actions/raptor_extra_options.py | 77 + .../actions/rebuild_cached_tasks.py | 37 + taskcluster/gecko_taskgraph/actions/registry.py | 371 + .../gecko_taskgraph/actions/release_promotion.py | 427 + taskcluster/gecko_taskgraph/actions/retrigger.py | 311 + .../gecko_taskgraph/actions/retrigger_custom.py | 185 + .../gecko_taskgraph/actions/run_missing_tests.py | 62 + .../gecko_taskgraph/actions/scriptworker_canary.py | 45 + .../gecko_taskgraph/actions/side_by_side.py | 189 + taskcluster/gecko_taskgraph/actions/util.py | 437 + taskcluster/gecko_taskgraph/config.py | 142 + taskcluster/gecko_taskgraph/decision.py | 498 + taskcluster/gecko_taskgraph/docker.py | 91 + taskcluster/gecko_taskgraph/files_changed.py | 95 + taskcluster/gecko_taskgraph/loader/__init__.py | 0 taskcluster/gecko_taskgraph/loader/test.py | 142 + taskcluster/gecko_taskgraph/loader/transform.py | 59 + taskcluster/gecko_taskgraph/main.py | 813 + .../gecko_taskgraph/manifests/fennec_geckoview.yml | 210 + .../manifests/firefox_candidates.yml | 433 + .../manifests/firefox_candidates_checksums.yml | 94 + .../gecko_taskgraph/manifests/firefox_nightly.yml | 523 + .../manifests/firefox_nightly_checksums.yml | 59 + .../manifests/release_checksums.yml | 70 + .../gecko_taskgraph/manifests/source_checksums.yml | 52 + .../gecko_taskgraph/manifests/source_files.yml | 52 + taskcluster/gecko_taskgraph/morph.py | 263 + taskcluster/gecko_taskgraph/optimize/__init__.py | 284 + taskcluster/gecko_taskgraph/optimize/backstop.py | 47 + taskcluster/gecko_taskgraph/optimize/bugbug.py | 321 + taskcluster/gecko_taskgraph/optimize/schema.py | 60 + taskcluster/gecko_taskgraph/optimize/strategies.py | 77 + taskcluster/gecko_taskgraph/parameters.py | 137 + taskcluster/gecko_taskgraph/target_tasks.py | 1606 ++ taskcluster/gecko_taskgraph/test/__init__.py | 0 .../gecko_taskgraph/test/automationrelevance.json | 358 + taskcluster/gecko_taskgraph/test/conftest.py | 218 + taskcluster/gecko_taskgraph/test/docs/kinds.rst | 12 + .../gecko_taskgraph/test/docs/parameters.rst | 14 + taskcluster/gecko_taskgraph/test/python.toml | 42 + .../gecko_taskgraph/test/test_actions_util.py | 179 + taskcluster/gecko_taskgraph/test/test_decision.py | 175 + .../gecko_taskgraph/test/test_files_changed.py | 90 + taskcluster/gecko_taskgraph/test/test_main.py | 67 + taskcluster/gecko_taskgraph/test/test_morph.py | 108 + .../test/test_optimize_strategies.py | 515 + .../gecko_taskgraph/test/test_target_tasks.py | 428 + .../gecko_taskgraph/test/test_taskcluster_yml.py | 145 + .../gecko_taskgraph/test/test_transforms_job.py | 111 + .../gecko_taskgraph/test/test_transforms_test.py | 330 + .../gecko_taskgraph/test/test_try_option_syntax.py | 430 + .../gecko_taskgraph/test/test_util_attributes.py | 99 + .../gecko_taskgraph/test/test_util_backstop.py | 155 + .../gecko_taskgraph/test/test_util_bugbug.py | 57 + .../gecko_taskgraph/test/test_util_chunking.py | 411 + .../gecko_taskgraph/test/test_util_docker.py | 255 + .../gecko_taskgraph/test/test_util_partials.py | 128 + .../test/test_util_runnable_jobs.py | 75 + .../gecko_taskgraph/test/test_util_templates.py | 79 + .../gecko_taskgraph/test/test_util_verify.py | 149 + taskcluster/gecko_taskgraph/transforms/__init__.py | 0 taskcluster/gecko_taskgraph/transforms/artifact.py | 116 + .../gecko_taskgraph/transforms/artifacts.yml | 20 + .../gecko_taskgraph/transforms/attribution.py | 69 + .../gecko_taskgraph/transforms/balrog_submit.py | 155 + .../gecko_taskgraph/transforms/balrog_toplevel.py | 42 + .../gecko_taskgraph/transforms/beetmover.py | 178 + .../gecko_taskgraph/transforms/beetmover_apt.py | 119 + .../transforms/beetmover_checksums.py | 145 + .../transforms/beetmover_emefree_checksums.py | 154 + .../transforms/beetmover_geckoview.py | 181 + .../transforms/beetmover_langpack_checksums.py | 143 + .../transforms/beetmover_push_to_release.py | 93 + .../transforms/beetmover_repackage.py | 358 + .../transforms/beetmover_repackage_l10n.py | 44 + .../transforms/beetmover_repackage_partner.py | 288 + .../gecko_taskgraph/transforms/beetmover_snap.py | 42 + .../gecko_taskgraph/transforms/beetmover_source.py | 35 + .../transforms/beetmover_source_checksums.py | 152 + .../gecko_taskgraph/transforms/bootstrap.py | 132 + .../gecko_taskgraph/transforms/bouncer_aliases.py | 108 + .../gecko_taskgraph/transforms/bouncer_check.py | 111 + .../transforms/bouncer_locations.py | 35 + .../transforms/bouncer_submission.py | 335 + .../transforms/bouncer_submission_partners.py | 193 + taskcluster/gecko_taskgraph/transforms/build.py | 235 + .../gecko_taskgraph/transforms/build_attrs.py | 50 + .../gecko_taskgraph/transforms/build_fat_aar.py | 78 + .../gecko_taskgraph/transforms/build_lints.py | 59 + .../gecko_taskgraph/transforms/build_signing.py | 74 + .../gecko_taskgraph/transforms/cached_tasks.py | 101 + .../gecko_taskgraph/transforms/chunk_partners.py | 78 + .../gecko_taskgraph/transforms/code_review.py | 33 + taskcluster/gecko_taskgraph/transforms/condprof.py | 91 + .../gecko_taskgraph/transforms/cross_channel.py | 44 + .../gecko_taskgraph/transforms/diffoscope.py | 172 + .../gecko_taskgraph/transforms/docker_image.py | 210 + taskcluster/gecko_taskgraph/transforms/fetch.py | 388 + .../gecko_taskgraph/transforms/final_verify.py | 35 + taskcluster/gecko_taskgraph/transforms/fxrecord.py | 30 + .../transforms/geckodriver_mac_notarization.py | 83 + .../transforms/geckodriver_signing.py | 139 + .../gecko_taskgraph/transforms/github_sync.py | 23 + .../gecko_taskgraph/transforms/hardened_signing.py | 111 + .../gecko_taskgraph/transforms/job/__init__.py | 507 + .../gecko_taskgraph/transforms/job/common.py | 269 + .../transforms/job/distro_package.py | 240 + .../gecko_taskgraph/transforms/job/hazard.py | 66 + taskcluster/gecko_taskgraph/transforms/job/mach.py | 80 + .../gecko_taskgraph/transforms/job/mozharness.py | 366 + .../transforms/job/mozharness_test.py | 477 + .../gecko_taskgraph/transforms/job/python_test.py | 47 + .../gecko_taskgraph/transforms/job/run_task.py | 268 + .../gecko_taskgraph/transforms/job/spidermonkey.py | 109 + .../gecko_taskgraph/transforms/job/toolchain.py | 257 + taskcluster/gecko_taskgraph/transforms/l10n.py | 423 + .../gecko_taskgraph/transforms/mac_dummy.py | 40 + .../gecko_taskgraph/transforms/mac_notarization.py | 19 + .../gecko_taskgraph/transforms/mar_signing.py | 143 + .../gecko_taskgraph/transforms/maybe_release.py | 23 + .../gecko_taskgraph/transforms/merge_automation.py | 81 + .../gecko_taskgraph/transforms/name_sanity.py | 48 + taskcluster/gecko_taskgraph/transforms/openh264.py | 26 + .../gecko_taskgraph/transforms/openh264_signing.py | 123 + taskcluster/gecko_taskgraph/transforms/partials.py | 174 + .../transforms/partner_attribution.py | 129 + .../transforms/partner_attribution_beetmover.py | 157 + .../gecko_taskgraph/transforms/partner_repack.py | 136 + .../gecko_taskgraph/transforms/partner_signing.py | 68 + .../transforms/per_platform_dummy.py | 36 + taskcluster/gecko_taskgraph/transforms/perftest.py | 358 + .../gecko_taskgraph/transforms/python_update.py | 25 + taskcluster/gecko_taskgraph/transforms/release.py | 20 + .../transforms/release_beetmover_signed_addons.py | 246 + .../gecko_taskgraph/transforms/release_deps.py | 61 + .../transforms/release_flatpak_push.py | 81 + .../transforms/release_flatpak_repackage.py | 42 + .../transforms/release_generate_checksums.py | 53 + .../release_generate_checksums_beetmover.py | 133 + .../release_generate_checksums_signing.py | 102 + .../transforms/release_mark_as_shipped.py | 39 + .../transforms/release_msix_push.py | 87 + .../transforms/release_notifications.py | 73 + .../transforms/release_sign_and_push_langpacks.py | 190 + .../transforms/release_snap_repackage.py | 39 + .../gecko_taskgraph/transforms/release_started.py | 52 + .../transforms/release_version_bump.py | 42 + .../gecko_taskgraph/transforms/repackage.py | 716 + .../gecko_taskgraph/transforms/repackage_l10n.py | 29 + .../transforms/repackage_partner.py | 316 + .../gecko_taskgraph/transforms/repackage_routes.py | 34 + .../transforms/repackage_set_upstream_mac_kind.py | 43 + .../transforms/repackage_signing.py | 153 + .../transforms/repackage_signing_partner.py | 163 + .../gecko_taskgraph/transforms/repo_update.py | 25 + .../transforms/reprocess_symbols.py | 72 + .../transforms/reverse_chunk_deps.py | 45 + .../gecko_taskgraph/transforms/run_pgo_profile.py | 34 + .../gecko_taskgraph/transforms/scriptworker.py | 18 + .../transforms/scriptworker_canary.py | 45 + taskcluster/gecko_taskgraph/transforms/sentry.py | 30 + .../transforms/shippable_l10n_signing.py | 88 + taskcluster/gecko_taskgraph/transforms/signing.py | 258 + .../gecko_taskgraph/transforms/snap_test.py | 48 + .../transforms/source_checksums_signing.py | 99 + .../gecko_taskgraph/transforms/source_test.py | 300 + .../gecko_taskgraph/transforms/spidermonkey.py | 21 + .../gecko_taskgraph/transforms/split_by_locale.py | 79 + .../gecko_taskgraph/transforms/startup_test.py | 40 + taskcluster/gecko_taskgraph/transforms/task.py | 2296 ++ .../gecko_taskgraph/transforms/test/__init__.py | 544 + .../gecko_taskgraph/transforms/test/chunk.py | 269 + .../transforms/test/confirm_failure.py | 46 + .../gecko_taskgraph/transforms/test/other.py | 1107 + .../gecko_taskgraph/transforms/test/raptor.py | 326 + .../gecko_taskgraph/transforms/test/variant.py | 124 + .../gecko_taskgraph/transforms/test/worker.py | 204 + .../transforms/trigger_comm_central.py | 24 + taskcluster/gecko_taskgraph/transforms/try_job.py | 18 + .../gecko_taskgraph/transforms/update_verify.py | 58 + .../transforms/update_verify_config.py | 148 + .../transforms/upload_generated_sources.py | 42 + .../gecko_taskgraph/transforms/upload_symbols.py | 94 + .../transforms/upstream_artifact_task.py | 29 + taskcluster/gecko_taskgraph/try_option_syntax.py | 750 + taskcluster/gecko_taskgraph/util/__init__.py | 0 taskcluster/gecko_taskgraph/util/attributes.py | 147 + taskcluster/gecko_taskgraph/util/backstop.py | 84 + taskcluster/gecko_taskgraph/util/bugbug.py | 125 + taskcluster/gecko_taskgraph/util/cached_tasks.py | 82 + taskcluster/gecko_taskgraph/util/chunking.py | 351 + taskcluster/gecko_taskgraph/util/copy_task.py | 40 + .../gecko_taskgraph/util/declarative_artifacts.py | 92 + taskcluster/gecko_taskgraph/util/dependencies.py | 156 + taskcluster/gecko_taskgraph/util/docker.py | 333 + taskcluster/gecko_taskgraph/util/hash.py | 68 + taskcluster/gecko_taskgraph/util/hg.py | 139 + taskcluster/gecko_taskgraph/util/partials.py | 297 + taskcluster/gecko_taskgraph/util/partners.py | 555 + taskcluster/gecko_taskgraph/util/perfile.py | 104 + taskcluster/gecko_taskgraph/util/platforms.py | 58 + taskcluster/gecko_taskgraph/util/scriptworker.py | 865 + .../gecko_taskgraph/util/signed_artifacts.py | 198 + taskcluster/gecko_taskgraph/util/taskcluster.py | 128 + taskcluster/gecko_taskgraph/util/taskgraph.py | 49 + taskcluster/gecko_taskgraph/util/templates.py | 59 + taskcluster/gecko_taskgraph/util/verify.py | 454 + taskcluster/gecko_taskgraph/util/workertypes.py | 103 + taskcluster/mach_commands.py | 400 + taskcluster/moz.build | 18 + taskcluster/scripts/builder/build-haz-linux.sh | 184 + taskcluster/scripts/builder/build-l10n.sh | 90 + taskcluster/scripts/builder/build-linux.sh | 125 + taskcluster/scripts/builder/build-sm-package.sh | 35 + taskcluster/scripts/builder/build-sm.sh | 64 + taskcluster/scripts/builder/repackage.sh | 95 + taskcluster/scripts/copy.sh | 9 + taskcluster/scripts/misc/afl-nyx.patch | 18 + .../misc/android-gradle-dependencies-lite.sh | 21 + .../scripts/misc/android-gradle-dependencies.sh | 21 + .../misc/android-gradle-dependencies/after.sh | 30 + .../misc/android-gradle-dependencies/before.sh | 30 + .../misc/android-gradle-dependencies/nexus.xml | 413 + taskcluster/scripts/misc/are-we-esmified-yet.py | 191 + taskcluster/scripts/misc/browsertime.sh | 19 + taskcluster/scripts/misc/build-afl.sh | 24 + taskcluster/scripts/misc/build-binutils-linux.sh | 14 + .../scripts/misc/build-breakpad-injector.sh | 29 + taskcluster/scripts/misc/build-cctools-port.sh | 102 + taskcluster/scripts/misc/build-clang-mingw.sh | 255 + .../scripts/misc/build-clang-tidy-external.sh | 11 + taskcluster/scripts/misc/build-clang.sh | 72 + taskcluster/scripts/misc/build-compiler-rt-wasi.sh | 36 + taskcluster/scripts/misc/build-compiler-rt.sh | 39 + taskcluster/scripts/misc/build-cpython.sh | 116 + taskcluster/scripts/misc/build-custom-car.sh | 205 + taskcluster/scripts/misc/build-custom-v8.sh | 47 + taskcluster/scripts/misc/build-dist-toolchains.sh | 12 + taskcluster/scripts/misc/build-dmg-hfsplus.sh | 36 + taskcluster/scripts/misc/build-gcc-linux.sh | 28 + .../scripts/misc/build-gcc-sixgill-plugin-linux.sh | 80 + taskcluster/scripts/misc/build-geckodriver.sh | 60 + taskcluster/scripts/misc/build-gn-common.sh | 36 + taskcluster/scripts/misc/build-gn-linux.sh | 13 + taskcluster/scripts/misc/build-gn-macosx.sh | 23 + taskcluster/scripts/misc/build-gn-win64.sh | 16 + taskcluster/scripts/misc/build-hfsplus-linux.sh | 14 + taskcluster/scripts/misc/build-libunwind.sh | 23 + taskcluster/scripts/misc/build-llvm-common.sh | 201 + taskcluster/scripts/misc/build-llvm-symbolizer.sh | 5 + taskcluster/scripts/misc/build-mar-tools.sh | 27 + taskcluster/scripts/misc/build-mingw-fxc2-x86.sh | 25 + taskcluster/scripts/misc/build-mingw32-nsis.sh | 71 + taskcluster/scripts/misc/build-mkbom-linux.sh | 17 + taskcluster/scripts/misc/build-mozmake.sh | 66 + taskcluster/scripts/misc/build-msix-packaging.sh | 33 + taskcluster/scripts/misc/build-nasm.sh | 63 + taskcluster/scripts/misc/build-nodejs.sh | 22 + taskcluster/scripts/misc/build-nsis.sh | 32 + taskcluster/scripts/misc/build-pkgconf.sh | 44 + taskcluster/scripts/misc/build-resourcemonitor.sh | 40 + .../scripts/misc/build-rust-based-toolchain.sh | 79 + taskcluster/scripts/misc/build-sysroot-wasi.sh | 53 + taskcluster/scripts/misc/build-sysroot.sh | 126 + taskcluster/scripts/misc/build-upx.sh | 26 + taskcluster/scripts/misc/build-winchecksec.sh | 198 + taskcluster/scripts/misc/build-wine.sh | 29 + taskcluster/scripts/misc/build-xar-linux.sh | 25 + taskcluster/scripts/misc/build-xz.sh | 25 + taskcluster/scripts/misc/cargo-apk-Cargo.lock | 802 + taskcluster/scripts/misc/cctools.patch | 17 + taskcluster/scripts/misc/dummy.sh | 1 + taskcluster/scripts/misc/fetch-chromium.py | 241 + taskcluster/scripts/misc/fetch-content | 900 + taskcluster/scripts/misc/get_vs.py | 111 + taskcluster/scripts/misc/mingw-composition.patch | 50 + taskcluster/scripts/misc/mingw-dispatchqueue.patch | 157 + taskcluster/scripts/misc/mingw-dwrite_3.patch | 87 + taskcluster/scripts/misc/mingw-enum.patch | 25 + taskcluster/scripts/misc/mingw-ts_sd.patch | 33 + taskcluster/scripts/misc/mingw-unknown.patch | 46 + taskcluster/scripts/misc/mingw-widl.patch | 35 + taskcluster/scripts/misc/moz.build | 8 + taskcluster/scripts/misc/osx-cross-linker | 8 + taskcluster/scripts/misc/pack-cpython.sh | 36 + taskcluster/scripts/misc/pack.sh | 24 + .../scripts/misc/private_local_toolchain.sh | 14 + .../scripts/misc/repack-android-avd-linux.sh | 30 + .../scripts/misc/repack-android-emulator-linux.sh | 28 + .../scripts/misc/repack-android-ndk-linux.sh | 17 + .../scripts/misc/repack-android-sdk-linux.sh | 22 + .../misc/repack-android-system-images-linux.sh | 17 + taskcluster/scripts/misc/repack-clang.sh | 52 + taskcluster/scripts/misc/repack-jdk-linux.sh | 17 + taskcluster/scripts/misc/repack-node.sh | 14 + taskcluster/scripts/misc/repack_rust.py | 674 + .../scripts/misc/run-profileserver-macos.sh | 20 + taskcluster/scripts/misc/run-profileserver.sh | 42 + .../scripts/misc/source-test-clang-setup.sh | 27 + taskcluster/scripts/misc/source-test-common.sh | 16 + .../scripts/misc/source-test-infer-setup.sh | 18 + taskcluster/scripts/misc/summarize-tgdiff.py | 52 + taskcluster/scripts/misc/tooltool-download.sh | 21 + taskcluster/scripts/misc/unify.sh | 42 + taskcluster/scripts/misc/unpack-sdk.py | 105 + taskcluster/scripts/misc/verify-devtools-bundle.py | 85 + taskcluster/scripts/misc/vs-cleanup.sh | 13 + taskcluster/scripts/misc/vs-setup.sh | 42 + taskcluster/scripts/misc/wasi-sdk-11.patch | 14 + taskcluster/scripts/misc/wasi-sdk.patch | 45 + .../scripts/misc/wr-cargotest-macos-build.sh | 24 + .../scripts/misc/wr-macos-cross-build-setup.sh | 60 + taskcluster/scripts/misc/wrench-android-build.sh | 26 + taskcluster/scripts/misc/wrench-deps-vendoring.sh | 23 + taskcluster/scripts/misc/wrench-macos-build.sh | 59 + taskcluster/scripts/misc/wrench-windows-tests.sh | 28 + taskcluster/scripts/misc/zstdpy | 79 + taskcluster/scripts/run-task | 1021 + taskcluster/scripts/tester/run-wizard | 176 + taskcluster/scripts/tester/test-linux.sh | 283 + taskcluster/test/conftest.py | 146 + taskcluster/test/data/automationrelevance.json | 169 + taskcluster/test/data/bugbug-push-schedules.json | 4638 +++++ taskcluster/test/data/pushes.json | 1 + taskcluster/test/params/autoland-onpush.yml | 49 + taskcluster/test/params/mb-onpush.yml | 49 + .../test/params/mb-promote-devedition-partials.yml | 14698 +++++++++++++ taskcluster/test/params/mb-promote-devedition.yml | 3064 +++ .../test/params/mb-promote-firefox-partials.yml | 14698 +++++++++++++ taskcluster/test/params/mb-promote-firefox.yml | 3064 +++ taskcluster/test/params/mb-push-devedition.yml | 9206 ++++++++ .../test/params/mb-push-firefox-partials.yml | 20816 +++++++++++++++++++ taskcluster/test/params/mb-push-firefox.yml | 9182 ++++++++ taskcluster/test/params/mb-ship-devedition.yml | 2666 +++ .../test/params/mb-ship-firefox-partials.yml | 14300 +++++++++++++ taskcluster/test/params/mb-ship-firefox.yml | 2666 +++ taskcluster/test/params/mb-ship-geckoview.yml | 2656 +++ taskcluster/test/params/mc-cron-system-symbols.yml | 49 + taskcluster/test/params/mc-desktop-nightly.yml | 13370 ++++++++++++ taskcluster/test/params/mc-onpush.yml | 49 + taskcluster/test/params/mc-ship-geckoview.yml | 4600 ++++ taskcluster/test/params/me-promote-firefox.yml | 1978 ++ taskcluster/test/params/me-push-firefox.yml | 7990 +++++++ taskcluster/test/params/me-ship-firefox.yml | 7995 +++++++ taskcluster/test/params/mr-onpush-geckoview.yml | 41 + taskcluster/test/params/mr-onpush.yml | 49 + taskcluster/test/params/mr-promote-firefox-rc.yml | 2420 +++ taskcluster/test/params/mr-promote-firefox.yml | 2408 +++ taskcluster/test/params/mr-push-firefox.yml | 11320 ++++++++++ taskcluster/test/params/mr-ship-firefox-rc.yml | 11320 ++++++++++ taskcluster/test/params/mr-ship-firefox.yml | 11325 ++++++++++ taskcluster/test/params/mr-ship-geckoview.yml | 1768 ++ taskcluster/test/params/try.yml | 59 + taskcluster/test/params/update.sh | 123 + taskcluster/test/python.toml | 14 + taskcluster/test/test_autoland.py | 48 + taskcluster/test/test_autoland_backstop.py | 56 + taskcluster/test/test_generate_params.py | 62 + taskcluster/test/test_mach_try_auto.py | 113 + taskcluster/test/test_mozilla_central.py | 69 + taskcluster/test/test_new_config.py | 98 + 830 files changed, 278544 insertions(+) create mode 100644 taskcluster/.yamllint create mode 100644 taskcluster/ci/addon/kind.yml create mode 100644 taskcluster/ci/are-we-esmified-yet/kind.yml create mode 100644 taskcluster/ci/artifact-build/kind.yml create mode 100644 taskcluster/ci/attribution-l10n/kind.yml create mode 100644 taskcluster/ci/attribution/kind.yml create mode 100644 taskcluster/ci/balrog/kind.yml create mode 100644 taskcluster/ci/beetmover-apt/kind.yml create mode 100644 taskcluster/ci/beetmover-checksums/kind.yml create mode 100644 taskcluster/ci/beetmover-geckoview/kind.yml create mode 100644 taskcluster/ci/beetmover-repackage/kind.yml create mode 100644 taskcluster/ci/beetmover-source/kind.yml create mode 100644 taskcluster/ci/bootstrap/kind.yml create mode 100644 taskcluster/ci/bouncer-locations/kind.yml create mode 100644 taskcluster/ci/build-fat-aar/kind.yml create mode 100644 taskcluster/ci/build-mac-notarization/kind.yml create mode 100644 taskcluster/ci/build-mac-signing/kind.yml create mode 100644 taskcluster/ci/build-signing/kind.yml create mode 100644 taskcluster/ci/build/android-asan.yml create mode 100644 taskcluster/ci/build/android-stuff.yml create mode 100644 taskcluster/ci/build/android.yml create mode 100644 taskcluster/ci/build/kind.yml create mode 100644 taskcluster/ci/build/linux-base-toolchains.yml create mode 100644 taskcluster/ci/build/linux.yml create mode 100644 taskcluster/ci/build/macosx-native.yml create mode 100644 taskcluster/ci/build/macosx.yml create mode 100644 taskcluster/ci/build/windows-mingw.yml create mode 100644 taskcluster/ci/build/windows.yml create mode 100644 taskcluster/ci/code-review/kind.yml create mode 100644 taskcluster/ci/condprof/kind.yml create mode 100644 taskcluster/ci/config.yml create mode 100644 taskcluster/ci/cron-bouncer-check/kind.yml create mode 100644 taskcluster/ci/diffoscope/artifacts.yml create mode 100644 taskcluster/ci/diffoscope/kind.yml create mode 100644 taskcluster/ci/diffoscope/reproducible.yml create mode 100644 taskcluster/ci/docker-image/kind.yml create mode 100644 taskcluster/ci/fetch/benchmarks.yml create mode 100644 taskcluster/ci/fetch/browsertime.yml create mode 100644 taskcluster/ci/fetch/chromium-fetch.yml create mode 100644 taskcluster/ci/fetch/kind.yml create mode 100644 taskcluster/ci/fetch/moz.build create mode 100644 taskcluster/ci/fetch/resource-monitor.yml create mode 100644 taskcluster/ci/fetch/toolchain-clang-tidy-external.yml create mode 100644 taskcluster/ci/fetch/toolchains.yml create mode 100644 taskcluster/ci/fetch/updatebot.yml create mode 100644 taskcluster/ci/fuzzing/kind.yml create mode 100644 taskcluster/ci/fxrecord/kind.yml create mode 100644 taskcluster/ci/geckodriver-mac-notarization/kind.yml create mode 100644 taskcluster/ci/geckodriver-signing/kind.yml create mode 100644 taskcluster/ci/generate-profile/kind.yml create mode 100644 taskcluster/ci/github-sync/kind.yml create mode 100644 taskcluster/ci/hazard/kind.yml create mode 100644 taskcluster/ci/instrumented-build/kind.yml create mode 100644 taskcluster/ci/l10n-bump/kind.yml create mode 100644 taskcluster/ci/l10n-cross-channel/kind.yml create mode 100644 taskcluster/ci/l10n/kind.yml create mode 100644 taskcluster/ci/mar-signing-autograph-stage/kind.yml create mode 100644 taskcluster/ci/mar-signing-l10n/kind.yml create mode 100644 taskcluster/ci/mar-signing/kind.yml create mode 100644 taskcluster/ci/maybe-release/kind.yml create mode 100644 taskcluster/ci/merge-automation/kind.yml create mode 100644 taskcluster/ci/openh264-plugin/kind.yml create mode 100644 taskcluster/ci/openh264-signing/kind.yml create mode 100644 taskcluster/ci/packages/debian.yml create mode 100644 taskcluster/ci/packages/kind.yml create mode 100644 taskcluster/ci/packages/ubuntu.yml create mode 100644 taskcluster/ci/partials-signing/kind.yml create mode 100644 taskcluster/ci/partials/kind.yml create mode 100644 taskcluster/ci/perftest/android.yml create mode 100644 taskcluster/ci/perftest/kind.yml create mode 100644 taskcluster/ci/perftest/linux.yml create mode 100644 taskcluster/ci/perftest/macosx.yml create mode 100644 taskcluster/ci/perftest/windows.yml create mode 100644 taskcluster/ci/post-balrog-dummy/kind.yml create mode 100644 taskcluster/ci/post-beetmover-checksums-dummy/kind.yml create mode 100644 taskcluster/ci/post-beetmover-dummy/kind.yml create mode 100644 taskcluster/ci/post-langpack-dummy/kind.yml create mode 100644 taskcluster/ci/post-update-verify-dummy/kind.yml create mode 100644 taskcluster/ci/release-balrog-scheduling/kind.yml create mode 100644 taskcluster/ci/release-balrog-submit-toplevel/kind.yml create mode 100644 taskcluster/ci/release-beetmover-push-to-release/kind.yml create mode 100644 taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml create mode 100644 taskcluster/ci/release-beetmover-signed-langpacks/kind.yml create mode 100644 taskcluster/ci/release-beetmover-source-checksums/kind.yml create mode 100644 taskcluster/ci/release-bouncer-aliases/kind.yml create mode 100644 taskcluster/ci/release-bouncer-check/kind.yml create mode 100644 taskcluster/ci/release-bouncer-sub/kind.yml create mode 100644 taskcluster/ci/release-early-tagging/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-beetmover-checksums/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-beetmover/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-mac-notarization/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-mac-signing/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack-repackage/kind.yml create mode 100644 taskcluster/ci/release-eme-free-repack/kind.yml create mode 100644 taskcluster/ci/release-final-verify/kind.yml create mode 100644 taskcluster/ci/release-flatpak-push/kind.yml create mode 100644 taskcluster/ci/release-flatpak-repackage/kind.yml create mode 100644 taskcluster/ci/release-generate-checksums-beetmover/kind.yml create mode 100644 taskcluster/ci/release-generate-checksums-signing/kind.yml create mode 100644 taskcluster/ci/release-generate-checksums/kind.yml create mode 100644 taskcluster/ci/release-mark-as-shipped/kind.yml create mode 100644 taskcluster/ci/release-msix-push/kind.yml create mode 100644 taskcluster/ci/release-notify-av-announce/kind.yml create mode 100644 taskcluster/ci/release-notify-promote/kind.yml create mode 100644 taskcluster/ci/release-notify-push/kind.yml create mode 100644 taskcluster/ci/release-notify-ship/kind.yml create mode 100644 taskcluster/ci/release-notify-started/kind.yml create mode 100644 taskcluster/ci/release-partner-attribution-beetmover/kind.yml create mode 100644 taskcluster/ci/release-partner-attribution/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-beetmover/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-mac-notarization/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-mac-signing/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-repackage-signing/kind.yml create mode 100644 taskcluster/ci/release-partner-repack-repackage/kind.yml create mode 100644 taskcluster/ci/release-partner-repack/kind.yml create mode 100644 taskcluster/ci/release-push-langpacks/kind.yml create mode 100644 taskcluster/ci/release-secondary-balrog-scheduling/kind.yml create mode 100644 taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml create mode 100644 taskcluster/ci/release-secondary-final-verify/kind.yml create mode 100644 taskcluster/ci/release-secondary-flatpak-push/kind.yml create mode 100644 taskcluster/ci/release-secondary-notify-ship/kind.yml create mode 100644 taskcluster/ci/release-secondary-update-verify-config/kind.yml create mode 100644 taskcluster/ci/release-secondary-update-verify/kind.yml create mode 100644 taskcluster/ci/release-snap-repackage/kind.yml create mode 100644 taskcluster/ci/release-source-checksums-signing/kind.yml create mode 100644 taskcluster/ci/release-source-signing/kind.yml create mode 100644 taskcluster/ci/release-source/kind.yml create mode 100644 taskcluster/ci/release-update-verify-config-next/kind.yml create mode 100644 taskcluster/ci/release-update-verify-config/kind.yml create mode 100644 taskcluster/ci/release-update-verify-next/kind.yml create mode 100644 taskcluster/ci/release-update-verify/kind.yml create mode 100644 taskcluster/ci/release-version-bump/kind.yml create mode 100644 taskcluster/ci/repackage-deb-l10n/kind.yml create mode 100644 taskcluster/ci/repackage-deb/kind.yml create mode 100644 taskcluster/ci/repackage-l10n/kind.yml create mode 100644 taskcluster/ci/repackage-msi/kind.yml create mode 100644 taskcluster/ci/repackage-msix/kind.yml create mode 100644 taskcluster/ci/repackage-shippable-l10n-msix/kind.yml create mode 100644 taskcluster/ci/repackage-signing-l10n/kind.yml create mode 100644 taskcluster/ci/repackage-signing-msi/kind.yml create mode 100644 taskcluster/ci/repackage-signing-msix/kind.yml create mode 100644 taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml create mode 100644 taskcluster/ci/repackage-signing/kind.yml create mode 100644 taskcluster/ci/repackage/kind.yml create mode 100644 taskcluster/ci/repo-update/kind.yml create mode 100644 taskcluster/ci/scriptworker-canary/kind.yml create mode 100644 taskcluster/ci/searchfox/kind.yml create mode 100644 taskcluster/ci/sentry/kind.yml create mode 100644 taskcluster/ci/shippable-l10n-mac-notarization/kind.yml create mode 100644 taskcluster/ci/shippable-l10n-mac-signing/kind.yml create mode 100644 taskcluster/ci/shippable-l10n-signing/kind.yml create mode 100644 taskcluster/ci/shippable-l10n/kind.yml create mode 100644 taskcluster/ci/snap-upstream-build/kind.yml create mode 100644 taskcluster/ci/snap-upstream-test/kind.yml create mode 100644 taskcluster/ci/source-test/clang.yml create mode 100644 taskcluster/ci/source-test/cram.yml create mode 100644 taskcluster/ci/source-test/doc.yml create mode 100644 taskcluster/ci/source-test/file-metadata.yml create mode 100644 taskcluster/ci/source-test/jsshell.yml create mode 100644 taskcluster/ci/source-test/kind.yml create mode 100644 taskcluster/ci/source-test/mozlint-android.yml create mode 100644 taskcluster/ci/source-test/mozlint.yml create mode 100644 taskcluster/ci/source-test/node.yml create mode 100644 taskcluster/ci/source-test/puppeteer.yml create mode 100644 taskcluster/ci/source-test/python.yml create mode 100644 taskcluster/ci/source-test/shadow-scheduler.yml create mode 100644 taskcluster/ci/source-test/taskgraph.yml create mode 100644 taskcluster/ci/source-test/vendor.yml create mode 100644 taskcluster/ci/source-test/webcompat.yml create mode 100644 taskcluster/ci/source-test/webidl.yml create mode 100644 taskcluster/ci/source-test/wpt-manifest.yml create mode 100644 taskcluster/ci/source-test/wpt-metadata.yml create mode 100644 taskcluster/ci/spidermonkey/kind.yml create mode 100644 taskcluster/ci/spidermonkey/linux.yml create mode 100644 taskcluster/ci/spidermonkey/windows.yml create mode 100644 taskcluster/ci/startup-test/kind.yml create mode 100644 taskcluster/ci/static-analysis-autotest/kind.yml create mode 100644 taskcluster/ci/system-symbols-reprocess/kind.yml create mode 100644 taskcluster/ci/system-symbols-upload/kind.yml create mode 100644 taskcluster/ci/system-symbols/kind.yml create mode 100644 taskcluster/ci/test/awsy.yml create mode 100644 taskcluster/ci/test/browsertime-desktop.yml create mode 100644 taskcluster/ci/test/browsertime-mobile.yml create mode 100644 taskcluster/ci/test/compiled.yml create mode 100644 taskcluster/ci/test/firefox-ui.yml create mode 100644 taskcluster/ci/test/kind.yml create mode 100644 taskcluster/ci/test/marionette.yml create mode 100644 taskcluster/ci/test/misc.yml create mode 100644 taskcluster/ci/test/mochitest.yml create mode 100644 taskcluster/ci/test/reftest.yml create mode 100644 taskcluster/ci/test/talos.yml create mode 100644 taskcluster/ci/test/test-platforms.yml create mode 100644 taskcluster/ci/test/test-sets.yml create mode 100644 taskcluster/ci/test/variants.yml create mode 100644 taskcluster/ci/test/web-platform.yml create mode 100644 taskcluster/ci/test/xpcshell.yml create mode 100644 taskcluster/ci/toolchain/android.yml create mode 100644 taskcluster/ci/toolchain/cargo-vet.yml create mode 100644 taskcluster/ci/toolchain/cbindgen.yml create mode 100644 taskcluster/ci/toolchain/clang-tidy.yml create mode 100644 taskcluster/ci/toolchain/clang.yml create mode 100644 taskcluster/ci/toolchain/compiler-rt.yml create mode 100644 taskcluster/ci/toolchain/dist-toolchains.yml create mode 100644 taskcluster/ci/toolchain/dump-syms.yml create mode 100644 taskcluster/ci/toolchain/fix-stacks.yml create mode 100644 taskcluster/ci/toolchain/gcc.yml create mode 100644 taskcluster/ci/toolchain/geckodriver.yml create mode 100644 taskcluster/ci/toolchain/gn.yml create mode 100644 taskcluster/ci/toolchain/grcov.yml create mode 100644 taskcluster/ci/toolchain/kind.yml create mode 100644 taskcluster/ci/toolchain/libunwind.yml create mode 100644 taskcluster/ci/toolchain/llvm-symbolizer.yml create mode 100644 taskcluster/ci/toolchain/macos-sdk.yml create mode 100644 taskcluster/ci/toolchain/mingw.yml create mode 100644 taskcluster/ci/toolchain/minidump-stackwalk.yml create mode 100644 taskcluster/ci/toolchain/misc.yml create mode 100644 taskcluster/ci/toolchain/nasm.yml create mode 100644 taskcluster/ci/toolchain/node.yml create mode 100644 taskcluster/ci/toolchain/pkgconf.yml create mode 100644 taskcluster/ci/toolchain/python.yml create mode 100644 taskcluster/ci/toolchain/resourcemonitor.yml create mode 100644 taskcluster/ci/toolchain/rust-size.yml create mode 100644 taskcluster/ci/toolchain/rust.yml create mode 100644 taskcluster/ci/toolchain/sccache.yml create mode 100644 taskcluster/ci/toolchain/sysroot.yml create mode 100644 taskcluster/ci/trigger-comm-central/kind.yml create mode 100644 taskcluster/ci/updatebot/kind.yml create mode 100644 taskcluster/ci/updatebot/moz.build create mode 100644 taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest create mode 100644 taskcluster/ci/upload-generated-sources-dummy/kind.yml create mode 100644 taskcluster/ci/upload-generated-sources/kind.yml create mode 100644 taskcluster/ci/upload-symbols-dummy/kind.yml create mode 100644 taskcluster/ci/upload-symbols/kind.yml create mode 100644 taskcluster/ci/valgrind/kind.yml create mode 100644 taskcluster/ci/webrender/kind.yml create mode 100644 taskcluster/docker/REGISTRY create mode 100644 taskcluster/docker/android-build/Dockerfile create mode 100644 taskcluster/docker/android-build/README.md create mode 100644 taskcluster/docker/android-build/VERSION create mode 100644 taskcluster/docker/condprof/Dockerfile create mode 100644 taskcluster/docker/custom-car-android/Dockerfile create mode 100644 taskcluster/docker/custom-car-linux/Dockerfile create mode 100644 taskcluster/docker/custom-v8/Dockerfile create mode 100644 taskcluster/docker/debian-base/Dockerfile create mode 100644 taskcluster/docker/debian-build/Dockerfile create mode 100644 taskcluster/docker/debian-packages/Dockerfile create mode 100644 taskcluster/docker/debian-raw/Dockerfile create mode 100755 taskcluster/docker/debian-raw/gpgvnoexpkeysig create mode 100755 taskcluster/docker/debian-raw/snapshot-hack.py create mode 100755 taskcluster/docker/debian-raw/taskcluster-hack.sh create mode 100644 taskcluster/docker/debian-repackage/Dockerfile create mode 100644 taskcluster/docker/decision/Dockerfile create mode 100644 taskcluster/docker/decision/HASH create mode 100644 taskcluster/docker/decision/README.md create mode 100644 taskcluster/docker/decision/VERSION create mode 100755 taskcluster/docker/decision/comm-task-env create mode 100644 taskcluster/docker/decision/system-setup.sh create mode 100644 taskcluster/docker/diffoscope/Dockerfile create mode 100644 taskcluster/docker/diffoscope/get_and_diffoscope create mode 100644 taskcluster/docker/diffoscope/readelf create mode 100755 taskcluster/docker/diffoscope/report_error create mode 100644 taskcluster/docker/diffoscope/test_diffoscope create mode 100644 taskcluster/docker/fetch/Dockerfile create mode 100644 taskcluster/docker/firefox-flatpak/Dockerfile create mode 100644 taskcluster/docker/firefox-flatpak/close_range.c create mode 100644 taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py create mode 100644 taskcluster/docker/firefox-flatpak/firefox-symbolic.svg create mode 100644 taskcluster/docker/firefox-flatpak/launch-script.sh create mode 100644 taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in create mode 100644 taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop create mode 100755 taskcluster/docker/firefox-flatpak/runme.sh create mode 100644 taskcluster/docker/firefox-snap/Dockerfile create mode 100644 taskcluster/docker/firefox-snap/Makefile create mode 100644 taskcluster/docker/firefox-snap/download_and_install_snap.sh create mode 100644 taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py create mode 100644 taskcluster/docker/firefox-snap/firefox.desktop create mode 100644 taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in create mode 100644 taskcluster/docker/firefox-snap/policies.json create mode 100755 taskcluster/docker/firefox-snap/runme.sh create mode 100755 taskcluster/docker/firefox-snap/tmpdir create mode 100644 taskcluster/docker/funsize-update-generator/Dockerfile create mode 100644 taskcluster/docker/funsize-update-generator/Makefile create mode 100644 taskcluster/docker/funsize-update-generator/README create mode 100644 taskcluster/docker/funsize-update-generator/dep1.pubkey create mode 100644 taskcluster/docker/funsize-update-generator/nightly.pubkey create mode 100644 taskcluster/docker/funsize-update-generator/release.pubkey create mode 100644 taskcluster/docker/funsize-update-generator/requirements.in create mode 100644 taskcluster/docker/funsize-update-generator/requirements.txt create mode 100644 taskcluster/docker/funsize-update-generator/runme.sh create mode 100644 taskcluster/docker/funsize-update-generator/scripts/funsize.py create mode 100755 taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh create mode 100644 taskcluster/docker/gdb-test/Dockerfile create mode 100644 taskcluster/docker/github-sync/Dockerfile create mode 100755 taskcluster/docker/github-sync/prepare.sh create mode 100644 taskcluster/docker/github-sync/requirements.txt create mode 100644 taskcluster/docker/image_builder/Dockerfile create mode 100644 taskcluster/docker/image_builder/VERSION create mode 100644 taskcluster/docker/image_builder/apt.conf create mode 100644 taskcluster/docker/image_builder/build-image/Cargo.lock create mode 100644 taskcluster/docker/image_builder/build-image/Cargo.toml create mode 100644 taskcluster/docker/image_builder/build-image/src/config.rs create mode 100644 taskcluster/docker/image_builder/build-image/src/main.rs create mode 100644 taskcluster/docker/image_builder/build-image/src/taskcluster.rs create mode 100644 taskcluster/docker/image_builder/policy.json create mode 100644 taskcluster/docker/index-task/.eslintrc.js create mode 100644 taskcluster/docker/index-task/Dockerfile create mode 100644 taskcluster/docker/index-task/README create mode 100644 taskcluster/docker/index-task/insert-indexes.js create mode 100644 taskcluster/docker/index-task/package.json create mode 100644 taskcluster/docker/index-task/yarn.lock create mode 100644 taskcluster/docker/lint/Dockerfile create mode 100644 taskcluster/docker/lint/system-setup.sh create mode 100644 taskcluster/docker/partner-repack/Dockerfile create mode 100644 taskcluster/docker/partner-repack/known_hosts create mode 100644 taskcluster/docker/periodic-updates/.eslintrc.js create mode 100644 taskcluster/docker/periodic-updates/Dockerfile create mode 100644 taskcluster/docker/periodic-updates/README.md create mode 100755 taskcluster/docker/periodic-updates/runme.sh create mode 100644 taskcluster/docker/periodic-updates/scripts/genHPKPStaticPins.js create mode 100644 taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js create mode 100755 taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh create mode 100755 taskcluster/docker/periodic-updates/setup.sh create mode 100644 taskcluster/docker/push-to-try/Dockerfile create mode 100644 taskcluster/docker/push-to-try/hgrc create mode 100644 taskcluster/docker/push-to-try/known_hosts create mode 100755 taskcluster/docker/recipes/clean_packages.sh create mode 100644 taskcluster/docker/recipes/common.sh create mode 100644 taskcluster/docker/recipes/debian-test-system-setup.sh create mode 100644 taskcluster/docker/recipes/dot-config/pip/pip.conf create mode 100755 taskcluster/docker/recipes/hgrc create mode 100644 taskcluster/docker/recipes/install-node.sh create mode 100755 taskcluster/docker/recipes/setup_packages.sh create mode 100644 taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh create mode 100644 taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh create mode 100644 taskcluster/docker/recipes/xvfb.sh create mode 100644 taskcluster/docker/sentry/Dockerfile create mode 100755 taskcluster/docker/sentry/prepare.sh create mode 100644 taskcluster/docker/sentry/submit_sentry_release.sh create mode 100644 taskcluster/docker/snap-coreXX-build/Dockerfile create mode 100755 taskcluster/docker/snap-coreXX-build/install-snap.sh create mode 100644 taskcluster/docker/snap-coreXX-build/parse.py create mode 100755 taskcluster/docker/snap-coreXX-build/run.sh create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/README.md create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/qa_expectations.json create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/requirements.txt create mode 100755 taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh create mode 100755 taskcluster/docker/snap-coreXX-build/snap-tests/update-references.sh create mode 100644 taskcluster/docker/static-analysis-build/Dockerfile create mode 100644 taskcluster/docker/system-symbols-linux-scraper/Dockerfile create mode 100644 taskcluster/docker/system-symbols-linux-scraper/SHA256SUMS.txt create mode 100755 taskcluster/docker/system-symbols-linux-scraper/run.sh create mode 100644 taskcluster/docker/system-symbols-mac/Dockerfile create mode 100644 taskcluster/docker/system-symbols-mac/requirements.txt create mode 100644 taskcluster/docker/system-symbols-mac/setup.sh create mode 100644 taskcluster/docker/system-symbols-win-gfx/Dockerfile create mode 100755 taskcluster/docker/system-symbols-win-gfx/run.sh create mode 100644 taskcluster/docker/system-symbols-win/Dockerfile create mode 100644 taskcluster/docker/system-symbols-win/requirements.txt create mode 100644 taskcluster/docker/toolchain-build/Dockerfile create mode 100644 taskcluster/docker/ubuntu1804-base/Dockerfile create mode 100644 taskcluster/docker/ubuntu1804-build-python/Dockerfile create mode 100644 taskcluster/docker/ubuntu1804-test-base/Dockerfile create mode 100644 taskcluster/docker/ubuntu1804-test/Dockerfile create mode 100644 taskcluster/docker/ubuntu1804-test/apport create mode 100644 taskcluster/docker/ubuntu1804-test/autostart/deja-dup-monitor.desktop create mode 100644 taskcluster/docker/ubuntu1804-test/autostart/gnome-software-service.desktop create mode 100644 taskcluster/docker/ubuntu1804-test/dbus.sh create mode 100644 taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.dirs create mode 100644 taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.locale create mode 100644 taskcluster/docker/ubuntu1804-test/dot-files/config/xorg/99-serverflags.conf create mode 100644 taskcluster/docker/ubuntu1804-test/dot-files/pulse/client.conf create mode 100644 taskcluster/docker/ubuntu1804-test/fonts.conf create mode 100644 taskcluster/docker/ubuntu1804-test/motd create mode 100644 taskcluster/docker/ubuntu1804-test/taskcluster-interactive-shell create mode 100644 taskcluster/docker/update-verify/Dockerfile create mode 100644 taskcluster/docker/update-verify/system-setup.sh create mode 100644 taskcluster/docker/updatebot/Dockerfile create mode 100644 taskcluster/docker/updatebot/VERSION create mode 100644 taskcluster/docker/updatebot/arcanist_patch_size.patch create mode 100644 taskcluster/docker/updatebot/arcanist_windows_stream.patch create mode 100644 taskcluster/docker/updatebot/hgrc create mode 100644 taskcluster/docker/updatebot/moz.build create mode 100755 taskcluster/docker/updatebot/privileged-setup.sh create mode 100755 taskcluster/docker/updatebot/run.py create mode 100755 taskcluster/docker/updatebot/setup.sh create mode 100644 taskcluster/docker/updatebot/updatebot-version.sh create mode 100644 taskcluster/docker/updatebot/windows-php.ini create mode 100644 taskcluster/docker/updatebot/windows-setup.sh create mode 100644 taskcluster/docker/valgrind-build/Dockerfile create mode 100644 taskcluster/docker/webrender/Dockerfile create mode 100644 taskcluster/docs/attributes.rst create mode 100644 taskcluster/docs/balrog.rst create mode 100644 taskcluster/docs/caches.rst create mode 100644 taskcluster/docs/config.rst create mode 100644 taskcluster/docs/cron.rst create mode 100644 taskcluster/docs/howto/index.rst create mode 100644 taskcluster/docs/img/enableSourceServer.png create mode 100644 taskcluster/docs/img/windbg-srcfix.png create mode 100644 taskcluster/docs/index.rst create mode 100644 taskcluster/docs/kinds.rst create mode 100644 taskcluster/docs/kinds/test.rst create mode 100644 taskcluster/docs/optimization/index.rst create mode 100644 taskcluster/docs/optimization/schedules.rst create mode 100644 taskcluster/docs/parameters.rst create mode 100644 taskcluster/docs/partials.rst create mode 100644 taskcluster/docs/partner-attribution.rst create mode 100644 taskcluster/docs/partner-repacks.rst create mode 100644 taskcluster/docs/platforms.rst create mode 100644 taskcluster/docs/reference.rst create mode 100644 taskcluster/docs/release-promotion-action.rst create mode 100644 taskcluster/docs/release-promotion.rst create mode 100644 taskcluster/docs/signing.rst create mode 100644 taskcluster/docs/taskgraph.rst create mode 100644 taskcluster/docs/transforms/index.rst create mode 100644 taskcluster/docs/transforms/job.rst create mode 100644 taskcluster/docs/transforms/signing.rst create mode 100644 taskcluster/docs/transforms/task.rst create mode 100644 taskcluster/docs/transforms/test.rst create mode 100644 taskcluster/docs/try.rst create mode 100644 taskcluster/docs/using-the-mozilla-source-server.rst create mode 100644 taskcluster/docs/versioncontrol.rst create mode 100644 taskcluster/gecko_taskgraph/.ruff.toml create mode 100644 taskcluster/gecko_taskgraph/__init__.py create mode 100644 taskcluster/gecko_taskgraph/actions/__init__.py create mode 100644 taskcluster/gecko_taskgraph/actions/add_new_jobs.py create mode 100644 taskcluster/gecko_taskgraph/actions/add_talos.py create mode 100644 taskcluster/gecko_taskgraph/actions/backfill.py create mode 100644 taskcluster/gecko_taskgraph/actions/cancel.py create mode 100644 taskcluster/gecko_taskgraph/actions/cancel_all.py create mode 100644 taskcluster/gecko_taskgraph/actions/confirm_failure.py create mode 100644 taskcluster/gecko_taskgraph/actions/create_interactive.py create mode 100644 taskcluster/gecko_taskgraph/actions/gecko_profile.py create mode 100644 taskcluster/gecko_taskgraph/actions/merge_automation.py create mode 100644 taskcluster/gecko_taskgraph/actions/openh264.py create mode 100644 taskcluster/gecko_taskgraph/actions/purge_caches.py create mode 100644 taskcluster/gecko_taskgraph/actions/raptor_extra_options.py create mode 100644 taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py create mode 100644 taskcluster/gecko_taskgraph/actions/registry.py create mode 100644 taskcluster/gecko_taskgraph/actions/release_promotion.py create mode 100644 taskcluster/gecko_taskgraph/actions/retrigger.py create mode 100644 taskcluster/gecko_taskgraph/actions/retrigger_custom.py create mode 100644 taskcluster/gecko_taskgraph/actions/run_missing_tests.py create mode 100644 taskcluster/gecko_taskgraph/actions/scriptworker_canary.py create mode 100644 taskcluster/gecko_taskgraph/actions/side_by_side.py create mode 100644 taskcluster/gecko_taskgraph/actions/util.py create mode 100644 taskcluster/gecko_taskgraph/config.py create mode 100644 taskcluster/gecko_taskgraph/decision.py create mode 100644 taskcluster/gecko_taskgraph/docker.py create mode 100644 taskcluster/gecko_taskgraph/files_changed.py create mode 100644 taskcluster/gecko_taskgraph/loader/__init__.py create mode 100644 taskcluster/gecko_taskgraph/loader/test.py create mode 100644 taskcluster/gecko_taskgraph/loader/transform.py create mode 100644 taskcluster/gecko_taskgraph/main.py create mode 100644 taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/release_checksums.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/source_checksums.yml create mode 100644 taskcluster/gecko_taskgraph/manifests/source_files.yml create mode 100644 taskcluster/gecko_taskgraph/morph.py create mode 100644 taskcluster/gecko_taskgraph/optimize/__init__.py create mode 100644 taskcluster/gecko_taskgraph/optimize/backstop.py create mode 100644 taskcluster/gecko_taskgraph/optimize/bugbug.py create mode 100644 taskcluster/gecko_taskgraph/optimize/schema.py create mode 100644 taskcluster/gecko_taskgraph/optimize/strategies.py create mode 100644 taskcluster/gecko_taskgraph/parameters.py create mode 100644 taskcluster/gecko_taskgraph/target_tasks.py create mode 100644 taskcluster/gecko_taskgraph/test/__init__.py create mode 100644 taskcluster/gecko_taskgraph/test/automationrelevance.json create mode 100644 taskcluster/gecko_taskgraph/test/conftest.py create mode 100644 taskcluster/gecko_taskgraph/test/docs/kinds.rst create mode 100644 taskcluster/gecko_taskgraph/test/docs/parameters.rst create mode 100644 taskcluster/gecko_taskgraph/test/python.toml create mode 100644 taskcluster/gecko_taskgraph/test/test_actions_util.py create mode 100644 taskcluster/gecko_taskgraph/test/test_decision.py create mode 100644 taskcluster/gecko_taskgraph/test/test_files_changed.py create mode 100644 taskcluster/gecko_taskgraph/test/test_main.py create mode 100644 taskcluster/gecko_taskgraph/test/test_morph.py create mode 100644 taskcluster/gecko_taskgraph/test/test_optimize_strategies.py create mode 100644 taskcluster/gecko_taskgraph/test/test_target_tasks.py create mode 100644 taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py create mode 100644 taskcluster/gecko_taskgraph/test/test_transforms_job.py create mode 100644 taskcluster/gecko_taskgraph/test/test_transforms_test.py create mode 100644 taskcluster/gecko_taskgraph/test/test_try_option_syntax.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_attributes.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_backstop.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_bugbug.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_chunking.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_docker.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_partials.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_templates.py create mode 100644 taskcluster/gecko_taskgraph/test/test_util_verify.py create mode 100644 taskcluster/gecko_taskgraph/transforms/__init__.py create mode 100644 taskcluster/gecko_taskgraph/transforms/artifact.py create mode 100644 taskcluster/gecko_taskgraph/transforms/artifacts.yml create mode 100644 taskcluster/gecko_taskgraph/transforms/attribution.py create mode 100644 taskcluster/gecko_taskgraph/transforms/balrog_submit.py create mode 100644 taskcluster/gecko_taskgraph/transforms/balrog_toplevel.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_apt.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_snap.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_source.py create mode 100644 taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bootstrap.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bouncer_check.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bouncer_locations.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bouncer_submission.py create mode 100644 taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py create mode 100644 taskcluster/gecko_taskgraph/transforms/build.py create mode 100644 taskcluster/gecko_taskgraph/transforms/build_attrs.py create mode 100644 taskcluster/gecko_taskgraph/transforms/build_fat_aar.py create mode 100644 taskcluster/gecko_taskgraph/transforms/build_lints.py create mode 100644 taskcluster/gecko_taskgraph/transforms/build_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/cached_tasks.py create mode 100644 taskcluster/gecko_taskgraph/transforms/chunk_partners.py create mode 100644 taskcluster/gecko_taskgraph/transforms/code_review.py create mode 100644 taskcluster/gecko_taskgraph/transforms/condprof.py create mode 100644 taskcluster/gecko_taskgraph/transforms/cross_channel.py create mode 100644 taskcluster/gecko_taskgraph/transforms/diffoscope.py create mode 100644 taskcluster/gecko_taskgraph/transforms/docker_image.py create mode 100644 taskcluster/gecko_taskgraph/transforms/fetch.py create mode 100644 taskcluster/gecko_taskgraph/transforms/final_verify.py create mode 100644 taskcluster/gecko_taskgraph/transforms/fxrecord.py create mode 100644 taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py create mode 100644 taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/github_sync.py create mode 100644 taskcluster/gecko_taskgraph/transforms/hardened_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/__init__.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/common.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/distro_package.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/hazard.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/mach.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/mozharness.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/python_test.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/run_task.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py create mode 100644 taskcluster/gecko_taskgraph/transforms/job/toolchain.py create mode 100644 taskcluster/gecko_taskgraph/transforms/l10n.py create mode 100644 taskcluster/gecko_taskgraph/transforms/mac_dummy.py create mode 100644 taskcluster/gecko_taskgraph/transforms/mac_notarization.py create mode 100644 taskcluster/gecko_taskgraph/transforms/mar_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/maybe_release.py create mode 100644 taskcluster/gecko_taskgraph/transforms/merge_automation.py create mode 100644 taskcluster/gecko_taskgraph/transforms/name_sanity.py create mode 100644 taskcluster/gecko_taskgraph/transforms/openh264.py create mode 100644 taskcluster/gecko_taskgraph/transforms/openh264_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/partials.py create mode 100644 taskcluster/gecko_taskgraph/transforms/partner_attribution.py create mode 100644 taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py create mode 100644 taskcluster/gecko_taskgraph/transforms/partner_repack.py create mode 100644 taskcluster/gecko_taskgraph/transforms/partner_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/per_platform_dummy.py create mode 100644 taskcluster/gecko_taskgraph/transforms/perftest.py create mode 100644 taskcluster/gecko_taskgraph/transforms/python_update.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_deps.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_msix_push.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_notifications.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_started.py create mode 100644 taskcluster/gecko_taskgraph/transforms/release_version_bump.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_l10n.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_partner.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_routes.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py create mode 100644 taskcluster/gecko_taskgraph/transforms/repo_update.py create mode 100644 taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py create mode 100644 taskcluster/gecko_taskgraph/transforms/reverse_chunk_deps.py create mode 100644 taskcluster/gecko_taskgraph/transforms/run_pgo_profile.py create mode 100644 taskcluster/gecko_taskgraph/transforms/scriptworker.py create mode 100644 taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py create mode 100644 taskcluster/gecko_taskgraph/transforms/sentry.py create mode 100644 taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/snap_test.py create mode 100644 taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py create mode 100644 taskcluster/gecko_taskgraph/transforms/source_test.py create mode 100644 taskcluster/gecko_taskgraph/transforms/spidermonkey.py create mode 100644 taskcluster/gecko_taskgraph/transforms/split_by_locale.py create mode 100644 taskcluster/gecko_taskgraph/transforms/startup_test.py create mode 100644 taskcluster/gecko_taskgraph/transforms/task.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/__init__.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/chunk.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/confirm_failure.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/other.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/raptor.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/variant.py create mode 100644 taskcluster/gecko_taskgraph/transforms/test/worker.py create mode 100644 taskcluster/gecko_taskgraph/transforms/trigger_comm_central.py create mode 100644 taskcluster/gecko_taskgraph/transforms/try_job.py create mode 100644 taskcluster/gecko_taskgraph/transforms/update_verify.py create mode 100644 taskcluster/gecko_taskgraph/transforms/update_verify_config.py create mode 100644 taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py create mode 100644 taskcluster/gecko_taskgraph/transforms/upload_symbols.py create mode 100644 taskcluster/gecko_taskgraph/transforms/upstream_artifact_task.py create mode 100644 taskcluster/gecko_taskgraph/try_option_syntax.py create mode 100644 taskcluster/gecko_taskgraph/util/__init__.py create mode 100644 taskcluster/gecko_taskgraph/util/attributes.py create mode 100644 taskcluster/gecko_taskgraph/util/backstop.py create mode 100644 taskcluster/gecko_taskgraph/util/bugbug.py create mode 100644 taskcluster/gecko_taskgraph/util/cached_tasks.py create mode 100644 taskcluster/gecko_taskgraph/util/chunking.py create mode 100644 taskcluster/gecko_taskgraph/util/copy_task.py create mode 100644 taskcluster/gecko_taskgraph/util/declarative_artifacts.py create mode 100644 taskcluster/gecko_taskgraph/util/dependencies.py create mode 100644 taskcluster/gecko_taskgraph/util/docker.py create mode 100644 taskcluster/gecko_taskgraph/util/hash.py create mode 100644 taskcluster/gecko_taskgraph/util/hg.py create mode 100644 taskcluster/gecko_taskgraph/util/partials.py create mode 100644 taskcluster/gecko_taskgraph/util/partners.py create mode 100644 taskcluster/gecko_taskgraph/util/perfile.py create mode 100644 taskcluster/gecko_taskgraph/util/platforms.py create mode 100644 taskcluster/gecko_taskgraph/util/scriptworker.py create mode 100644 taskcluster/gecko_taskgraph/util/signed_artifacts.py create mode 100644 taskcluster/gecko_taskgraph/util/taskcluster.py create mode 100644 taskcluster/gecko_taskgraph/util/taskgraph.py create mode 100644 taskcluster/gecko_taskgraph/util/templates.py create mode 100644 taskcluster/gecko_taskgraph/util/verify.py create mode 100644 taskcluster/gecko_taskgraph/util/workertypes.py create mode 100644 taskcluster/mach_commands.py create mode 100644 taskcluster/moz.build create mode 100755 taskcluster/scripts/builder/build-haz-linux.sh create mode 100755 taskcluster/scripts/builder/build-l10n.sh create mode 100755 taskcluster/scripts/builder/build-linux.sh create mode 100755 taskcluster/scripts/builder/build-sm-package.sh create mode 100755 taskcluster/scripts/builder/build-sm.sh create mode 100755 taskcluster/scripts/builder/repackage.sh create mode 100755 taskcluster/scripts/copy.sh create mode 100644 taskcluster/scripts/misc/afl-nyx.patch create mode 100755 taskcluster/scripts/misc/android-gradle-dependencies-lite.sh create mode 100755 taskcluster/scripts/misc/android-gradle-dependencies.sh create mode 100755 taskcluster/scripts/misc/android-gradle-dependencies/after.sh create mode 100755 taskcluster/scripts/misc/android-gradle-dependencies/before.sh create mode 100644 taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml create mode 100644 taskcluster/scripts/misc/are-we-esmified-yet.py create mode 100755 taskcluster/scripts/misc/browsertime.sh create mode 100755 taskcluster/scripts/misc/build-afl.sh create mode 100755 taskcluster/scripts/misc/build-binutils-linux.sh create mode 100755 taskcluster/scripts/misc/build-breakpad-injector.sh create mode 100755 taskcluster/scripts/misc/build-cctools-port.sh create mode 100755 taskcluster/scripts/misc/build-clang-mingw.sh create mode 100755 taskcluster/scripts/misc/build-clang-tidy-external.sh create mode 100755 taskcluster/scripts/misc/build-clang.sh create mode 100755 taskcluster/scripts/misc/build-compiler-rt-wasi.sh create mode 100755 taskcluster/scripts/misc/build-compiler-rt.sh create mode 100755 taskcluster/scripts/misc/build-cpython.sh create mode 100755 taskcluster/scripts/misc/build-custom-car.sh create mode 100755 taskcluster/scripts/misc/build-custom-v8.sh create mode 100755 taskcluster/scripts/misc/build-dist-toolchains.sh create mode 100755 taskcluster/scripts/misc/build-dmg-hfsplus.sh create mode 100755 taskcluster/scripts/misc/build-gcc-linux.sh create mode 100755 taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh create mode 100755 taskcluster/scripts/misc/build-geckodriver.sh create mode 100755 taskcluster/scripts/misc/build-gn-common.sh create mode 100755 taskcluster/scripts/misc/build-gn-linux.sh create mode 100755 taskcluster/scripts/misc/build-gn-macosx.sh create mode 100755 taskcluster/scripts/misc/build-gn-win64.sh create mode 100755 taskcluster/scripts/misc/build-hfsplus-linux.sh create mode 100755 taskcluster/scripts/misc/build-libunwind.sh create mode 100755 taskcluster/scripts/misc/build-llvm-common.sh create mode 100755 taskcluster/scripts/misc/build-llvm-symbolizer.sh create mode 100755 taskcluster/scripts/misc/build-mar-tools.sh create mode 100755 taskcluster/scripts/misc/build-mingw-fxc2-x86.sh create mode 100755 taskcluster/scripts/misc/build-mingw32-nsis.sh create mode 100755 taskcluster/scripts/misc/build-mkbom-linux.sh create mode 100755 taskcluster/scripts/misc/build-mozmake.sh create mode 100755 taskcluster/scripts/misc/build-msix-packaging.sh create mode 100755 taskcluster/scripts/misc/build-nasm.sh create mode 100755 taskcluster/scripts/misc/build-nodejs.sh create mode 100755 taskcluster/scripts/misc/build-nsis.sh create mode 100755 taskcluster/scripts/misc/build-pkgconf.sh create mode 100755 taskcluster/scripts/misc/build-resourcemonitor.sh create mode 100755 taskcluster/scripts/misc/build-rust-based-toolchain.sh create mode 100755 taskcluster/scripts/misc/build-sysroot-wasi.sh create mode 100755 taskcluster/scripts/misc/build-sysroot.sh create mode 100755 taskcluster/scripts/misc/build-upx.sh create mode 100755 taskcluster/scripts/misc/build-winchecksec.sh create mode 100755 taskcluster/scripts/misc/build-wine.sh create mode 100755 taskcluster/scripts/misc/build-xar-linux.sh create mode 100755 taskcluster/scripts/misc/build-xz.sh create mode 100644 taskcluster/scripts/misc/cargo-apk-Cargo.lock create mode 100644 taskcluster/scripts/misc/cctools.patch create mode 100755 taskcluster/scripts/misc/dummy.sh create mode 100644 taskcluster/scripts/misc/fetch-chromium.py create mode 100755 taskcluster/scripts/misc/fetch-content create mode 100755 taskcluster/scripts/misc/get_vs.py create mode 100644 taskcluster/scripts/misc/mingw-composition.patch create mode 100644 taskcluster/scripts/misc/mingw-dispatchqueue.patch create mode 100644 taskcluster/scripts/misc/mingw-dwrite_3.patch create mode 100644 taskcluster/scripts/misc/mingw-enum.patch create mode 100644 taskcluster/scripts/misc/mingw-ts_sd.patch create mode 100644 taskcluster/scripts/misc/mingw-unknown.patch create mode 100644 taskcluster/scripts/misc/mingw-widl.patch create mode 100644 taskcluster/scripts/misc/moz.build create mode 100755 taskcluster/scripts/misc/osx-cross-linker create mode 100755 taskcluster/scripts/misc/pack-cpython.sh create mode 100755 taskcluster/scripts/misc/pack.sh create mode 100755 taskcluster/scripts/misc/private_local_toolchain.sh create mode 100755 taskcluster/scripts/misc/repack-android-avd-linux.sh create mode 100755 taskcluster/scripts/misc/repack-android-emulator-linux.sh create mode 100755 taskcluster/scripts/misc/repack-android-ndk-linux.sh create mode 100755 taskcluster/scripts/misc/repack-android-sdk-linux.sh create mode 100755 taskcluster/scripts/misc/repack-android-system-images-linux.sh create mode 100755 taskcluster/scripts/misc/repack-clang.sh create mode 100755 taskcluster/scripts/misc/repack-jdk-linux.sh create mode 100755 taskcluster/scripts/misc/repack-node.sh create mode 100755 taskcluster/scripts/misc/repack_rust.py create mode 100755 taskcluster/scripts/misc/run-profileserver-macos.sh create mode 100755 taskcluster/scripts/misc/run-profileserver.sh create mode 100755 taskcluster/scripts/misc/source-test-clang-setup.sh create mode 100755 taskcluster/scripts/misc/source-test-common.sh create mode 100755 taskcluster/scripts/misc/source-test-infer-setup.sh create mode 100644 taskcluster/scripts/misc/summarize-tgdiff.py create mode 100644 taskcluster/scripts/misc/tooltool-download.sh create mode 100755 taskcluster/scripts/misc/unify.sh create mode 100644 taskcluster/scripts/misc/unpack-sdk.py create mode 100644 taskcluster/scripts/misc/verify-devtools-bundle.py create mode 100644 taskcluster/scripts/misc/vs-cleanup.sh create mode 100644 taskcluster/scripts/misc/vs-setup.sh create mode 100644 taskcluster/scripts/misc/wasi-sdk-11.patch create mode 100644 taskcluster/scripts/misc/wasi-sdk.patch create mode 100755 taskcluster/scripts/misc/wr-cargotest-macos-build.sh create mode 100755 taskcluster/scripts/misc/wr-macos-cross-build-setup.sh create mode 100755 taskcluster/scripts/misc/wrench-android-build.sh create mode 100755 taskcluster/scripts/misc/wrench-deps-vendoring.sh create mode 100755 taskcluster/scripts/misc/wrench-macos-build.sh create mode 100644 taskcluster/scripts/misc/wrench-windows-tests.sh create mode 100755 taskcluster/scripts/misc/zstdpy create mode 100755 taskcluster/scripts/run-task create mode 100755 taskcluster/scripts/tester/run-wizard create mode 100755 taskcluster/scripts/tester/test-linux.sh create mode 100644 taskcluster/test/conftest.py create mode 100644 taskcluster/test/data/automationrelevance.json create mode 100644 taskcluster/test/data/bugbug-push-schedules.json create mode 100644 taskcluster/test/data/pushes.json create mode 100644 taskcluster/test/params/autoland-onpush.yml create mode 100644 taskcluster/test/params/mb-onpush.yml create mode 100644 taskcluster/test/params/mb-promote-devedition-partials.yml create mode 100644 taskcluster/test/params/mb-promote-devedition.yml create mode 100644 taskcluster/test/params/mb-promote-firefox-partials.yml create mode 100644 taskcluster/test/params/mb-promote-firefox.yml create mode 100644 taskcluster/test/params/mb-push-devedition.yml create mode 100644 taskcluster/test/params/mb-push-firefox-partials.yml create mode 100644 taskcluster/test/params/mb-push-firefox.yml create mode 100644 taskcluster/test/params/mb-ship-devedition.yml create mode 100644 taskcluster/test/params/mb-ship-firefox-partials.yml create mode 100644 taskcluster/test/params/mb-ship-firefox.yml create mode 100644 taskcluster/test/params/mb-ship-geckoview.yml create mode 100644 taskcluster/test/params/mc-cron-system-symbols.yml create mode 100644 taskcluster/test/params/mc-desktop-nightly.yml create mode 100644 taskcluster/test/params/mc-onpush.yml create mode 100644 taskcluster/test/params/mc-ship-geckoview.yml create mode 100644 taskcluster/test/params/me-promote-firefox.yml create mode 100644 taskcluster/test/params/me-push-firefox.yml create mode 100644 taskcluster/test/params/me-ship-firefox.yml create mode 100644 taskcluster/test/params/mr-onpush-geckoview.yml create mode 100644 taskcluster/test/params/mr-onpush.yml create mode 100644 taskcluster/test/params/mr-promote-firefox-rc.yml create mode 100644 taskcluster/test/params/mr-promote-firefox.yml create mode 100644 taskcluster/test/params/mr-push-firefox.yml create mode 100644 taskcluster/test/params/mr-ship-firefox-rc.yml create mode 100644 taskcluster/test/params/mr-ship-firefox.yml create mode 100644 taskcluster/test/params/mr-ship-geckoview.yml create mode 100644 taskcluster/test/params/try.yml create mode 100644 taskcluster/test/params/update.sh create mode 100644 taskcluster/test/python.toml create mode 100644 taskcluster/test/test_autoland.py create mode 100644 taskcluster/test/test_autoland_backstop.py create mode 100644 taskcluster/test/test_generate_params.py create mode 100644 taskcluster/test/test_mach_try_auto.py create mode 100644 taskcluster/test/test_mozilla_central.py create mode 100644 taskcluster/test/test_new_config.py (limited to 'taskcluster') diff --git a/taskcluster/.yamllint b/taskcluster/.yamllint new file mode 100644 index 0000000000..8b464b1e2d --- /dev/null +++ b/taskcluster/.yamllint @@ -0,0 +1,16 @@ +--- +ignore: | + *node_modules* + **/test/params/ + +extends: default + +rules: + document-end: + present: false + document-start: + present: true + indentation: + spaces: 4 + # Checks currently failing + line-length: disable diff --git a/taskcluster/ci/addon/kind.yml b/taskcluster/ci/addon/kind.yml new file mode 100644 index 0000000000..84d616581b --- /dev/null +++ b/taskcluster/ci/addon/kind.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + # non-system python + - toolchain + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + use-system-python: false + +jobs: + tps-xpi: + description: Build the TPS add-on + index: + product: firefox + job-name: addons.tps + treeherder: + platform: linux64/opt + symbol: TPS(addon) + kind: build + tier: 1 + run-on-projects: [mozilla-central] + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: debian12-base} + max-run-time: 1800 + artifacts: + - type: file + name: public/tps.xpi + path: /builds/worker/checkouts/gecko/tps-out/tps.xpi + run: + using: mach + mach: tps-build --dest tps-out + sparse-profile: tps + when: + files-changed: + - 'services/sync/tps/extensions/tps/**' diff --git a/taskcluster/ci/are-we-esmified-yet/kind.yml b/taskcluster/ci/are-we-esmified-yet/kind.yml new file mode 100644 index 0000000000..f2c462f455 --- /dev/null +++ b/taskcluster/ci/are-we-esmified-yet/kind.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + check: + description: Extracts statistics on remaining JSMs in the codebase + run-on-projects: [] + attributes: + cron: true + index: + product: firefox + job-name: are-we-esmified-yet + treeherder: + platform: lint/opt + kind: other + symbol: js(esm) + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 3600 + artifacts: + - type: file + name: public/are-we-esmified-yet.json + path: /builds/worker/artifacts/are-we-esmified-yet.json + run: + using: run-task + command: >- + mkdir -p /builds/worker/artifacts && + cd ${GECKO_PATH} && + python3 ./taskcluster/scripts/misc/are-we-esmified-yet.py > /builds/worker/artifacts/are-we-esmified-yet.json diff --git a/taskcluster/ci/artifact-build/kind.yml b/taskcluster/ci/artifact-build/kind.yml new file mode 100644 index 0000000000..aa15674935 --- /dev/null +++ b/taskcluster/ci/artifact-build/kind.yml @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + - toolchain + +transforms: + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + index: + product: firefox + treeherder: + kind: build + symbol: AB + tier: 2 + run-on-projects: ['mozilla-central'] + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + env: + PERFHERDER_EXTRA_OPTIONS: artifact + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + USE_ARTIFACT: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + keep-artifacts: false + use-system-python: false + +jobs: + linux64-artifact/opt: + description: "Linux64 Opt Artifact Build" + index: + job-name: linux64-artifact-opt + treeherder: + platform: linux64/opt + worker: + env: + MOZ_ARTIFACT_TASK: {task-reference: ''} + run: + config: + - builds/releng_base_linux_64_builds.py + dependencies: + linux64-opt: build-linux64/opt + fetches: + toolchain: + - linux64-node + + macosx64-artifact/opt: + description: "MacOS X x64 Opt Artifact Build" + index: + job-name: macosx64-artifact-opt + treeherder: + platform: osx-cross/opt + worker: + env: + MOZ_ARTIFACT_TASK: {task-reference: ''} + run: + config: + - builds/releng_base_mac_64_cross_builds.py + dependencies: + macosx64-opt: build-macosx64/opt + fetches: + toolchain: + - linux64-hfsplus + - linux64-libdmg + - linux64-node + + win64-artifact/opt: + description: "Win64 Opt Artifact Build" + index: + job-name: win64-artifact-opt + treeherder: + platform: windows2012-64/opt + worker: + env: + MOZ_ARTIFACT_TASK: {task-reference: ''} + run: + config: + - builds/releng_base_linux_64_builds.py + extra-config: + mozconfig_platform: win64 + dependencies: + win64-opt: build-win64/opt + fetches: + toolchain: + - linux64-node + - linux64-wine + - nsis + fetch: + - upx-3.95-win diff --git a/taskcluster/ci/attribution-l10n/kind.yml b/taskcluster/ci/attribution-l10n/kind.yml new file mode 100644 index 0000000000..7042411fa2 --- /dev/null +++ b/taskcluster/ci/attribution-l10n/kind.yml @@ -0,0 +1,187 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms + - gecko_taskgraph.transforms.split_by_locale:transforms + - taskgraph.transforms.task_context + - gecko_taskgraph.transforms.attribution:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + # Windows + - repackage-signing-l10n + # Mac + - repackage-l10n + +job-defaults: + description: Attribute release builds + shipping-phase: promote + # never run as part of CI + run-on-projects: [] + worker-type: b-linux-gcp + locales-file: browser/locales/l10n-changesets.json + properties-with-locale: + by-build-platform: + win.*: + - label + - dependencies.repackage-signing-l10n + - treeherder.symbol + - fetches.repackage-signing-l10n + mac.*: + - label + - dependencies.repackage-l10n + - treeherder.symbol + - fetches.repackage-l10n + attributes: + build_type: opt + release_artifacts: + by-build-platform: + win.*: + - public/build/target.installer.exe + mac.*: + - public/build/target.dmg + treeherder: + symbol: Attr-L10n({locale}) + kind: other + tier: 1 + fetches: + by-build-platform: + win.*: + repackage-signing-l10n: + - "{locale}/target.installer.exe" + mac.*: + repackage-l10n: + - "{locale}/target.dmg" + toolchain: + - "linux64-libdmg" + task-context: + from-file: browser/installer/attribution.yml + substitution-fields: + - run.command + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts + docker-image: + in-tree: "partner-repack" + chain-of-trust: true + # Each task only processes one installer...this should be quick. + max-run-time: 600 + run: + using: run-task + cwd: "{checkout}" + command: + by-build-platform: + win.*: + - ./mach + - python + - python/mozrelease/mozrelease/attribute_builds.py + - --attribution + - dlsource={attribution_code[json][dlsource]} + - --output + - /builds/worker/artifacts + # All jobs require attribution of target.installer.exe. Some may require + # attribution of other files, which they can append themselves. + - --input + - /builds/worker/fetches/target.installer.exe + mac.*: + - mkdir -p /builds/worker/artifacts && + - /builds/worker/fetches/dmg/dmg + - attribute + - /builds/worker/fetches/target.dmg + - /builds/worker/artifacts/target.dmg + - __MOZCUSTOM__ + - __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]} + use-system-python: false + +jobs: + win32-devedition/opt: + label: attribution-win32-{locale}-devedition/opt + shipping-product: devedition + attributes: + build_platform: win32-devedition + treeherder: + platform: win32-devedition/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win32-devedition/opt + + win32-shippable/opt: + label: attribution-win32-{locale}-shippable/opt + shipping-product: firefox + locale-file-platform: win32 + attributes: + build_platform: win32-shippable + treeherder: + platform: win32-shippable/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win32-shippable/opt + + win64-devedition/opt: + label: attribution-win64-{locale}-devedition/opt + shipping-product: devedition + attributes: + build_platform: win64-devedition + treeherder: + platform: win64-devedition/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-devedition/opt + + win64-shippable/opt: + label: attribution-win64-{locale}-shippable/opt + shipping-product: firefox + locale-file-platform: win64 + attributes: + build_platform: win64-shippable + treeherder: + platform: win64-shippable/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-shippable/opt + + win64-aarch64-shippable/opt: + label: attribution-win64-aarch64-{locale}-shippable/opt + shipping-product: firefox + locale-file-platform: win64-aarch64 + attributes: + build_platform: win64-aarch64-shippable + treeherder: + platform: win64-aarch64-shippable/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-aarch64-shippable/opt + + win64-aarch64-devedition/opt: + label: attribution-win64-aarch64-{locale}-devedition/opt + shipping-product: devedition + attributes: + build_platform: win64-aarch64-devedition + treeherder: + platform: win64-aarch64-devedition/opt + dependencies: + repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-aarch64-devedition/opt + + macosx64-devedition/opt: + label: attribution-macosx64-{locale}-devedition/opt + shipping-product: devedition + attributes: + build_platform: macosx64-devedition + treeherder: + platform: macosx64-devedition/opt + dependencies: + repackage-l10n: repackage-l10n-{locale}-macosx64-devedition/opt + + macosx64-shippable/opt: + label: attribution-macosx64-{locale}-shippable/opt + locale-file-platform: macosx64 + shipping-product: firefox + attributes: + build_platform: macosx64-shippable + treeherder: + platform: macosx64-shippable/opt + dependencies: + repackage-l10n: repackage-l10n-{locale}-macosx64-shippable/opt diff --git a/taskcluster/ci/attribution/kind.yml b/taskcluster/ci/attribution/kind.yml new file mode 100644 index 0000000000..6a13ee5096 --- /dev/null +++ b/taskcluster/ci/attribution/kind.yml @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms + - taskgraph.transforms.task_context + - gecko_taskgraph.transforms.attribution:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + # Windows + - repackage-signing + # Mac + - repackage + +job-defaults: + description: Attribute release builds + shipping-phase: promote + # never run as part of CI + run-on-projects: [] + worker-type: b-linux-gcp + fetches: + by-build-platform: + win.*: + repackage-signing: + - target.installer.exe + mac.*: + repackage: + - target.dmg + toolchain: + - linux64-libdmg + attributes: + release_artifacts: + by-build-platform: + win.*: + - public/build/target.installer.exe + mac.*: + - public/build/target.dmg + treeherder: + symbol: Attr + kind: other + tier: 1 + task-context: + from-file: browser/installer/attribution.yml + substitution-fields: + - run.command + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts + docker-image: + in-tree: "partner-repack" + chain-of-trust: true + # Each task only processes one installer...this should be quick. + max-run-time: 600 + run: + using: run-task + cwd: "{checkout}" + command: + by-build-platform: + win.*: + - ./mach + - python + - python/mozrelease/mozrelease/attribute_builds.py + - --attribution + - dlsource={attribution_code[json][dlsource]} + - --output + - /builds/worker/artifacts + # All jobs require attribution of target.installer.exe. Some may require + # attribution of other files, which they can append themselves. + - --input + - /builds/worker/fetches/target.installer.exe + mac.*: + - mkdir -p /builds/worker/artifacts && + - /builds/worker/fetches/dmg/dmg + - attribute + - /builds/worker/fetches/target.dmg + - /builds/worker/artifacts/target.dmg + - __MOZCUSTOM__ + - __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]} + use-system-python: false + +jobs: + win32-devedition/opt: + label: attribution-win32-devedition/opt + shipping-product: devedition + attributes: + build_platform: win32-devedition + build_type: opt + treeherder: + platform: win32-devedition/opt + dependencies: + repackage-signing: repackage-signing-win32-devedition/opt + + win32-shippable/opt: + label: attribution-win32-shippable/opt + shipping-product: firefox + attributes: + build_platform: win32-shippable + build_type: opt + treeherder: + platform: win32-shippable/opt + dependencies: + repackage-signing: repackage-signing-win32-shippable/opt + + win64-devedition/opt: + label: attribution-win64-devedition/opt + shipping-product: devedition + attributes: + build_platform: win64-devedition + build_type: opt + treeherder: + platform: win64-devedition/opt + dependencies: + repackage-signing: repackage-signing-win64-devedition/opt + + win64-shippable/opt: + label: attribution-win64-shippable/opt + shipping-product: firefox + attributes: + build_platform: win64-shippable + build_type: opt + treeherder: + platform: win64-shippable/opt + dependencies: + repackage-signing: repackage-signing-win64-shippable/opt + + win64-aarch64-shippable/opt: + label: attribution-win64-aarch64-shippable/opt + shipping-product: firefox + attributes: + build_platform: win64-aarch64-shippable + build_type: opt + treeherder: + platform: win64-aarch64-shippable/opt + dependencies: + repackage-signing: repackage-signing-win64-aarch64-shippable/opt + + win64-aarch64-devedition/opt: + label: attribution-win64-aarch64-devedition/opt + shipping-product: devedition + attributes: + build_platform: win64-aarch64-devedition + build_type: opt + treeherder: + platform: win64-aarch64-devedition/opt + dependencies: + repackage-signing: repackage-signing-win64-aarch64-devedition/opt + + macosx64-devedition/opt: + label: attribution-macosx64-devedition/opt + shipping-product: devedition + attributes: + build_platform: macosx64-devedition + build_type: opt + treeherder: + platform: macosx64-devedition/opt + dependencies: + repackage: repackage-macosx64-devedition/opt + + macosx64-shippable/opt: + label: attribution-macosx64-shippable/opt + shipping-product: firefox + attributes: + build_platform: macosx64-shippable + build_type: opt + treeherder: + platform: macosx64-shippable/opt + dependencies: + repackage: repackage-macosx64-shippable/opt diff --git a/taskcluster/ci/balrog/kind.yml b/taskcluster/ci/balrog/kind.yml new file mode 100644 index 0000000000..324ef2c521 --- /dev/null +++ b/taskcluster/ci/balrog/kind.yml @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.balrog_submit + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.task + +kind-dependencies: + - beetmover-repackage + - startup-test + +only-for-attributes: + - nightly + - shippable + +not-for-build-platforms: + - android-arm-shippable/opt + - android-x86_64-shippable/opt + - android-x86-shippable/opt + - android-aarch64-shippable/opt + +jobs: + balrog: + from-deps: + group-by: single-with-filters + update-no-wnp: + by-release-type: + nightly.*: false + default: true diff --git a/taskcluster/ci/beetmover-apt/kind.yml b/taskcluster/ci/beetmover-apt/kind.yml new file mode 100644 index 0000000000..9eb65f8858 --- /dev/null +++ b/taskcluster/ci/beetmover-apt/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.beetmover_apt + - gecko_taskgraph.transforms.task + +kind-dependencies: + - beetmover-repackage + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux-devedition/opt + - linux64-devedition/opt + +jobs: + beetmover-apt: + from-deps: + group-by: single-with-filters + run-on-releases: + - nightly + - beta + - release diff --git a/taskcluster/ci/beetmover-checksums/kind.yml b/taskcluster/ci/beetmover-checksums/kind.yml new file mode 100644 index 0000000000..7ecf79ea58 --- /dev/null +++ b/taskcluster/ci/beetmover-checksums/kind.yml @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_checksums + - gecko_taskgraph.transforms.task + +kind-dependencies: + - beetmover-repackage + +only-for-attributes: + - nightly + - shippable + +jobs: + beetmover-checksums: + from-deps: + group-by: single-with-filters + shipping-phase: promote + attributes: + artifact_prefix: public + artifact_map: + by-release-type: + beta|release.*|esr.*: taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml + default: taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml diff --git a/taskcluster/ci/beetmover-geckoview/kind.yml b/taskcluster/ci/beetmover-geckoview/kind.yml new file mode 100644 index 0000000000..109b42eb6a --- /dev/null +++ b/taskcluster/ci/beetmover-geckoview/kind.yml @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_geckoview + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build-signing + +only-for-attributes: + - nightly + - shippable + +not-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux64-nightlyasrelease/opt + - macosx64-shippable/opt + - macosx64-nightlyasrelease/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - win64-nightlyasrelease/opt + - linux-devedition/opt + - linux64-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + - linux64-asan-reporter-shippable/opt + - win64-asan-reporter-shippable/opt + +jobs: + beetmover-geckoview: + from-deps: + group-by: single-with-filters + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml + run-on-projects: ['mozilla-release'] + run-on-hg-branches: + by-project: + mozilla-release: + - '^GECKOVIEW_\d+_RELBRANCH$' + default: + - 'all' + shipping-phase: + by-project: + # Beetmoving geckoview makes it available to the official maven repo. + # That happens: + # 1. as part of the ship-geckoview graph on mozilla-central + # 2. as part of Firefox promote graphs on both mozilla-beta and mozilla-release + # 3. at every patch uplifted on the GECKOVIEW_XX_RELBRANCH on mozilla-release + # Reminder: There is no Android/geckoview build on ESR. + # + # The logic of point 2 is handled in target_tasks directly because it's mixing and + # matching 2 different shipping-products + mozilla-release: build + default: promote + bucket-scope: + by-release-level: + production: 'project:releng:beetmover:bucket:maven-production' + staging: 'project:releng:beetmover:bucket:maven-staging' diff --git a/taskcluster/ci/beetmover-repackage/kind.yml b/taskcluster/ci/beetmover-repackage/kind.yml new file mode 100644 index 0000000000..8746bb5b44 --- /dev/null +++ b/taskcluster/ci/beetmover-repackage/kind.yml @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_repackage_l10n + - gecko_taskgraph.transforms.beetmover_repackage + - gecko_taskgraph.transforms.task + +kind-dependencies: + # Primary dependencies + - repackage + - repackage-l10n + # Other dependencies + - build + - build-signing + - build-mac-signing + - build-mac-notarization + - repackage-signing + - shippable-l10n + - shippable-l10n-signing + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + - repackage-signing-l10n + - partials + - partials-signing + - repackage-signing-msi + - repackage-signing-shippable-l10n-msix + - mar-signing + - mar-signing-l10n + - attribution + - attribution-l10n + - repackage-deb + - repackage-deb-l10n + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - linux-devedition/opt + - linux64-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + - linux64-asan-reporter-shippable/opt + - win64-asan-reporter-shippable/opt + +jobs: + beetmover-repackage: + from-deps: + group-by: single-locale + shipping-phase: promote + attributes: + artifact_map: + by-release-type: + beta|release.*|esr.*: taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml + default: taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml diff --git a/taskcluster/ci/beetmover-source/kind.yml b/taskcluster/ci/beetmover-source/kind.yml new file mode 100644 index 0000000000..cd86d812b5 --- /dev/null +++ b/taskcluster/ci/beetmover-source/kind.yml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover + - gecko_taskgraph.transforms.beetmover_source + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-source-signing + +jobs: + beetmover-source: + from-deps: + group-by: single-with-filters + shipping-phase: promote + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/source_files.yml diff --git a/taskcluster/ci/bootstrap/kind.yml b/taskcluster/ci/bootstrap/kind.yml new file mode 100644 index 0000000000..0ce8f8f68a --- /dev/null +++ b/taskcluster/ci/bootstrap/kind.yml @@ -0,0 +1,103 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.bootstrap:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + debian11: + image: debian:bullseye + pre-commands: + - apt-get update + - apt-get install -y --no-install-recommends curl mercurial python3-pip + debian12: + image: debian:bookworm + pre-commands: + - apt-get update + - apt-get install -y --no-install-recommends curl mercurial python3-pip + debian13: + image: debian:trixie + pre-commands: + - apt-get update + - apt-get install -y --no-install-recommends curl mercurial python3-pip + ubuntu2204: + image: ubuntu:jammy + pre-commands: + - apt-get update + - apt-get install -y --no-install-recommends curl mercurial python3-pip + ubuntu2304: + image: ubuntu:lunar + pre-commands: + - apt-get update + - apt-get install -y --no-install-recommends curl mercurial python3-pip + ubuntu2310: + image: ubuntu:mantic + pre-commands: + - apt-get update + # For some reason, this is required, now. + - export DEBIAN_FRONTEND=noninteractive + - apt-get install -y --no-install-recommends curl mercurial python3-pip + fedora36: + image: fedora:36 + pre-commands: + - dnf install -y mercurial python3-pip + fedora37: + image: fedora:37 + pre-commands: + - dnf install -y mercurial python3-pip + # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6761 + - mkdir -p /etc/mercurial/hgrc.d + - (echo "[devel]"; echo discovery.randomize=false) > /etc/mercurial/hgrc.d/workaround.rc + fedora38: + image: fedora:38 + pre-commands: + - dnf install -y mercurial python3-pip + fedora39: + image: fedora:39 + pre-commands: + - dnf install -y mercurial python3-pip + rockylinux8: + image: rockylinux:8 + pre-commands: + - dnf install -y mercurial python38-pip + rockylinux9: + image: rockylinux:9 + pre-commands: + # Mercurial is in EPEL. https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + - dnf install -y epel-release + - dnf install -y mercurial python3-pip + opensuse-leap: + image: opensuse/leap + pre-commands: + - zypper install -y curl mercurial python311-pip python311-curses gzip + # We can't replace /usr/bin/python3 because mercurial uses that, so + # we override it via /usr/local/bin. + - ln -s /usr/bin/python3.11 /usr/local/bin/python3 + # Make the shell forget about where it found python3 in the past. + - hash -r python3 + opensuse-tumbleweed: + image: opensuse/tumbleweed + pre-commands: + - zypper install -y mercurial python3-pip + gentoo: + image: gentoo/stage3 + pre-commands: + - emerge --sync + - emerge mercurial dev-python/pip + archlinux: + image: archlinux + pre-commands: + - pacman -Syu --noconfirm mercurial python-pip + voidlinux: + image: voidlinux/voidlinux + pre-commands: + - sed -i s/alpha.de.repo.voidlinux.org/repo-default.voidlinux.org/g /usr/share/xbps.d/* + - xbps-install -Suyv xbps + - xbps-install -Suyv curl mercurial python3-pip diff --git a/taskcluster/ci/bouncer-locations/kind.yml b/taskcluster/ci/bouncer-locations/kind.yml new file mode 100644 index 0000000000..4234939369 --- /dev/null +++ b/taskcluster/ci/bouncer-locations/kind.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-beetmover-dummy + +transforms: + - gecko_taskgraph.transforms.bouncer_locations:transforms + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: nightly bouncer locations job + attributes: + shippable: true + worker-type: bouncer + worker: + implementation: bouncer-locations + scopes: + by-project: + mozilla-central: + - project:releng:bouncer:action:locations + - project:releng:bouncer:server:production + default: + - project:releng:bouncer:action:locations + - project:releng:bouncer:server:staging + shipping-phase: ship + treeherder: + symbol: BncLoc + kind: other + tier: 1 + +jobs: + firefox: + run-on-releases: ['nightly'] + run-on-projects: ['mozilla-central'] + shipping-product: firefox + bouncer-products: + - firefox-nightly-latest + - firefox-nightly-latest-ssl + - firefox-nightly-latest-l10n + - firefox-nightly-latest-l10n-ssl + - firefox-nightly-msi-latest-ssl + - firefox-nightly-msi-latest-l10n-ssl + - firefox-nightly-pkg-latest-ssl + - firefox-nightly-pkg-latest-l10n-ssl + treeherder: + platform: firefox-release/opt diff --git a/taskcluster/ci/build-fat-aar/kind.yml b/taskcluster/ci/build-fat-aar/kind.yml new file mode 100644 index 0000000000..3d45e76d79 --- /dev/null +++ b/taskcluster/ci/build-fat-aar/kind.yml @@ -0,0 +1,200 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - build + - toolchain + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.build_fat_aar:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml + index: + product: mobile + worker-type: b-linux-gcp + use-system-python: false + worker: + docker-image: {in-tree: android-build} + max-run-time: 7200 + env: + # Online in order to download the per-architecture AARs. + GRADLE_USER_HOME: "/builds/worker/checkouts/build/gecko/mobile/android/gradle/dotgradle-online" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_ANDROID_FAT_AAR_ARCHITECTURES: "armeabi-v7a,arm64-v8a,x86,x86_64" + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + artifacts: + - name: public/build/maven + path: /builds/worker/workspace/obj-build/gradle/maven/ + type: directory + - name: public/build/target.maven.tar.xz + path: /builds/worker/workspace/obj-build/gradle/target.maven.tar.xz + type: file + - name: public/build + path: /builds/worker/artifacts/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview-test_runner.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/bundle/withGeckoBinariesDebug/test_runner-withGeckoBinaries-debug.aab + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/bundle/withGeckoBinariesDebug/geckoview_example-withGeckoBinaries-debug.aab + type: file + run: + using: mozharness + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: null + tooltool-downloads: internal + # Note: These settings are only honored by nightly (i.e. shipping) builds + update-channel: + by-release-type: + nightly: + by-project: + try: nightly-try + default: nightly + nightly-larch: nightly-larch + nightly-oak: nightly-oak + nightly-pine: nightly-pine + beta: + by-shipping-product: + devedition: aurora + default: beta + release.*: release + esr.*: esr + default: null + fetches: + toolchain: + - android-ndk-linux + - android-sdk-linux + - linux64-clang + - linux64-jdk + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +jobs: + android-geckoview-fat-aar-shippable/opt: + description: "Android GeckoView multi-architecture fat AAR Shippable" + attributes: + shippable: true + enable-full-crashsymbols: true + disable-push-apk: true + maven_packages: + - geckoview + shipping-phase: build + shipping-product: fennec + index: + job-name: android-geckoview-fat-aar-opt + type: shippable-with-multi-l10n + treeherder: + platform: android-5-0-geckoview-fat-aar-shippable/opt + symbol: Bgv + dependencies: + android-x86-shippable: build-android-x86-shippable/opt + android-x86_64-shippable: build-android-x86_64-shippable/opt + android-arm-shippable: build-android-arm-shippable/opt + android-aarch64-shippable: build-android-aarch64-shippable/opt + worker: + env: + PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-nightly + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: arm + fetches: + toolchain: + - android-gradle-dependencies + optimization: + skip-unless-backstop: null + + android-geckoview-fat-aar-shippable-lite/opt: + description: "Android GeckoView Lite multi-architecture fat AAR Shippable" + attributes: + shippable: true + enable-full-crashsymbols: true + disable-push-apk: true + maven_packages: + - geckoview + - geckoview-exoplayer2 + shipping-phase: build + shipping-product: fennec + index: + job-name: android-geckoview-fat-aar-opt-lite + type: shippable-with-multi-l10n + treeherder: + platform: android-4-1-geckoview-fat-aar-shippable/opt + symbol: Bgvl + dependencies: + android-x86-shippable-lite: build-android-x86-shippable-lite/opt + android-x86_64-shippable-lite: build-android-x86_64-shippable-lite/opt + android-arm-shippable-lite: build-android-arm-shippable-lite/opt + android-aarch64-shippable-lite: build-android-aarch64-shippable-lite/opt + worker: + env: + PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-nightly-lite + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: arm-lite + fetches: + toolchain: + - android-gradle-dependencies-lite + optimization: + skip-unless-backstop: null + + android-geckoview-fat-aar/opt: + description: "Android GeckoView multi-architecture fat AAR Opt" + attributes: + maven_packages: + - geckoview + index: + job-name: android-geckoview-fat-aar-opt + treeherder: + platform: android-5-0-geckoview-fat-aar/opt + symbol: Bgv + dependencies: + android-x86-opt: build-android-x86/opt + android-x86_64-opt: build-android-x86_64/opt + android-arm-opt: build-android-arm/opt + android-aarch64-opt: build-android-aarch64/opt + worker: + env: + PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt + USE_ARTIFACT: '1' + MOZ_ARTIFACT_TASK: {task-reference: ''} + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: arm + fetches: + toolchain: + - linux64-sccache + - android-gradle-dependencies diff --git a/taskcluster/ci/build-mac-notarization/kind.yml b/taskcluster/ci/build-mac-notarization/kind.yml new file mode 100644 index 0000000000..8d049b3695 --- /dev/null +++ b/taskcluster/ci/build-mac-notarization/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.mac_notarization + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build-mac-signing + +only-for-attributes: + - shippable + +jobs: + build-mac-notarization: + from-deps: + group-by: single-with-filters + copy-attributes: true + treeherder: + symbol: BMN + upstream-artifacts: + - taskType: signing + paths: + - public/build/target.tar.gz + - public/build/target.pkg + formats: ["apple_notarization"] + taskId: + task-reference: diff --git a/taskcluster/ci/build-mac-signing/kind.yml b/taskcluster/ci/build-mac-signing/kind.yml new file mode 100644 index 0000000000..d90813cd96 --- /dev/null +++ b/taskcluster/ci/build-mac-signing/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.build_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.hardened_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + +only-for-attributes: + - enable-build-signing + +only-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + - macosx64/opt + - macosx64/debug + - macosx64-aarch64/debug + +jobs: + build-mac-signing: + from-deps: + group-by: single-with-filters + treeherder: + symbol: BMS + enable-signing-routes: false diff --git a/taskcluster/ci/build-signing/kind.yml b/taskcluster/ci/build-signing/kind.yml new file mode 100644 index 0000000000..7e4bbab419 --- /dev/null +++ b/taskcluster/ci/build-signing/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps:transforms + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.upstream_artifact_task + - gecko_taskgraph.transforms.build_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + - build-fat-aar + +only-for-attributes: + - nightly + - shippable + - enable-build-signing + +not-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + - macosx64/opt + - macosx64/debug + - macosx64-aarch64/debug + +jobs: + build-signing: + from-deps: + group-by: platform diff --git a/taskcluster/ci/build/android-asan.yml b/taskcluster/ci/build/android-asan.yml new file mode 100644 index 0000000000..4cc846dece --- /dev/null +++ b/taskcluster/ci/build/android-asan.yml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml + maven_packages: + - geckoview + index: + product: mobile + worker: + docker-image: {in-tree: android-build} + max-run-time: 7200 + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + artifacts: + - name: public/build/maven + path: /builds/worker/workspace/obj-build/gradle/maven/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build + path: /builds/worker/artifacts/ + type: directory + run: + using: mozharness + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: null + tooltool-downloads: internal + fetches: + toolchain: + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +android-x86_64-asan-fuzzing/opt: + description: "Android 5.0 x86-64 Fuzzing Opt ASAN" + index: + job-name: android-x86_64-fuzzing-asan + treeherder: + platform: android-5-0-x86_64/asan + symbol: Bof + worker-type: b-linux-large-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: fuzzing-asan-tc + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + optimization: + skip-unless-expanded: null diff --git a/taskcluster/ci/build/android-stuff.yml b/taskcluster/ci/build/android-stuff.yml new file mode 100644 index 0000000000..52bed7b18f --- /dev/null +++ b/taskcluster/ci/build/android-stuff.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + tags: + android-stuff: "true" + run: + mozconfig-variant: null + +android-geckoview-docs/opt: + description: "Android GeckoView docs" + index: + product: mobile + job-name: android-geckoview-docs + treeherder: + platform: android-4-1-armv7/opt + kind: build + tier: 2 + symbol: A(gv-docs) + run-on-projects: [mozilla-central-only] + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + PERFHERDER_EXTRA_OPTIONS: android-geckoview-docs + GECKOVIEW_DOCS_UPLOAD_SECRET: "project/releng/gecko/build/level-{level}/geckoview-docs-upload" + artifacts: + - name: public/android/geckoview-docs/geckoview-javadoc.jar + path: /builds/worker/workspace/obj-build/mobile/android/geckoview/docs/javadoc/withGeckoBinaries-debug/geckoview-javadoc.jar + type: file + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_android_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: android-geckoview-docs + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-sdk-linux + - linux64-jdk + - linux64-node + optimization: + skip-unless-changed: + - "mobile/android/**/*.java" + - "mobile/android/gradle.configure" + - "**/*.gradle" diff --git a/taskcluster/ci/build/android.yml b/taskcluster/ci/build/android.yml new file mode 100644 index 0000000000..7c37b2680f --- /dev/null +++ b/taskcluster/ci/build/android.yml @@ -0,0 +1,644 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml + maven_packages: + - geckoview + index: + product: mobile + worker: + docker-image: {in-tree: android-build} + max-run-time: 7200 + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + artifacts: + - name: public/build/maven + path: /builds/worker/workspace/obj-build/gradle/maven/ + type: directory + - name: public/build/target.maven.tar.xz + path: /builds/worker/workspace/obj-build/gradle/target.maven.tar.xz + type: file + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview-test_runner.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/bundle/withGeckoBinariesDebug/test_runner-withGeckoBinaries-debug.aab + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/bundle/withGeckoBinariesDebug/geckoview_example-withGeckoBinaries-debug.aab + type: file + - name: public/build + path: /builds/worker/artifacts/ + type: directory + run: + using: mozharness + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: null + tooltool-downloads: internal + fetches: + toolchain: + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +android-arm/debug: + description: "Android 5.0 Arm Debug" + index: + job-name: android-arm-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-armv7/debug + symbol: B + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: arm-debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-x86/opt: + description: "Android 5.0 x86 Opt" + index: + job-name: android-x86-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-x86/opt + symbol: B + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-x86-lite/opt: + description: "Android 4.1 x86 Opt" + index: + job-name: android-x86-lite-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-4-1-x86/opt + symbol: Bl + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + PERFHERDER_EXTRA_OPTIONS: lite + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86-lite + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies-lite + +android-x86-shippable/opt: + description: "Android 5.0 x86 Shippable" + use-pgo: true + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-x86-opt + type: android-shippable + treeherder: + platform: android-5-0-x86-shippable/opt + symbol: B + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: x86 + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies + +android-x86-shippable-lite/opt: + description: "Android 4.1 x86 Shippable Lite" + use-pgo: android-x86-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-x86-lite-opt + type: android-shippable + treeherder: + platform: android-4-1-x86-shippable-lite/opt + symbol: Bl + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: x86-lite + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies-lite + +android-arm/opt: + description: "Android 5.0 Arm Opt" + index: + job-name: android-arm-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-armv7/opt + symbol: B + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: arm + use-sccache: true + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies + - linux64-sccache + +android-arm-lite/opt: + description: "Android 4.1 Arm Opt Lite" + index: + job-name: android-arm-lite-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-4-1-armv7/opt + symbol: Bl + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: arm-lite + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies-lite + +android-arm-shippable/opt: + description: "Android 5.0 Arm Shippable" + use-pgo: android-x86-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-arm-opt + type: android-shippable-with-multi-l10n + treeherder: + platform: android-5-0-armv7-shippable/opt + symbol: Bpgo(B) + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: arm + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies + +android-arm-shippable-lite/opt: + description: "Android 4.1 Arm Shippable Lite" + use-pgo: android-x86-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-arm-lite-opt + type: android-shippable-with-multi-l10n + treeherder: + platform: android-4-1-armv7-shippable-lite/opt + symbol: Bpgo(Bl) + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: arm-lite + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies-lite + +android-aarch64/opt: + description: "Android 5.0 AArch64 Opt" + index: + job-name: android-aarch64-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-aarch64/opt + symbol: B + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: aarch64 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-aarch64-lite/opt: + description: "Android 5.0 AArch64 Opt Lite" + index: + job-name: android-aarch64-lite-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-aarch64/opt + symbol: Bl + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + PERFHERDER_EXTRA_OPTIONS: lite + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: aarch64-lite + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies-lite + +android-aarch64/debug: + description: "Android 5.0 AArch64 Debug" + index: + job-name: android-aarch64-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-aarch64/debug + symbol: B + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: aarch64-debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-aarch64-shippable/opt: + description: "Android 5.0 AArch64 Shippable" + use-pgo: android-aarch64-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-aarch64-opt + type: android-shippable + treeherder: + platform: android-5-0-aarch64-shippable/opt + symbol: Bpgo(B) + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: aarch64 + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies + +android-aarch64-shippable-lite/opt: + description: "Android 5.0 AArch64 Shippable Lite" + use-pgo: android-aarch64-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-aarch64-lite-opt + type: android-shippable + treeherder: + platform: android-5-0-aarch64-shippable-lite/opt + symbol: Bpgo(Bl) + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: lite + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: aarch64-lite + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies-lite + +android-x86_64/opt: + description: "Android 5.0 x86-64 Opt" + index: + job-name: android-x86_64-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-x86_64/opt + symbol: B + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86_64 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-x86_64-lite/opt: + description: "Android 5.0 x86-64 Opt Lite" + index: + job-name: android-x86_64-lite-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-x86_64/opt + symbol: Bl + run-on-projects: ['integration'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: lite + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86_64-lite + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies-lite + +android-x86_64-fuzzing/debug: + description: "Android x86_64 Fuzzing Debug" + index: + job-name: android-x86_64-fuzzing-debug + treeherder: + platform: android-5-0-x86_64/debug + symbol: Bf + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + PERFHERDER_EXTRA_OPTIONS: fuzzing + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: fuzzing-debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + optimization: + skip-unless-expanded: null + +android-x86_64-shippable/opt: + description: "Android 5.0 x86-64 Shippable" + use-pgo: true + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-x86_64-opt + type: android-shippable + treeherder: + platform: android-5-0-x86_64-shippable/opt + symbol: B + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: x86_64 + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies + +android-x86_64-shippable-lite/opt: + description: "Android 5.0 x86-64 Shippable Lite" + use-pgo: android-x86_64-shippable/opt + attributes: + enable-full-crashsymbols: true + shippable: true + shipping-phase: build + shipping-product: fennec + index: + job-name: android-x86_64-lite-opt + type: android-shippable + treeherder: + platform: android-5-0-x86_64-shippable-lite/opt + symbol: Bl + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: lite + run: + actions: [get-secrets, build, multi-l10n] + config: + - builds/releng_base_android_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: x86_64-lite + fetches: + toolchain: + - linux64-clang + - android-gradle-dependencies-lite + +android-x86_64/debug: + description: "Android 5.0 x86-64 Debug" + index: + job-name: android-x86_64-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-x86_64/debug + symbol: B + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86_64-debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies + +android-x86_64/debug-isolated-process: + description: "Android 5.0 x86-64 Debug with Isolated Process" + index: + job-name: android-x86_64-debug-isolated-process + attributes: + enable-full-crashsymbols: true + treeherder: + platform: android-5-0-x86_64/debug-isolated-process + symbol: Bip + worker-type: b-linux-gcp + worker: + env: + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-x86/releng.manifest" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: isolated-process + run: + actions: [get-secrets, build] + config: ["builds/releng_base_android_64_builds.py"] + custom-build-variant-cfg: x86_64-debug-isolated-process + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - android-gradle-dependencies diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml new file mode 100644 index 0000000000..886f1ef433 --- /dev/null +++ b/taskcluster/ci/build/kind.yml @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + - generate-profile + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - android.yml + - android-asan.yml + - android-stuff.yml + - linux.yml + - linux-base-toolchains.yml + - macosx.yml + - macosx-native.yml + - windows.yml + - windows-mingw.yml + +job-defaults: + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + use-system-python: false + run: + mozconfig-variant: + by-release-type: + beta: beta + release.*: release + # We keep using the release configs as the beta and release configs are + # identical except for + # https://searchfox.org/mozilla-central/rev/ce9ff94ffed34dc17ec0bfa406156d489eaa8ee1/browser/config/mozconfigs/linux32/release#1 # noqa + esr.*: release + default: nightly + # Note: These settings are only honored by nightly (i.e. shipping) builds + update-channel: + by-release-type: + nightly: + by-project: + try: nightly-try + default: nightly + nightly-larch: nightly-larch + nightly-oak: nightly-oak + nightly-pine: nightly-pine + beta: + by-shipping-product: + devedition: aurora + default: beta + release.*: release + esr.*: esr + default: null + # ACCEPTED_MAR_CHANNEL_IDS should usually be the same as the value MAR_CHANNEL_ID. + # If more than one ID is needed, then you should use a comma separated list + # of values. + # The MAR_CHANNEL_ID must not contain the following 3 characters: ",\t " + mar-channel-id: + by-release-type: + nightly-larch: firefox-nightly-larch + nightly-oak: firefox-nightly-oak + nightly-pine: firefox-nightly-pine + beta: + by-shipping-product: + devedition: firefox-mozilla-aurora + default: firefox-mozilla-beta + release.*: firefox-mozilla-release + esr.*: firefox-mozilla-esr + default: firefox-mozilla-central + accepted-mar-channel-ids: + by-release-type: + nightly-larch: firefox-nightly-larch + nightly-oak: firefox-nightly-oak + nightly-pine: firefox-nightly-pine + beta: + by-shipping-product: + devedition: firefox-mozilla-aurora + default: firefox-mozilla-beta,firefox-mozilla-release + release.*: firefox-mozilla-release + esr.*: firefox-mozilla-esr + default: firefox-mozilla-central diff --git a/taskcluster/ci/build/linux-base-toolchains.yml b/taskcluster/ci/build/linux-base-toolchains.yml new file mode 100644 index 0000000000..1cde0b229f --- /dev/null +++ b/taskcluster/ci/build/linux-base-toolchains.yml @@ -0,0 +1,173 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# We have these build jobs in a separate file because their definitions are +# different enough from the main Linux build jobs that their presence in +# linux.yml would make using job-defaults there significantly less useful. +--- +job-defaults: + worker-type: b-linux-gcp + run: + extra-config: + extra_mozconfig_content: + # Builds on automation don't normally build geckodriver, + # but to ensure we also successfully build geckodriver with + # the MSRV, force it to be built. + - ac_add_options --enable-geckodriver + +linux64-base-toolchains/opt: + description: "Linux64 base toolchains Opt" + index: + product: firefox + job-name: linux64-base-toolchains-opt + treeherder: + platform: linux64/opt + symbol: Bb + run-on-projects: ['mozilla-central'] + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: base-toolchains + FORCE_GCC: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-binutils-2.31.1 + - linux64-clang-base + - linux64-gcc-base + - linux64-rust-base + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm-2.14.02 + - linux64-node-12 + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi-clang-8.0 + +linux64-base-toolchains/debug: + description: "Linux64 base toolchains Debug" + index: + product: firefox + job-name: linux64-base-toolchains-debug + treeherder: + platform: linux64/debug + symbol: Bb + optimization: + skip-unless-expanded: null + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: base-toolchains + FORCE_GCC: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + mozconfig-variant: debug + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-binutils-2.31.1 + - linux64-clang-base + - linux64-gcc-base + - linux64-rust-base + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm-2.14.02 + - linux64-node-12 + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi-clang-8.0 + +linux64-base-toolchains-clang/opt: + description: "Linux64 base toolchains clang Opt" + index: + product: firefox + job-name: linux64-base-toolchains-clang-opt + treeherder: + platform: linux64/opt + symbol: Bbc + run-on-projects: ['mozilla-central'] + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: base-toolchains-clang + DISABLE_CLANG_PLUGIN: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + fetches: + toolchain: + - linux64-binutils-2.31.1 + - linux64-clang-base + - linux64-rust-base + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm-2.14.02 + - linux64-node-12 + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi-clang-8.0 + +linux64-base-toolchains-clang/debug: + description: "Linux64 base toolchains clang Debug" + index: + product: firefox + job-name: linux64-base-toolchains-clang-debug + treeherder: + platform: linux64/debug + symbol: Bbc + optimization: + skip-unless-expanded: null + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: base-toolchains-clang + DISABLE_CLANG_PLUGIN: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + mozconfig-variant: debug + tooltool-downloads: public + fetches: + toolchain: + - linux64-binutils-2.31.1 + - linux64-clang-base + - linux64-rust-base + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm-2.14.02 + - linux64-node-12 + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi-clang-8.0 diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml new file mode 100644 index 0000000000..73301313d6 --- /dev/null +++ b/taskcluster/ci/build/linux.yml @@ -0,0 +1,1799 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + +linux64/opt: + description: "Linux64 Opt" + index: + product: firefox + job-name: linux64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: linux64/opt + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-x11/opt: + description: "Linux64 Opt (X11-only)" + index: + product: firefox + job-name: linux64-x11-opt + treeherder: + platform: linux64/opt + symbol: Bx + tier: 2 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: x11 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu-x11 + - sysroot-wasm32-wasi + +linux64-plain/opt: + description: "Linux64 Opt Plain" + index: + product: firefox + job-name: linux64-plain-opt + treeherder: + platform: linux64/opt + symbol: Bp + tier: 1 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + using: mozharness + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: mozharness/scripts/fx_desktop_build.py + tooltool-downloads: public + extra-config: + disable_package_metrics: true + mozconfig-variant: plain-opt + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-non-unified/plain: + description: "Linux64 Plain Hybrid" + index: + product: firefox + job-name: linux64-non-unified-plain + treeherder: + platform: linux64/plain + symbol: Bp-nu + tier: 1 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: non-unified + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: mozharness/scripts/fx_desktop_build.py + secrets: true + tooltool-downloads: public + extra-config: + disable_package_metrics: true + mozconfig-variant: non-unified + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-gcc/opt: + description: "Linux64 GCC Opt" + index: + product: firefox + job-name: linux64-gcc-opt + treeherder: + platform: linux64/opt + symbol: Bgcc + tier: 2 + run-on-projects: ['mozilla-central'] + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: gcc + FORCE_GCC: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-binutils + - linux64-clang + - linux64-gcc-11 + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-plain-clang-trunk/opt: + description: "Linux64 Opt Plain with clang trunk" + index: + product: firefox + job-name: linux64-plain-opt-ct + treeherder: + platform: linux64/opt + symbol: Bp-clang-trunk + tier: 2 + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: plain-clang-trunk + run: + using: mozharness + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: mozharness/scripts/fx_desktop_build.py + tooltool-downloads: public + extra-config: + disable_package_metrics: true + mozconfig-variant: plain-opt-trunk + run-on-projects: [] + fetches: + toolchain: + - linux64-clang-trunk + - linux64-rust + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-toolchain-sysroot + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-fuzzing/debug: + description: "Linux64 Fuzzing Debug" + index: + product: firefox + job-name: linux64-fuzzing-debug + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/debug + symbol: Bf + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-debug + mozconfig-variant: debug-fuzzing + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-rust + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-fuzzing-noopt/debug: + description: "Linux64 Fuzzing No-optimize Debug" + index: + product: firefox + job-name: linux64-fuzzing-noopt-debug + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64-noopt/debug + symbol: Bf + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing-noopt-debug + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-debug + mozconfig-variant: debug-fuzzing-noopt + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-rust + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64/debug: + description: "Linux64 Debug" + index: + product: firefox + job-name: linux64-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: linux64/debug + symbol: B + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + mozconfig-variant: debug + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-rust + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-plain/debug: + description: "Linux64 Debug Plain" + index: + product: firefox + job-name: linux64-plain-debug + treeherder: + platform: linux64/debug + symbol: Bp + tier: 1 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + using: mozharness + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + disable_package_metrics: true + mozconfig-variant: plain-debug + script: mozharness/scripts/fx_desktop_build.py + tooltool-downloads: public + run-on-projects: [trunk] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-devedition/opt: + description: "Linux64 devedition (shippable)" + use-pgo: linux64-shippable/opt + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: devedition + index: + product: devedition + job-name: linux64-opt + type: shippable + treeherder: + platform: linux64-devedition/opt + symbol: B + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: 'linux64-devedition' + max_build_output_timeout: 3600 + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux/opt: + description: "Linux32 Opt" + index: + product: firefox + job-name: linux-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: linux32/opt + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + run-on-projects: + - 'mozilla-beta' + - 'mozilla-release' + - 'mozilla-esr115' + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-fuzzing/debug: + description: "Linux32 Fuzzing Debug" + index: + product: firefox + job-name: linux-fuzzing-debug + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux32/debug + symbol: Bf + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-debug + mozconfig-variant: debug-fuzzing + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux32-llvm-symbolizer + - linux64-sccache + - linux64-rust + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux/debug: + description: "Linux32 Debug" + index: + product: firefox + job-name: linux-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: linux32/debug + symbol: B + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + mozconfig-variant: debug + tooltool-downloads: public + run-on-projects: ['release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-rusttests/opt: + description: "Linux32 Rust Tests Opt" + index: + product: firefox + job-name: linux-rusttests-opt + treeherder: + platform: linux32/opt + symbol: BR + tier: 1 + worker: + max-run-time: 2700 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: rusttests + mozconfig-variant: rusttests + tooltool-downloads: public + keep-artifacts: false + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + optimization: + test-inclusive: [rusttests] + +linux-rusttests/debug: + description: "Linux32 Rust Tests Debug" + index: + product: firefox + job-name: linux-rusttests-debug + treeherder: + platform: linux32/debug + symbol: BR + tier: 1 + worker: + max-run-time: 2700 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: rusttests-debug + mozconfig-variant: rusttests-debug + tooltool-downloads: public + keep-artifacts: false + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + optimization: + test-inclusive: [rusttests] + +linux-devedition/opt: + description: "Linux32 devedition (shippable)" + use-pgo: linux-shippable/opt + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: devedition + index: + product: devedition + job-name: linux-opt + type: shippable + treeherder: + platform: linux32-devedition/opt + symbol: B + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: linux-devedition + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-shippable/opt: + description: "Linux32 Shippable" + use-pgo: true + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux-opt + type: shippable + treeherder: + platform: linux32-shippable/opt + symbol: Bpgo(B) + worker-type: b-linux-gcp + worker: + max-run-time: 9000 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + - taskcluster_nightly.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + run-on-projects: ['release'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-reproduced/opt: + description: "Linux32 Reproduced" + use-pgo: linux-shippable/opt + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux-reproduced-opt + treeherder: + platform: linux32-shippable/opt + symbol: Bpgo(repr) + worker-type: b-linux-gcp + worker: + max-run-time: 9000 + env: + PERFHERDER_EXTRA_OPTIONS: repr + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + - taskcluster_nightly.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + # Original comment: Limited to trunk until bug 1596350 is fixed. + # Addendum: Do not run on autoland, Bug 1599197 + run-on-projects: ['mozilla-central', 'comm-central'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux-asan-fuzzing/opt: + description: "Linux32 Fuzzing Opt ASAN" + index: + product: firefox + job-name: linux-fuzzing-asan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux32/asan + symbol: Bof + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-asan-tc + mozconfig-variant: nightly-fuzzing-asan + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux32-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-reproduced/opt: + description: "Linux64 Reproduced" + use-pgo: linux64-shippable/opt + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux64-reproduced-opt + treeherder: + platform: linux64-shippable/opt + symbol: Bpgo(repr) + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 9000 + env: + PERFHERDER_EXTRA_OPTIONS: repr + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + max_build_output_timeout: 3600 + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + # Original comment: Limited to trunk until bug 1596350 is fixed. + # Addendum: Do not run on autoland, Bug 1599197 + run-on-projects: ['mozilla-central', 'comm-central'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-asan/opt: + description: "Linux64 Opt ASAN" + index: + product: firefox + job-name: linux64-asan-opt + treeherder: + platform: linux64/asan + symbol: Bo + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt asan" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: asan-tc + mozconfig-variant: nightly-asan + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-tsan/opt: + description: "Linux64 Opt TSAN" + index: + product: firefox + job-name: linux64-tsan-opt + treeherder: + platform: linux64/tsan + symbol: Bo + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt tsan" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: tsan-tc + mozconfig-variant: tsan + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-tsan-fuzzing/opt: + description: "Linux64 Fuzzing Opt TSAN" + index: + product: firefox + job-name: linux64-fuzzing-tsan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/tsan + symbol: Bof + worker: + env: + PERFHERDER_EXTRA_OPTIONS: tsan-fuzzing + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-tsan-tc + mozconfig-variant: tsan-fuzzing + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-asan-fuzzing/noopt: + description: "Linux64 Fuzzing No-optimize ASAN" + index: + product: firefox + job-name: linux64-fuzzing-asan-noopt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64-noopt/asan + symbol: Bf + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing-noopt + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-asan-tc + mozconfig-variant: nightly-fuzzing-asan-noopt + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-asan-fuzzing-nyx/opt: + description: "Linux64 Fuzzing Opt ASAN NYX" + index: + product: firefox + job-name: linux64-fuzzing-asan-nyx-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/asan + symbol: Bonf + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing-nyx + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + AFL_CC: /builds/worker/fetches/clang/bin/clang + AFL_CXX: /builds/worker/fetches/clang/bin/clang++ + AFL_LLVM_INSTRUMENT: llvmnative + AFL_LLVM_NO_RPATH: "1" + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-asan-tc + mozconfig-variant: nightly-fuzzing-asan-nyx + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-afl-instrumentation + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-asan-fuzzing-nyx-ccov/opt: + description: "Linux64 Fuzzing Opt ASAN NYX w/ Coverage" + index: + product: firefox + job-name: linux64-ccov-fuzzing-asan-nyx-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/asan + symbol: Bncf + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing-ccov-nyx + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + AFL_CC: /builds/worker/fetches/clang/bin/clang + AFL_CXX: /builds/worker/fetches/clang/bin/clang++ + AFL_LLVM_INSTRUMENT: llvmcodecov + AFL_LLVM_NO_RPATH: "1" + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-asan-tc + mozconfig-variant: nightly-fuzzing-asan-nyx + tooltool-downloads: public + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-afl-instrumentation + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + + +linux64-asan-fuzzing/opt: + description: "Linux64 Fuzzing Opt ASAN" + index: + product: firefox + job-name: linux64-fuzzing-asan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/asan + symbol: Bof + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: fuzzing-asan-tc + mozconfig-variant: nightly-fuzzing-asan + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-asan-fuzzing-ccov/opt: + description: "Linux64 Fuzzing Opt ASAN w/ Coverage" + index: + product: firefox + job-name: linux64-ccov-fuzzing-asan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: linux64/asan + symbol: Bocf + run-on-projects: ['mozilla-central'] + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing-ccov + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: asan-fuzzing-ccov + tooltool-downloads: public + use-sccache: false + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-fuzzing-ccov/opt: + description: "Linux64 Fuzzing Opt w/ Coverage" + index: + product: firefox + job-name: linux64-ccov-fuzzing-opt + treeherder: + platform: linux64/opt + symbol: Bocf + run-on-projects: ['mozilla-central'] + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing-ccov + RUSTC_BOOTSTRAP: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: fuzzing-ccov + tooltool-downloads: public + use-sccache: false + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-rust + - linux64-rust-size + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-asan-reporter-shippable/opt: + description: "Linux64 Opt ASAN Reporter (Shippable)" + attributes: + shippable: true + shipping-product: + by-release-type: + nightly: firefox + default: null + index: + product: firefox + job-name: linux64-asan-reporter-opt + type: shippable + treeherder: + platform: linux64-asan-reporter/opt + symbol: BoR + run-on-projects: ['mozilla-central'] + worker-type: b-linux-xlarge-gcp + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-reporter + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 3600 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + max_build_output_timeout: 3600 + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: asan-reporter-tc + mozconfig-variant: nightly-asan-reporter + tooltool-downloads: public + mar-channel-id: + firefox-mozilla-central-asan + accepted-mar-channel-ids: + firefox-mozilla-central-asan + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-asan/debug: + description: "Linux64 Debug ASAN" + index: + product: firefox + job-name: linux64-asan-debug + treeherder: + platform: linux64/asan + symbol: Bd + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "debug asan" + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: asan-tc-and-debug + mozconfig-variant: debug-asan + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-dev + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-llvm-symbolizer + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +linux64-shippable/opt: + description: "Linux64 Shippable" + use-pgo: true + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux64-opt + type: shippable + treeherder: + platform: linux64-shippable/opt + symbol: Bpgo(B) + run-on-projects: ['release'] + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 9000 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + max_build_output_timeout: 3600 + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-nightlyasrelease/opt: + description: "Linux64 Nightly-As-Release" + use-pgo: linux64-shippable/opt + attributes: + enable-full-crashsymbols: true + index: + product: firefox + job-name: linux64-nightlyasrelease-opt + treeherder: + platform: linux64-nightlyasrelease/opt + symbol: Bpgo(Bnar) + run-on-projects: [trunk-only] + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 9000 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + # bug 1690154 - enable full symbols on all branches, but only + # upload them on the official set. In conjunction with + # `enable-full-crashsymbols`. + MOZ_ENABLE_FULL_SYMBOLS: "1" + PERFHERDER_EXTRA_OPTIONS: nightly-as-release + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + max_build_output_timeout: 3600 + script: "mozharness/scripts/fx_desktop_build.py" + mozconfig-variant: nightly-as-release + secrets: true + tooltool-downloads: public + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-noopt/debug: + description: "Linux64 No-optimize Debug" + index: + product: firefox + job-name: linux64-noopt-debug + treeherder: + platform: linux64-noopt/debug + symbol: B + tier: 2 + attributes: + enable-full-crashsymbols: true + worker: + max-run-time: 5400 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: noopt-debug + mozconfig-variant: noopt-debug + tooltool-downloads: public + keep-artifacts: false + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-rusttests/opt: + description: "Linux64 Rust Tests Opt" + index: + product: firefox + job-name: linux64-rusttests-opt + treeherder: + platform: linux64/opt + symbol: BR + tier: 1 + worker: + max-run-time: 2700 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: rusttests + mozconfig-variant: rusttests + tooltool-downloads: public + keep-artifacts: false + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + optimization: + test-inclusive: [rusttests] + +linux64-rusttests/debug: + description: "Linux64 Rust Tests Debug" + index: + product: firefox + job-name: linux64-rusttests-debug + treeherder: + platform: linux64/debug + symbol: BR + tier: 1 + worker: + max-run-time: 2700 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: rusttests-debug + mozconfig-variant: rusttests-debug + tooltool-downloads: public + keep-artifacts: false + run-on-projects: ['trunk'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + optimization: + test-inclusive: [rusttests] + +linux64-ccov/opt: + description: "Linux64-CCov Opt" + index: + product: firefox + job-name: linux64-ccov-opt + treeherder: + platform: linux64-ccov/opt + symbol: B + tier: 2 + run-on-projects: ['mozilla-central'] + worker: + artifacts: + - name: public/code-coverage-grcov.zip + path: /builds/worker/workspace/obj-build/code-coverage-grcov.zip + type: file + max-run-time: 7200 + env: + RUSTC_BOOTSTRAP: '1' + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: code-coverage-opt + mozconfig-variant: code-coverage-opt + tooltool-downloads: public + use-sccache: false + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-grcov + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-ccov/debug: + description: "Linux64-CCov Debug" + index: + product: firefox + job-name: linux64-ccov-debug + treeherder: + platform: linux64-ccov/debug + symbol: B + tier: 2 + run-on-projects: [] + worker: + max-run-time: 7200 + env: + RUSTC_BOOTSTRAP: '1' + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: code-coverage-debug + mozconfig-variant: code-coverage-debug + tooltool-downloads: public + use-sccache: false + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-grcov + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-add-on-devel/opt: + description: "Linux64 add-on-devel" + index: + product: firefox + job-name: linux64-add-on-devel + treeherder: + platform: linux64-add-on-devel/opt + symbol: B + tier: 2 + worker: + max-run-time: 3600 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: add-on-devel + mozconfig-variant: add-on-devel + tooltool-downloads: public + run-on-projects: ['mozilla-beta', 'mozilla-release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +linux64-aarch64/opt: + description: "Linux64 aarch64 Opt" + use-pgo: linux64-shippable/opt + index: + product: firefox + job-name: linux64-aarch64-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: linux64-aarch64/opt + symbol: B + tier: 2 + worker: + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + extra_mozconfig_content: + - ac_add_options --target=aarch64 + - unset MOZ_STDCXX_COMPAT + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + run-on-projects: ['integration'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-node + - linux64-pkgconf + - sysroot-aarch64-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null diff --git a/taskcluster/ci/build/macosx-native.yml b/taskcluster/ci/build/macosx-native.yml new file mode 100644 index 0000000000..ad77b6e5aa --- /dev/null +++ b/taskcluster/ci/build/macosx-native.yml @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + index: + product: firefox + worker-type: b-osx-1015 + worker: + max-run-time: 7200 + run: + using: mozharness + script: "mozharness/scripts/fx_desktop_build.py" + +macosx64-plain/opt: + description: "MacOS X x64 Opt Plain" + index: + job-name: macosx64-plain-opt + treeherder: + symbol: Bp + platform: osx/opt + tier: 1 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_macosx.py + extra-config: + disable_package_metrics: true + stage_platform: macosx64 + mozconfig_platform: macosx64 + mozconfig-variant: plain-opt + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - macosx64-clang + - macosx64-cbindgen + - macosx64-nasm + - macosx64-node + - macosx64-rust + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-plain/debug: + description: "MacOS X x64 Debug Plain" + index: + job-name: macosx64-plain-debug + treeherder: + symbol: Bp + platform: osx/debug + tier: 1 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_macosx.py + extra-config: + disable_package_metrics: true + stage_platform: macosx64 + mozconfig_platform: macosx64 + mozconfig-variant: plain-debug + run-on-projects: ['trunk'] + fetches: + toolchain: + - macosx64-clang + - macosx64-cbindgen + - macosx64-nasm + - macosx64-node + - macosx64-rust + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-rusttests/opt: + description: "MacOS X x64 Rust Tests Opt" + index: + job-name: macosx64-rusttests-opt + treeherder: + platform: macosx64/opt + symbol: BR + tier: 1 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_macosx.py + - builds/releng_sub_mac_configs/rusttests.py + extra-config: + mozconfig_platform: macosx64 + mozconfig-variant: rusttests + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - macosx64-clang + - macosx64-rust + - macosx64-cbindgen + - macosx64-nasm + - macosx64-node + - macosx64-sdk + optimization: + test-inclusive: [rusttests] + +macosx64-rusttests/debug: + description: "MacOS X x64 Rust Tests Debug" + index: + job-name: macosx64-rusttests-debug + treeherder: + platform: macosx64/debug + symbol: BR + tier: 1 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_macosx.py + - builds/releng_sub_mac_configs/rusttests.py + extra-config: + mozconfig_platform: macosx64 + mozconfig-variant: rusttests-debug + run-on-projects: ['trunk'] + fetches: + toolchain: + - macosx64-clang + - macosx64-rust + - macosx64-cbindgen + - macosx64-nasm + - macosx64-node + - macosx64-sdk + optimization: + test-inclusive: [rusttests] diff --git a/taskcluster/ci/build/macosx.yml b/taskcluster/ci/build/macosx.yml new file mode 100644 index 0000000000..0b148f01d5 --- /dev/null +++ b/taskcluster/ci/build/macosx.yml @@ -0,0 +1,943 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + index: + product: firefox + worker-type: b-linux-gcp + run: + using: mozharness + actions: [get-secrets, build] + secrets: true + script: "mozharness/scripts/fx_desktop_build.py" + fetches: + toolchain: + - linux64-dump_syms + - linux64-hfsplus + - linux64-libdmg + - linux64-rust-macos + - linux64-rust-size + - linux64-cbindgen + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + +macosx64/debug: + description: "MacOS X x64 Cross-compile" + index: + job-name: macosx64-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: osx-cross/debug + symbol: B + tier: 1 + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-debug + mozconfig-variant: debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-aarch64/debug: + description: "MacOS X Aarch64 Cross-compile" + index: + job-name: macosx64-aarch64-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: osx-cross-aarch64/debug + symbol: B + tier: 1 + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-debug + mozconfig-variant: debug + extra-config: + extra_mozconfig_content: + - ac_add_options --target=aarch64-apple-darwin + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-non-unified/plain: + description: "MacOS X x64 Hybrid Cross-compile" + index: + job-name: macosx64-non-unified + treeherder: + platform: osx-cross/plain + symbol: Bp-nu + tier: 1 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: non-unified + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + mozconfig-variant: non-unified + extra-config: + disable_package_metrics: true + use-sccache: true + run-on-projects: ['integration'] + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64/opt: + description: "MacOS X x64 Cross-compile" + index: + job-name: macosx64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: osx-cross/opt + symbol: B + tier: 1 + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-asan-fuzzing/opt: + description: "MacOS X x64 Cross-compile Fuzzing ASAN" + index: + job-name: macosx64-fuzzing-asan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: osx-cross/asan + symbol: Bof + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-fuzzing-asan + mozconfig-variant: nightly-fuzzing-asan + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-llvm-symbolizer + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-fuzzing/debug: + description: "MacOS X x64 Cross-compile Fuzzing Debug" + index: + job-name: macosx64-fuzzing-debug + attributes: + enable-full-crashsymbols: true + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: osx-cross/debug + symbol: Bf + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: fuzzing + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-fuzzing-debug + mozconfig-variant: debug-fuzzing + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-aarch64-asan-fuzzing/opt: + description: "MacOS X Aarch64 Cross-compile Fuzzing ASAN" + index: + job-name: macosx64-aarch64-fuzzing-asan-opt + attributes: + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: osx-cross-aarch64/asan + symbol: Bof + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64-asan-fuzzing + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-fuzzing-asan + mozconfig-variant: nightly-fuzzing-asan + extra-config: + mozconfig_platform: macosx64-aarch64 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-aarch64-llvm-symbolizer + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-aarch64-fuzzing/debug: + description: "MacOS X Aarch64 Cross-compile Fuzzing Debug" + index: + job-name: macosx64-aarch64-fuzzing-debug + attributes: + enable-full-crashsymbols: true + # The gtest libxul contains libFuzzer targets + skip-verify-test-packaging: true + treeherder: + platform: osx-cross-aarch64/debug + symbol: Bf + worker: + max-run-time: 5400 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64-fuzzing + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-fuzzing-debug + mozconfig-variant: debug-fuzzing + extra-config: + mozconfig_platform: macosx64-aarch64 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-aarch64-llvm-symbolizer + - macosx64-sdk + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +macosx64-x64-devedition/opt: + description: "MacOS X Dev Edition x64 (shippable)" + use-pgo: macosx64-shippable/opt + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + upload-generated-sources: true + shipping-phase: build + shipping-product: devedition + index: + product: devedition + job-name: macosx64-x64-opt + type: shippable + treeherder: + platform: osx-cross-devedition/opt + symbol: Bx + tier: 1 + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: macosx64-devedition + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-aarch64-devedition/opt: + description: "MacOS X Dev Edition Aarch64 (shippable)" + use-pgo: macosx64-shippable/opt + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + upload-generated-sources: true + shipping-phase: build + shipping-product: devedition + index: + product: devedition + job-name: macosx64-aarch64-opt + type: shippable + treeherder: + platform: osx-aarch64-devedition/opt + symbol: B + tier: 1 + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: macosx64-aarch64-devedition + mozconfig_platform: macosx64-aarch64 + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-devedition/opt: + description: "MacOS X Dev Edition Universal" + attributes: + shippable: true + enable-build-signing: true + skip-upload-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: devedition + index: + product: devedition + job-name: macosx64-opt + type: shippable + treeherder: + platform: osx-cross-devedition/opt + symbol: B + tier: 1 + run-on-projects: ['mozilla-beta'] + worker: + max-run-time: 1800 + run: + job-script: taskcluster/scripts/misc/unify.sh + config: + - builds/releng_base_firefox.py + dependencies: + macosx64-x64-devedition-opt: build-macosx64-x64-devedition/opt + macosx64-aarch64-devedition-opt: build-macosx64-aarch64-devedition/opt + fetches: + macosx64-x64-devedition-opt: + - artifact: target.dmg + extract: false + dest: x64 + - artifact: buildhub.json + extract: false + dest: ../artifacts + - artifact: host/bin/mar + extract: false + dest: ../artifacts/host/bin + - artifact: host/bin/mbsdiff + extract: false + dest: ../artifacts/host/bin + - artifact: target.txt + extract: false + dest: ../artifacts + - artifact: target_info.txt + extract: false + dest: ../artifacts + - artifact: target.json + extract: false + dest: ../artifacts + - artifact: target.mozinfo.json + extract: false + dest: ../artifacts + - artifact: target.test_packages.json + extract: false + dest: ../artifacts + - artifact: target.perftests.tests.tar.gz + extract: false + dest: ../artifacts + - artifact: target.xpt_artifacts.zip + extract: false + dest: ../artifacts + - artifact: target.common.tests.tar.gz + dest: x64/common + - artifact: target.cppunittest.tests.tar.gz + dest: x64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: x64/gtest + - artifact: target.jittest.tests.tar.gz + dest: x64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: x64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: x64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: x64/reftest + - artifact: target.talos.tests.tar.gz + dest: x64/talos + - artifact: target.raptor.tests.tar.gz + dest: x64/raptor + - artifact: target.condprof.tests.tar.gz + dest: x64/confprof + - artifact: target.awsy.tests.tar.gz + dest: x64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: x64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: x64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: x64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: x64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: x64/jsshell + - artifact: target.generated-files.tar.gz + extract: false + dest: ../artifacts + - artifact: target.langpack.xpi + extract: false + dest: ../artifacts + - artifact: mozharness.zip + extract: false + dest: ../artifacts + macosx64-aarch64-devedition-opt: + - artifact: target.dmg + extract: false + dest: aarch64 + - artifact: target.common.tests.tar.gz + dest: aarch64/common + - artifact: target.cppunittest.tests.tar.gz + dest: aarch64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: aarch64/gtest + - artifact: target.jittest.tests.tar.gz + dest: aarch64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: aarch64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: aarch64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: aarch64/reftest + - artifact: target.talos.tests.tar.gz + dest: aarch64/talos + - artifact: target.raptor.tests.tar.gz + dest: aarch64/raptor + - artifact: target.condprof.tests.tar.gz + dest: aarch64/confprof + - artifact: target.awsy.tests.tar.gz + dest: aarch64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: aarch64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: aarch64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: aarch64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: aarch64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: aarch64/jsshell + toolchain: + - linux64-cctools-port + +macosx64-noopt/debug: + description: "MacOS X x64 No-optimize Debug" + index: + job-name: macosx64-noopt-debug + treeherder: + platform: osx-cross-noopt/debug + symbol: B + tier: 2 + worker: + max-run-time: 3600 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-noopt-debug + mozconfig-variant: cross-noopt-debug + keep-artifacts: false + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-aarch64-noopt/debug: + description: "MacOS X aarch64 No-optimize Debug" + index: + job-name: macosx64-aarch64-noopt-debug + treeherder: + platform: osx-cross-noopt/debug + symbol: Baarch64 + tier: 2 + worker: + max-run-time: 3600 + env: + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-noopt-debug + mozconfig-variant: cross-noopt-debug + keep-artifacts: false + extra-config: + mozconfig_platform: macosx64-aarch64 + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-x64-add-on-devel/opt: + description: "MacOS X x64 add-on-devel" + index: + job-name: macosx64-x64-add-on-devel + treeherder: + platform: osx-cross-add-on-devel/opt + symbol: Bx + tier: 2 + worker: + max-run-time: 3600 + env: + PERFHERDER_EXTRA_OPTIONS: x64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: add-on-devel + mozconfig-variant: add-on-devel + run-on-projects: ['mozilla-beta', 'mozilla-release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-aarch64-add-on-devel/opt: + description: "MacOS X Aarch64 add-on-devel" + index: + job-name: macosx64-aarch64-add-on-devel + treeherder: + platform: osx-cross-aarch64-add-on-devel/opt + symbol: B + tier: 2 + worker: + max-run-time: 3600 + env: + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: add-on-devel + mozconfig-variant: add-on-devel + extra-config: + mozconfig_platform: macosx64-aarch64 + run-on-projects: ['mozilla-beta', 'mozilla-release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-add-on-devel/opt: + description: "MacOS X add-on-devel" + index: + job-name: macosx64-add-on-devel + treeherder: + platform: osx-cross-add-on-devel/opt + symbol: B + tier: 2 + worker: + max-run-time: 1800 + run: + job-script: taskcluster/scripts/misc/unify.sh + config: + - builds/releng_base_firefox.py + run-on-projects: ['mozilla-beta', 'mozilla-release'] + dependencies: + macosx64-x64-add-on-devel-opt: build-macosx64-x64-add-on-devel/opt + macosx64-aarch64-add-on-devel-opt: build-macosx64-aarch64-add-on-devel/opt + fetches: + macosx64-x64-add-on-devel-opt: + - artifact: target.dmg + extract: false + dest: x64 + - artifact: buildhub.json + extract: false + dest: ../artifacts + - artifact: host/bin/mar + extract: false + dest: ../artifacts/host/bin + - artifact: host/bin/mbsdiff + extract: false + dest: ../artifacts/host/bin + - artifact: target.txt + extract: false + dest: ../artifacts + - artifact: target_info.txt + extract: false + dest: ../artifacts + - artifact: target.json + extract: false + dest: ../artifacts + - artifact: target.mozinfo.json + extract: false + dest: ../artifacts + - artifact: target.crashreporter-symbols.zip + dest: x64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: x64/jsshell + - artifact: target.generated-files.tar.gz + extract: false + dest: ../artifacts + - artifact: target.langpack.xpi + extract: false + dest: ../artifacts + - artifact: mozharness.zip + extract: false + dest: ../artifacts + macosx64-aarch64-add-on-devel-opt: + - artifact: target.dmg + extract: false + dest: aarch64 + - artifact: target.crashreporter-symbols.zip + dest: aarch64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: aarch64/jsshell + toolchain: + - linux64-cctools-port + +macosx64-nightlyasrelease/opt: + description: "MacOS X Nightly-As-Release" + use-pgo: macosx64-shippable/opt + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + upload-generated-sources: true + index: + job-name: macosx64-nightlyasrelease-opt + treeherder: + platform: osx-nightlyasrelease/opt + symbol: Bpgo(Bnar) + tier: 1 + run-on-projects: [trunk-only] + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: nightly-as-release + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + mozconfig-variant: nightly-as-release + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-x64-shippable/opt: + description: "MacOS X x64 Cross-compile" + use-pgo: macosx64-shippable/opt + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + upload-generated-sources: true + shipping-phase: build + shipping-product: firefox + index: + job-name: macosx64-x64-opt + type: shippable + treeherder: + platform: osx-shippable/opt + symbol: Bpgo(Bx) + tier: 1 + run-on-projects: ['release'] + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-aarch64-shippable/opt: + description: "MacOS X aarch64 Cross-compile" + use-pgo: macosx64-aarch64-shippable/opt + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + upload-generated-sources: true + shipping-phase: build + shipping-product: firefox + index: + job-name: macosx64-aarch64-opt + type: shippable + treeherder: + platform: osx-aarch64-shippable/opt + symbol: Bpgo(B) + tier: 1 + run-on-projects: ['release'] + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + extra-config: + mozconfig_platform: macosx64-aarch64 + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + - sysroot-wasm32-wasi + +macosx64-shippable/opt: + description: "MacOS X Universal" + attributes: + shippable: true + enable-build-signing: true + skip-upload-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: firefox + index: + job-name: macosx64-opt + type: shippable + treeherder: + platform: osx-shippable/opt + symbol: Bpgo(B) + tier: 1 + run-on-projects: ['release'] + worker: + max-run-time: 1800 + run: + job-script: taskcluster/scripts/misc/unify.sh + config: + - builds/releng_base_firefox.py + dependencies: + macosx64-x64-shippable-opt: build-macosx64-x64-shippable/opt + macosx64-aarch64-shippable-opt: build-macosx64-aarch64-shippable/opt + fetches: + macosx64-x64-shippable-opt: + - artifact: target.dmg + extract: false + dest: x64 + - artifact: buildhub.json + extract: false + dest: ../artifacts + - artifact: host/bin/mar + extract: false + dest: ../artifacts/host/bin + - artifact: host/bin/mbsdiff + extract: false + dest: ../artifacts/host/bin + - artifact: target.txt + extract: false + dest: ../artifacts + - artifact: target_info.txt + extract: false + dest: ../artifacts + - artifact: target.json + extract: false + dest: ../artifacts + - artifact: target.mozinfo.json + extract: false + dest: ../artifacts + - artifact: target.test_packages.json + extract: false + dest: ../artifacts + - artifact: target.perftests.tests.tar.gz + extract: false + dest: ../artifacts + - artifact: target.xpt_artifacts.zip + extract: false + dest: ../artifacts + - artifact: target.common.tests.tar.gz + dest: x64/common + - artifact: target.cppunittest.tests.tar.gz + dest: x64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: x64/gtest + - artifact: target.jittest.tests.tar.gz + dest: x64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: x64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: x64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: x64/reftest + - artifact: target.talos.tests.tar.gz + dest: x64/talos + - artifact: target.raptor.tests.tar.gz + dest: x64/raptor + - artifact: target.condprof.tests.tar.gz + dest: x64/confprof + - artifact: target.awsy.tests.tar.gz + dest: x64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: x64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: x64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: x64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: x64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: x64/jsshell + - artifact: target.generated-files.tar.gz + extract: false + dest: ../artifacts + - artifact: target.langpack.xpi + extract: false + dest: ../artifacts + - artifact: mozharness.zip + extract: false + dest: ../artifacts + macosx64-aarch64-shippable-opt: + - artifact: target.dmg + extract: false + dest: aarch64 + - artifact: target.common.tests.tar.gz + dest: aarch64/common + - artifact: target.cppunittest.tests.tar.gz + dest: aarch64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: aarch64/gtest + - artifact: target.jittest.tests.tar.gz + dest: aarch64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: aarch64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: aarch64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: aarch64/reftest + - artifact: target.talos.tests.tar.gz + dest: aarch64/talos + - artifact: target.raptor.tests.tar.gz + dest: aarch64/raptor + - artifact: target.condprof.tests.tar.gz + dest: aarch64/confprof + - artifact: target.awsy.tests.tar.gz + dest: aarch64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: aarch64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: aarch64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: aarch64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: aarch64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: aarch64/jsshell + toolchain: + - linux64-cctools-port + +macosx64-ccov/opt: + description: "MacOS X x64 Cross-compile Code Coverage" + index: + job-name: macosx64-ccov-opt + treeherder: + platform: osx-cross-ccov/opt + symbol: B + tier: 1 + worker: + max-run-time: 7200 + env: + RUSTC_BOOTSTRAP: '1' + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: code-coverage-opt + mozconfig-variant: code-coverage + run-on-projects: ['mozilla-central'] + use-sccache: false + fetches: + toolchain: + - linux64-clang + - linux64-sccache + - macosx64-sdk + - sysroot-wasm32-wasi diff --git a/taskcluster/ci/build/windows-mingw.yml b/taskcluster/ci/build/windows-mingw.yml new file mode 100644 index 0000000000..09c3ee0022 --- /dev/null +++ b/taskcluster/ci/build/windows-mingw.yml @@ -0,0 +1,173 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run: + using: mozharness + actions: [build] + script: mozharness/scripts/fx_desktop_build.py + optimization: + skip-unless-expanded: null + fetches: + toolchain: + - windows-rs + +win32-mingwclang/opt: + description: "Win32 MinGW-Clang Opt" + index: + product: firefox + job-name: win32-mingwclang-opt + treeherder: + platform: windows-mingw32/all + symbol: WMC32(Bo) + tier: 2 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt 32 clang" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_windows_32_mingw_builds.py + - builds/releng_sub_windows_configs/32_mingwclang.py + mozconfig-variant: mingwclang + run-on-projects: + - 'mozilla-central' + - 'mozilla-esr115' + use-sccache: true + fetches: + toolchain: + - mingw32-rust + - linux64-upx + - linux64-wine + - linux64-sccache + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-clang-mingw-x86 + - linux64-mingw32-nsis + - linux64-mingw-fxc2-x86 + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +win32-mingwclang/debug: + description: "Win32 MinGW-Clang Debug" + index: + product: firefox + job-name: win32-mingwclang-debug + treeherder: + platform: windows-mingw32/all + symbol: WMC32(Bd) + tier: 2 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "debug 32 clang" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_windows_32_mingw_builds.py + - builds/releng_sub_windows_configs/32_mingwclang.py + mozconfig-variant: mingwclang-debug + run-on-projects: + - 'mozilla-central' + - 'mozilla-esr115' + use-sccache: true + fetches: + toolchain: + - mingw32-rust + - linux64-upx + - linux64-wine + - linux64-sccache + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-clang-mingw-x86 + - linux64-mingw32-nsis + - linux64-mingw-fxc2-x86 + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +win64-mingwclang/opt: + description: "Win64 MinGW-Clang Opt" + index: + product: firefox + job-name: win64-mingwclang-opt + treeherder: + platform: windows-mingw32/all + symbol: WMC64(Bo) + tier: 2 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt 64 clang" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_windows_64_mingw_builds.py + - builds/releng_sub_windows_configs/64_mingwclang.py + mozconfig-variant: mingwclang + run-on-projects: + - 'mozilla-central' + - 'mozilla-esr115' + use-sccache: true + fetches: + toolchain: + - mingw32-rust + - linux64-upx + - linux64-wine + - linux64-sccache + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-clang-mingw-x64 + - linux64-mingw32-nsis + - linux64-mingw-fxc2-x86 + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +win64-mingwclang/debug: + description: "Win64 MinGW-Clang Debug" + index: + product: firefox + job-name: win64-mingwclang-debug + treeherder: + platform: windows-mingw32/all + symbol: WMC64(Bd) + tier: 2 + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "debug 64 clang" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_windows_64_mingw_builds.py + - builds/releng_sub_windows_configs/64_mingwclang.py + mozconfig-variant: mingwclang-debug + run-on-projects: + - 'mozilla-central' + - 'mozilla-esr115' + use-sccache: true + fetches: + toolchain: + - mingw32-rust + - linux64-upx + - linux64-wine + - linux64-sccache + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-clang-mingw-x64 + - linux64-mingw32-nsis + - linux64-mingw-fxc2-x86 + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi diff --git a/taskcluster/ci/build/windows.yml b/taskcluster/ci/build/windows.yml new file mode 100644 index 0000000000..cde5e216da --- /dev/null +++ b/taskcluster/ci/build/windows.yml @@ -0,0 +1,1583 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: mozharness + fetches: + toolchain: + - windows-rs + +win32/debug: + description: "Win32 Debug" + index: + product: firefox + job-name: win32-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-32/debug + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win32/debug.py + mozconfig-variant: debug + extra-config: + mozconfig_platform: win32 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win32/opt: + description: "Win32 Opt" + index: + product: firefox + job-name: win32-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-32/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win32-fuzzing/debug: + description: "Win32 Fuzzing Debug" + index: + product: firefox + job-name: win32-fuzzing-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: windows2012-32/debug + symbol: Bf + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win32/debug.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + mozconfig-variant: debug-fuzzing + run-on-projects: ['trunk'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win64/debug: + description: "Win64 Debug" + index: + product: firefox + job-name: win64-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-64/debug + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/debug.py + mozconfig-variant: debug + extra-config: + mozconfig_platform: win64 + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-non-unified/plain: + description: "Win64 Hybrid Plain" + index: + product: firefox + job-name: win64-non-unified-plain + treeherder: + platform: windows2012-64/debug + symbol: Bp-nu + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: non-unified + run: + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + mozconfig-variant: non-unified + extra-config: + mozconfig_platform: win64 + use-sccache: true + run-on-projects: ['integration'] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-fuzzing/debug: + description: "Win64 Fuzzing Debug" + index: + product: firefox + job-name: win64-fuzzing-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: windows2012-64/debug + symbol: Bf + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/debug.py + extra-config: + mozconfig_platform: win64 + mozconfig-variant: debug-fuzzing + run-on-projects: ['trunk'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win64-plain/debug: + description: "Win64 Debug Plain" + index: + product: firefox + job-name: win64-plain-debug + treeherder: + platform: windows2012-64/debug + symbol: Bp + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + extra-config: + disable_package_metrics: true + stage_platform: win64 + mozconfig-variant: plain-debug + run-on-projects: ['trunk'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-node + - win64-nasm + - win64-cbindgen + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + - nsis + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +win64/opt: + description: "Win64 Opt" + index: + product: firefox + job-name: win64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-64/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-plain/opt: + description: "Win64 Opt Plain" + index: + product: firefox + job-name: win64-plain-opt + treeherder: + platform: windows2012-64/opt + symbol: Bp + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + extra-config: + disable_package_metrics: true + stage_platform: win64 + mozconfig-variant: plain-opt + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-nasm + - win64-node + - win64-cbindgen + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + - nsis + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null + +win32-shippable/opt: + description: "Win32 Opt Shippable" + use-pgo: true + index: + product: firefox + job-name: win32-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + stub-installer: + by-release-type: + nightly: true + beta: true + release.*: true + esr.*: false + default: + by-project: + # browser/confvars.sh looks for nightly-try + try: true + default: false + shipping-phase: build + shipping-product: firefox + treeherder: + platform: windows2012-32-shippable/opt + symbol: Bpgo(B) + tier: 1 + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-shippable/opt: + description: "Win64 Shippable" + use-pgo: true + index: + product: firefox + job-name: win64-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: firefox + treeherder: + platform: windows2012-64-shippable/opt + symbol: Bpgo(B) + tier: 1 + run-on-projects: ['release'] + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + max_build_output_timeout: 3600 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-nightlyasrelease/opt: + description: "Win64 Opt Nightly-As-Release" + use-pgo: win64-shippable/opt + attributes: + enable-full-crashsymbols: true + index: + product: firefox + job-name: win64-nightlyasrelease-opt + treeherder: + platform: win64-nightlyasrelease/opt + symbol: Bpgo(Bnar) + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: "nightly-as-release" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + max_build_output_timeout: 3600 + mozconfig-variant: nightly-as-release + run-on-projects: [trunk-only] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win32-add-on-devel/opt: + description: "Windows32 add-on-devel" + index: + product: firefox + job-name: win32-add-on-devel + treeherder: + platform: windows2012-32-add-on-devel/opt + symbol: B + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win32-add-on-devel + mozconfig_platform: win32 + mozconfig-variant: add-on-devel + run-on-projects: ['mozilla-beta', 'mozilla-release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-add-on-devel/opt: + description: "Windows64 add-on-devel" + index: + product: firefox + job-name: win64-add-on-devel + treeherder: + platform: windows2012-64-add-on-devel/opt + symbol: B + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64-on-devel + mozconfig_platform: win64 + mozconfig-variant: add-on-devel + run-on-projects: ['mozilla-beta', 'mozilla-release'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-noopt/debug: + description: "Win64 No-optimize Debug" + index: + product: firefox + job-name: win64-noopt-debug + treeherder: + platform: windows2012-64-noopt/debug + symbol: B + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/noopt_debug.py + mozconfig-variant: noopt-debug + extra-config: + mozconfig_platform: win64 + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win32-noopt/debug: + description: "Win32 No-optimize Debug" + index: + product: firefox + job-name: win32-noopt-debug + treeherder: + platform: windows2012-32-noopt/debug + symbol: B + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win32/noopt_debug.py + mozconfig-variant: noopt-debug + extra-config: + mozconfig_platform: win32 + run-on-projects: ['mozilla-central'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win32-rusttests/opt: + description: "Win32 Rust Tests Opt" + index: + product: firefox + job-name: win32-rusttests-opt + treeherder: + platform: windows2012-32/opt + symbol: BR + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win32.py + extra-config: + stage_platform: win32-rusttests + app_name: tools/rusttests + disable_package_metrics: true + mozconfig-variant: rusttests + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-cbindgen + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + optimization: + test-inclusive: [rusttests] + +win32-rusttests/debug: + description: "Win32 Rust Tests Debug" + index: + product: firefox + job-name: win32-rusttests-debug + treeherder: + platform: windows2012-32/debug + symbol: BR + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win32.py + extra-config: + stage_platform: win32-rusttests + app_name: tools/rusttests + disable_package_metrics: true + mozconfig-variant: rusttests-debug + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-cbindgen + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + optimization: + test-inclusive: [rusttests] + +win64-rusttests/opt: + description: "Win64 Rust Tests Opt" + index: + product: firefox + job-name: win64-rusttests-opt + treeherder: + platform: windows2012-64/opt + symbol: BR + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + - builds/taskcluster_sub_win64/rusttests_opt.py + mozconfig-variant: rusttests + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-cbindgen + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + optimization: + test-inclusive: [rusttests] + +win64-rusttests/debug: + description: "Win64 Rust Tests Debug" + index: + product: firefox + job-name: win64-rusttests-debug + treeherder: + platform: windows2012-64/debug + symbol: BR + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 5400 + env: + PERFHERDER_EXTRA_OPTIONS: rusttests + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + - builds/taskcluster_sub_win64/rusttests_opt.py + mozconfig-variant: rusttests-debug + run-on-projects: ['trunk'] + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-cbindgen + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + optimization: + test-inclusive: [rusttests] + +win64-ccov/opt: + description: "Win64 Opt Code Coverage" + index: + product: firefox + job-name: win64-ccov-opt + attributes: + enable-build-signing: true + treeherder: + platform: windows2012-64/ccov + symbol: B + tier: 2 + worker-type: b-win2022 + worker: + artifacts: + - name: public/code-coverage-grcov.zip + path: workspace\obj-build\code-coverage-grcov.zip + type: file + max-run-time: 9000 + env: + RUSTC_BOOTSTRAP: '1' + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + - builds/taskcluster_sub_win64/ccov_opt.py + mozconfig-variant: code-coverage + run-on-projects: ['mozilla-central'] + use-sccache: false + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-rust-size + - win64-cbindgen + - win64-grcov + - win64-sccache + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + - nsis + - sysroot-wasm32-wasi + +win64-fuzzing-ccov/opt: + description: "Win64 Fuzzing Opt w/ Coverage" + index: + product: firefox + job-name: win64-ccov-fuzzing-opt + treeherder: + platform: windows2012-64/ccov + symbol: Bocf + worker-type: b-win2022 + worker: + artifacts: + - name: public/code-coverage-grcov.zip + path: workspace\obj-build\code-coverage-grcov.zip + type: file + max-run-time: 9000 + env: + RUSTC_BOOTSTRAP: '1' + PERFHERDER_EXTRA_OPTIONS: "fuzzing-ccov" + run: + using: mozharness + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + - builds/taskcluster_sub_win64/ccov_opt.py + script: mozharness/scripts/fx_desktop_build.py + secrets: true + mozconfig-variant: fuzzing-ccov + tooltool-downloads: public + run-on-projects: ['mozilla-central'] + use-sccache: false + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-rust-size + - win64-cbindgen + - win64-grcov + - win64-sccache + - win64-nasm + - win64-node + - win64-winchecksec + - win64-mozmake + - win64-dump_syms + - vs + - nsis + - sysroot-wasm32-wasi + +win64-asan/debug: + description: "Win64 Debug ASAN" + index: + product: firefox + job-name: win64-asan-debug + treeherder: + platform: windows2012-64/asan + symbol: Bd + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: "debug asan" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/asan_debug.py + extra-config: + mozconfig_platform: win64 + mozconfig-variant: debug-asan + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win64-asan/opt: + description: "Win64 Opt ASAN" + index: + product: firefox + job-name: win64-asan-opt + treeherder: + platform: windows2012-64/asan + symbol: Bo + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: "opt asan" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64-asan + mozconfig_platform: win64 + mozconfig-variant: nightly-asan + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win + +win64-asan-reporter-shippable/opt: + description: "Win64 Opt ASAN Reporter (Shippable)" + attributes: + shippable: true + shipping-product: + by-release-type: + nightly: firefox + default: null + index: + product: firefox + job-name: win64-asan-reporter-opt + type: shippable + treeherder: + platform: win64-asan-reporter/opt + symbol: BoR + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: "asan-reporter" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/asan_reporter_opt.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64-asan-reporter + mozconfig_platform: win64 + max_build_output_timeout: 3600 + mozconfig-variant: nightly-asan-reporter + mar-channel-id: + firefox-mozilla-central-asan + accepted-mar-channel-ids: + firefox-mozilla-central-asan + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win + +win64-asan-fuzzing/opt: + description: "Win64 Fuzzing Opt ASAN" + index: + product: firefox + job-name: win64-fuzzing-asan-opt + treeherder: + platform: windows2012-64/asan + symbol: Bof + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: fuzzing-asan + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64-fuzzing-asan + mozconfig_platform: win64 + mozconfig-variant: nightly-fuzzing-asan + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win32-devedition/opt: + description: "Win32 Dev Edition (shippable)" + use-pgo: win32-shippable/opt + index: + product: devedition + job-name: win32-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + stub-installer: + by-release-type: + nightly: true + beta: true + release.*: true + default: + by-project: + # browser/confvars.sh looks for nightly-try + try: true + default: false + shipping-phase: build + shipping-product: devedition + treeherder: + platform: windows2012-32-devedition/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + options: [append-env-variables-from-configs] + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win32-devedition + mozconfig_platform: win32 + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-devedition/opt: + description: "Win64 Dev Edition (shippable)" + use-pgo: win64-shippable/opt + index: + product: devedition + job-name: win64-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: devedition + treeherder: + platform: windows2012-64-devedition/opt + symbol: B + tier: 1 + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64-devedition + mozconfig_platform: win64 + max_build_output_timeout: 3600 + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-aarch64/debug: + description: "AArch64 Win64 Debug" + index: + product: firefox + job-name: win64-aarch64-debug + attributes: + enable-full-crashsymbols: true + treeherder: + platform: windows2012-aarch64/debug + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/debug.py + extra-config: + mozconfig_platform: win64-aarch64 + mozconfig-variant: debug + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win64-aarch64/opt: + description: "AArch64 Win64 Opt" + index: + product: firefox + job-name: win64-aarch64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-aarch64/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64-aarch64 + mozconfig_platform: win64-aarch64 + run-on-projects: ['integration'] + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-sccache + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + optimization: + skip-unless-expanded: null + +win64-aarch64-shippable/opt: + description: "AArch64 Win64 Shippable" + use-pgo: win64-shippable/opt + index: + product: firefox + job-name: win64-aarch64-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: firefox + treeherder: + platform: windows2012-aarch64-shippable/opt + symbol: Bpgo(B) + tier: 1 + run-on-projects: ['release'] + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64-aarch64 + mozconfig_platform: win64-aarch64 + max_build_output_timeout: 3600 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win + +win64-aarch64-devedition/opt: + description: "AArch64 Win64 Dev Edition (shippable)" + use-pgo: win64-shippable/opt + index: + product: devedition + job-name: win64-aarch64-opt + type: shippable + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: devedition + treeherder: + platform: windows2012-aarch64-devedition/opt + symbol: Bpgo(B) + tier: 1 + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 10800 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [get-secrets, build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64-aarch64 + mozconfig_platform: win64-aarch64 + max_build_output_timeout: 3600 + mozconfig-variant: devedition + run-on-projects: ['mozilla-beta'] + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + fetch: + - upx-3.95-win diff --git a/taskcluster/ci/code-review/kind.yml b/taskcluster/ci/code-review/kind.yml new file mode 100644 index 0000000000..b57af6acaf --- /dev/null +++ b/taskcluster/ci/code-review/kind.yml @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.code_review:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - source-test + - toolchain + +job-defaults: + use-system-python: false + +jobs: + issues: + label: code-review-issues + description: List all issues found in static analysis and linting tasks + worker-type: b-linux-gcp + + # Only run by using the code-review target_tasks_method + run-on-projects: [] + + # This option permits to run the task + # regardless of the soft-dependencies tasks exit status + # as we are interested in the task failures + requires: all-resolved + + # Publish on pulse + routes: + - project.relman.codereview.v1.try_ending + + # Dummy execution for now, we only need the pulse message + worker: + docker-image: + in-tree: debian12-base + max-run-time: 600 + run: + using: run-task + checkout: false + command: /bin/true diff --git a/taskcluster/ci/condprof/kind.yml b/taskcluster/ci/condprof/kind.yml new file mode 100644 index 0000000000..75afd4a3e2 --- /dev/null +++ b/taskcluster/ci/condprof/kind.yml @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - build + - toolchain + +transforms: + - gecko_taskgraph.transforms.condprof:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + kind: other + tier: 2 + scenarios: + - settled + run: + using: run-task + cwd: fetches/condprofile + checkout: false + tooltool-downloads: public + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + worker: + taskcluster-proxy: true + max-run-time: 10800 + use-system-python: false +jobs: + windows2012-64-firefox: + worker-type: b-win2022 + description: Creates or updates conditioned profiles on Win64 + run-on-projects: ['mozilla-central'] + treeherder: + symbol: condprof(firefox) + platform: windows2012-64/opt + index: + product: firefox + job-name: condprof-win64 + dependencies: + build: build-win64-shippable/opt + scenarios: + - settled + - full + fetches: + build: + - artifact: target.condprof.tests.tar.gz + - artifact: target.zip + toolchain: + - win64-geckodriver + run: + command: >- + python3.exe condprof/main.py + ${EXTRA_ARGS} + --geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --firefox ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --scenario ${SCENARIO} + ../../archive + worker: + artifacts: + - name: public/condprof + path: archive + type: directory + linux64-firefox: + worker-type: b-linux-gcp + description: Creates or updates conditioned profiles on Linux + run-on-projects: ['mozilla-central'] + treeherder: + symbol: condprof(firefox) + platform: linux64/opt + index: + product: firefox + job-name: condprof-linux64 + dependencies: + build: build-linux64-shippable/opt + scenarios: + - settled + - full + fetches: + build: + - artifact: target.condprof.tests.tar.gz + - artifact: target.tar.bz2 + toolchain: + - linux64-geckodriver + run: + command: >- + python3 -m venv . && + bin/python3 condprof/main.py + ${EXTRA_ARGS} + --geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --firefox ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --scenario ${SCENARIO} + ../../archive + worker: + docker-image: {in-tree: condprof} + artifacts: + - name: public/condprof + path: /builds/worker/archive + type: directory + macosx64-firefox: + worker-type: t-osx-1015-r8 + description: Creates or updates conditioned profiles on macOS + run-on-projects: ['mozilla-central'] + treeherder: + symbol: condprof(firefox) + platform: macosx1015-64-qr/opt + index: + product: firefox + job-name: condprof-macosx64 + dependencies: + build: build-macosx64-shippable/opt + fetches: + build: + - artifact: target.condprof.tests.tar.gz + - artifact: target.dmg + extract: false + toolchain: + - macosx64-geckodriver + run: + command: >- + python3 -m venv . && + bin/python3 condprof/main.py + ${EXTRA_ARGS} + --geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --firefox ${MOZ_FETCHES_DIR}/target.dmg + --scenario ${SCENARIO} + ../../archive + worker: + artifacts: + - name: public/condprof + path: archive + type: directory diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 0000000000..73496c538b --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,879 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +trust-domain: gecko +project-repo-param-prefix: '' +product-dir: 'browser' +treeherder: + group-names: + 'cram': 'Cram tests' + 'js-bench-sm': 'JavaScript shell benchmarks with Spidermonkey' + 'js-bench-v8': 'JavaScript shell benchmarks with Google V8' + 'node': 'Node tests' + 'py3': 'Python 3 unit tests' + 'A': 'Android Gradle tests' + 'Bpgo': 'Profile-guided optimization builds' + 'Btime': 'Browsertime performance tests on Firefox' + 'Btime-cache': 'Browsertime performance tests on Firefox with populated bytecode cache.' + 'Btime-1proc': 'Browsertime performance tests on Firefox without e10s' + 'Btime-nofis': 'Browsertime tests on Firefox without fission enabled' + 'Btime-P-nofis': 'Browsertime power tests on Firefox without fission enabled' + 'Btime-Prof-nofis': 'Browsertime tests on Firefox with profiling and without fission enabled' + 'Btime-P-nofis-refbrow': 'Browsertime Power Usage Tests on reference browser without fission enabled' + 'Btime-P-nofis-fenix': 'Browsertime Power Usage Tests on Fenix without fission enabled' + 'Btime-live-nofis-fenix': 'Browsertime tests on Fenix without fission enabled using live sites' + 'Btime-nofis-fenix': 'Browsertime tests on Fenix without fission enabled' + 'Btime-webext-nofis-fenix': 'Browsertime performance tests on Fenix with extensions and without fission enabled' + 'Btime-webext': 'Browsertime performance tests on Firefox with extensions' + 'Btime-live-nofis': 'Browsertime tests on Firefox without fission enabled using live sites' + 'Btime-nofis-refbrow': 'Browsertime tests on reference browser without fission enabled' + 'Btime-live-nofis-ChR': 'Browsertime tests on Chrome without fission enabled using live sites' + 'Btime-Prof': 'Browsertime performance tests on Firefox with Gecko Profiling' + 'Btime-Prof-1proc': 'Browsertime performance tests on Firefox with Gecko Profiling and without e10s' + 'Btime-live': 'Browsertime performance tests on Firefox' + 'Btime-live-fenix': 'Browsertime performance tests on Firefox' + 'Btime-live-ChR': 'Browsertime performance tests on Firefox' + 'Btime-live-Cr': 'Browsertime performance tests on Google Chromium' + 'Btime-live-Saf': 'Browsertime performance tests on Safari' + 'Btime-ChR': 'Browsertime performance tests on Google Chrome Release' + 'Btime-nofis-ChR': 'Browsertime performance tests on Google Chrome Release without fission enabled' + 'Btime-Cr': 'Browsertime performance tests on Google Chromium' + 'Btime-P': 'Browsertime power tests on Firefox/Geckoview' + 'Btime-P-fenix': 'Browsertime power tests on Fenix' + 'Btime-P-refbrow': 'Browsertime power tests on reference browser' + 'Btime-fenix': 'Browsertime performance tests on Fenix' + 'Btime-refbrow': 'Browsertime performance tests on the reference browser' + 'Btime-Saf': 'Browsertime performance tests on Safari' + 'Btime-CaR': 'Browsertime performance tests on Chromium-as-Release' + 'Btime-nofis-CaR': 'Browsertime performance tests on Chromium-as-Release without fission enabled' + 'Fetch': 'Fetch and store content' + 'Fxfn': 'Firefox functional tests' + 'l10n-bump': 'L10n Bumper' + 'M': 'Mochitests' + 'M-cf': 'Mochitests confirm failure' + 'M-condprof': "Mochitests with conditioned profile" + 'M-condprof-cf': "Mochitests confirm failure with conditioned profile" + 'M-fis-hv': "Mochitests that run on Fission with isolateHighValue isolation strategy (Android-specific)" + 'M-fis-hv-cf': "Mochitests confirm failure that run on Fission with isolateHighValue isolation strategy (Android-specific)" + 'M-headless': 'Headless Mochitests' + 'M-headless-cf': 'Headless Mochitests confirm failure' + 'M-headless-spi-nw': 'Headless Mochitests with fission and socketprocess networking.' + 'M-headless-spi-nw-cf': 'Headless Mochitests confirm failure with fission and socketprocess networking.' + 'M-http3': 'Mochitests with Http/3 server' + 'M-http3-cf': 'Mochitests confirm failure with Http/3 server' + 'M-http2': 'Mochitests with Http/2 server' + 'M-http2-cf': 'Mochitests confirm failure with Http/2 server' + 'M-aab': 'Mochitests with AAB test_runner.' + 'M-aab-cf': 'Mochitests confirm failure with AAB test_runner.' + 'M-aab-nofis': 'Mochitests with AAB test_runner without fission enabled.' + 'M-aab-nofis-cf': 'Mochitests confirm failure with AAB test_runner without fission enabled.' + 'M-1proc': 'Mochitests without e10s or fission' + 'M-1proc-cf': 'Mochitests confirm failure without e10s or fission' + 'M-a11y-checks': 'Mochitests with accessibility checks enabled' + 'M-a11y-checks-cf': 'Mochitests confirm failure with accessibility checks enabled' + 'M-xorig': 'Mochitests with cross-origin and fission enabled' + 'M-xorig-cf': 'Mochitests confirm failure with cross-origin and fission enabled' + 'M-gli': 'Mochitests with WebGL over IPC' + 'M-gli-cf': 'Mochitests confirm failure with WebGL over IPC' + 'M-nofis': 'Mochitests without fission enabled' + 'M-nofis-cf': 'Mochitests confirm failure without fission enabled' + 'M-spi': 'Mochitests with socket process' + 'M-spi-cf': 'Mochitests confirm failure with socket process' + 'M-spi-nofis': 'Mochitests with socket process without fission enabled' + 'M-spi-nofis-cf': 'Mochitests confirm failure with socket process without fission enabled' + 'M-spi-nw': 'Mochitests with networking on socket process' + 'M-spi-nw-cf': 'Mochitests confirm failure with networking on socket process' + 'M-spi-nw-nofis': 'Mochitests with networking on socket process without fission enabled' + 'M-spi-nw-nofis-cf': 'Mochitests confirm failure with networking on socket process without fission enabled' + 'M-spi-nw-1proc': 'Mochitests with networking on socket process without e10s' + 'M-spi-nw-1proc-cf': 'Mochitests confirm failure with networking on socket process without e10s' + 'M-swr': 'Mochitests with software webrender enabled' + 'M-swr-cf': 'Mochitests confirm failure with software webrender enabled' + 'M-swr-a11y-checks': 'Mochitests with software webrender and accessibility checks enabled' + 'M-swr-a11y-checks-cf': 'Mochitests confirm failure with software webrender and accessibility checks enabled' + 'M-swr-1proc': 'Mochitests with software webrender enabled without e10s' + 'M-swr-1proc-cf': 'Mochitests confirm failure with software webrender enabled without e10s' + 'M-swr-nofis': 'Mochitests with software webrender without fission enabled' + 'M-swr-nofis-cf': 'Mochitests confirm failure with software webrender without fission enabled' + 'M-wmfme': 'Mochitests with Windows Media Foundation media engine enabled' + 'M-wmfme-cf': 'Mochitests confirm failure with Windows Media Foundation media engine enabled' + 'M-mda-gpu': 'Mochitests Media on GPU worker' + 'M-mda-gpu-cf': 'Mochitests confirm failure Media on GPU worker' + 'M-f': 'Mochitest failures' + 'M-f-swr': 'Mochitest failures software webrender' + 'M-f-cf': 'Mochitest failures w/confirm failure' + 'M-f-swr-cf': 'Mochitest failures software webrender w/confirm failure' + 'M-dt-no-eft-nofis': 'DevTools Mochitests with EFT disabled without fission enabled' + 'M-dt-no-eft-nofis-cf': 'DevTools Mochitests confirm failure with EFT disabled without fission enabled' + 'M-msix': 'Mochitests from MSIX builds' + 'M-msix-cf': 'Mochitests confirm failure from MSIX builds' + 'MSI': 'Repack installers into MSIs' + 'MSIs': 'Signing of Repacked installers of MSIs' + 'MSIX': 'Repack into MSIX package' + 'MSIXs': 'Signing of Repack into MSIX package' + 'Pup': "Puppeteer tests" + 'R': 'Reftests' + 'R-cf': 'Reftests confirm failure' + 'R-nofis': 'Reftests without fission enabled' + 'R-nofis-cf': 'Reftests confirm failure without fission enabled' + 'R-swr': 'Reftests with software webrender enabled' + 'R-swr-cf': 'Reftests confirm failure with software webrender enabled' + 'R-swr-nofis': 'Reftests with software webrender enabled without fission enabled' + 'R-swr-nofis-cf': 'Reftests confirm failure with software webrender enabled without fission enabled' + 'R-wr-dc0': 'Reftests with dcomp disabled' + 'R-wr-dc0-cf': 'Reftests confirm failure with dcomp disabled' + 'R-wr-dc1-p': 'Reftests with dcomp present but not overlays' + 'R-wr-dc1-p-cf': 'Reftests confirm failure with dcomp present but not overlays' + 'R-wr-dc2-o': 'Reftests with dcomp overlays but not compositing' + 'R-wr-dc2-o-cf': 'Reftests confirm failure with dcomp overlays but not compositing' + 'R-wr-dc3-c': 'Reftests with dcomp compositing' + 'R-wr-dc3-c-cf': 'Reftests confirm failure with dcomp compositing' + 'Rap': 'Raptor performance tests on Firefox' + 'Rap-live': 'Raptor performance tests on Firefox with live sites' + 'Rap-Prof': 'Raptor performance tests on Firefox with Gecko Profiling' + 'Rap-ChR': 'Raptor performance tests on Google Chrome Release' + 'Rap-Cr': 'Raptor performance tests on Google Chromium' + 'Rap-refbrow': 'Raptor performance tests on the reference browser' + 'T': 'Talos performance tests' + 'T-gli': 'Talos performance tests with WebGL over IPC' + 'T-Prof': 'Talos performance tests on Firefox with Gecko Profiling' + 'T-Prof-gli': 'Talos performance tests with WebGL over IPC and Gecko Profiling' + 'T-swr': 'Talos performance tests with software webrender enabled' + 'tt': 'Telemetry tests' + 'tt-nofis': 'Telemetry tests without fission enabled' + 'SY': 'Are we slim yet tests by TaskCluster' + 'W': 'Web platform tests' + 'W-cf': 'Web platform tests confirm failure' + 'W-nofis': 'Web platform tests without fission enabled' + 'W-nofis-cf': 'Web platform tests confirm failure without fission enabled' + 'W-headless': 'Headless web platform tests' + 'W-headless-cf': 'Headless web platform tests confirm failure' + 'W-swr-nofis': 'Web platform tests with software webrender enabled without fission enabled' + 'W-swr-nofis-cf': 'Web platform tests confirm failure with software webrender enabled without fission enabled' + 'W-swr': 'Web platform tests with software webrender enabled' + 'W-swr-cf': 'Web platform tests confirm falure with software webrender enabled' + 'W-b': 'Web platform tests (backlog)' + 'W-b-nofis': 'Web platform tests (backlog) without fission enabled' + 'W-pb': 'Web platform tests with private browsing enabled' + 'X': 'Xpcshell tests' + 'X-cf': 'Xpcshell tests confirm failure' + 'X-condprof': 'Xpcshell tests with a conditioned profile' + 'X-condprof-cf': 'Xpcshell tests confirm failure with a conditioned profile' + 'X-nofis': 'Xpcshell tests without fission enabled' + 'X-nofis-cf': 'Xpcshell tests confirm failure without fission enabled' + 'X-spi-nw': 'Xpcshell tests with networking on socket process' + 'X-spi-nw-cf': 'Xpcshell tests confirm failure with networking on socket process' + 'X-spi-nw-nofis': 'Xpcshell tests with networking on socket process without fission enabled' + 'X-spi-nw-nofis-cf': 'Xpcshell tests confirm failure with networking on socket process without fission enabled' + 'X-f': 'Xpcshell tests that fail on a given config' + 'X-f-spi-nw': 'Xpcshell tests that fail on a given config w/socket process' + 'X-msix': 'Xpcshell tests on msix packages' + 'X-msix-cf': 'Xpcshell tests confirm failure on msix packages' + 'L10n': 'Localised Repacks' + 'L10n-Rpk': 'Localized Repackaged Repacks' + 'deb-L10n': 'Localized Debian Repacks' + 'BM': 'Beetmover' + 'BMR': 'Beetmover repackages' + 'BM-apt': 'Beetmover .deb packages' + 'c-Up': 'Balrog submission of complete updates' + 'css': 'Checksum signing for source' + 'rs': 'Repackage signing' + 'BMcs': 'Beetmover checksums' + 'BMcslang': 'Beetmover checksums for language packs' + 'BMcss': 'Beetmover checksums for source' + 'Deb8': 'Packages for Debian 8' + 'Deb8-32': 'Packages for Debian 8 32-bits' + 'Deb9': 'Packages for Debian 9' + 'Deb10': 'Packages for Debian 10' + 'Deb11': 'Packages for Debian 11' + 'Deb12': 'Packages for Debian 12' + 'Ub18': 'Packages for Ubuntu 18.04' + 'Ub20': 'Packages for Ubuntu 20.04' + 'Ub22': 'Packages for Ubuntu 22.04' + 'I': 'Docker Image Builds' + 'TA': 'Toolchain builds for Android' + 'TL': 'Toolchain builds for Linux 64-bits' + 'TL32': 'Toolchain builds for Linux 32-bits' + 'TM': 'Toolchain builds for OSX' + 'TMW': 'Toolchain builds for Windows MinGW' + 'TW32': 'Toolchain builds for Windows 32-bits' + 'TW64': 'Toolchain builds for Windows 64-bits' + 'WMC32': 'MinGW-Clang builds for Windows 32-bits' + 'WMC64': 'MinGW-Clang builds for Windows 64-bits' + 'Searchfox': 'Searchfox builds' + 'SM': 'Spidermonkey builds' + 'p': 'Partial generation' + 'ps': 'Partials signing' + 'ms': 'Complete MAR signing' + 'ms-stage': 'Autograph-stage MAR signing test' + 'Rel': 'Release promotion' + 'Snap': 'Snap image generation' + 'Flatpak': 'Flatpak image generation' + 'langpack': 'Langpack sigatures and uploads' + 'TPS': 'Sync tests' + 'UV': 'Update verify' + 'UVnext': 'Update verify for esr-next' + 'pydep': 'python dependency update' + 'WR': 'WebRender standalone' + 'Wgpu': 'WebGPU standalone' + 'cpp': 'C/C++ checks' + 'pedantic': 'pedantic checks' + 'text': 'Check on texts' + 'misc': 'Misc checks' + 'js': 'JavaScript checks' + 'py': 'Python checks' + 'java': 'Java checks' + 'rust': 'Rust checks' + 'Static-Analysis': 'Full tree static-analysis' + 'SS': 'Shadow scheduler' + 'Sel': 'Selenium Snap tests' + 'Sentry': 'Sentry synchronization' + 'test-info': 'Test manifest skip/fail information' + 'condprof': 'Conditioned Profile Builder' + 'doc': 'Documentation' + 'GhS': 'GitHub Synchronization' + 'perftest': 'Performance tests' + 'perftest-detect': 'Performance regression detection tooling' + 'perftest-chrome': 'Performance tests using Chrone' + 'perftest-http3': 'Performance tests with HTTP/3' + 'l10n': 'Localization checks' + 'fxrec': 'Desktop startup recorder (fxrecord)' + 'wc': 'webcompat' + 'Boot': 'Bootstrap' + 'Attr-L10n': 'Build Attribution' + +index: + products: + - 'firefox' + - 'fennec' + - 'mobile' + - 'static-analysis' + - 'devedition' + - 'source' + - 'system-symbols' + - 'geckodriver' + +try: + # We have a few platforms for which we want to do some "extra" builds, or at + # least build-ish things. Sort of. Anyway, these other things are implemented + # as different "platforms". These do *not* automatically ride along with "-p + # all" + ridealong-builds: + 'linux': + - 'sm-plain-linux32' + - 'sm-arm-sim-linux32' + 'linux64': + - 'sm-plain-linux64' + - 'sm-nojit-linux64' + - 'sm-nonunified-linux64' + - 'sm-arm-sim-linux32' + - 'sm-arm64-sim-linux64' + - 'sm-compacting-linux64' + - 'sm-rootanalysis-linux64' + - 'sm-package-linux64' + - 'sm-tsan-linux64' + - 'sm-asan-linux64' + - 'sm-msan-linux64' + - 'sm-fuzzing-linux64' + 'win32': + - 'sm-plain-win32' + - 'sm-compacting-win32' + +release-promotion: + products: + - 'devedition' + - 'fennec' + - 'firefox' + rebuild-kinds: + - docker-image + - fetch + - packages + - toolchain + flavors: + promote_devedition: + product: devedition + target-tasks-method: promote_desktop + partial-updates: true + promote_firefox: + product: firefox + target-tasks-method: promote_desktop + partial-updates: true + promote_firefox_partner_repack: + product: firefox + rebuild-kinds: + - release-partner-repack + - release-partner-repack-chunking-dummy + - release-partner-repack-signing + - release-partner-repack-mac-signing + - release-partner-repack-mac-notarization + - release-partner-repack-repackage + - release-partner-repack-repackage-signing + - release-partner-repack-beetmover + target-tasks-method: promote_desktop + promote_firefox_partner_attribution: + product: firefox + rebuild-kinds: + - release-partner-attribution + - release-partner-attribution-beetmover + target-tasks-method: promote_desktop + promote_firefox_rc: + product: firefox + is-rc: true + target-tasks-method: promote_desktop + partial-updates: true + push_devedition: + product: devedition + target-tasks-method: push_desktop + partial-updates: true + push_firefox: + product: firefox + target-tasks-method: push_desktop + partial-updates: true + ship_devedition: + product: devedition + target-tasks-method: ship_desktop + version-bump: true + partial-updates: true + ship_firefox: + product: firefox + target-tasks-method: ship_desktop + version-bump: true + partial-updates: true + ship_firefox_rc: + product: firefox + is-rc: true + target-tasks-method: ship_desktop + partial-updates: true + + +merge-automation: + behaviors: + central-to-beta: + fetch-version-from: "browser/config/version.txt" + version-files: + - filename: "config/milestone.txt" + new-suffix: '' + - filename: "browser/config/version.txt" + new-suffix: '' + - filename: "browser/config/version_display.txt" + new-suffix: 'b1' + replacements: + - - browser/config/mozconfigs/linux32/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + - - browser/config/mozconfigs/linux64/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + - - browser/config/mozconfigs/win32/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + - - browser/config/mozconfigs/win64/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + - - browser/config/mozconfigs/win64-aarch64/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + - - browser/config/mozconfigs/macosx64/l10n-mozconfig + - ac_add_options --with-branding=browser/branding/nightly + - ac_add_options --enable-official-branding + merge-old-head: true + base-tag: 'FIREFOX_BETA_{major_version}_BASE' + end-tag: 'FIREFOX_BETA_{major_version}_END' + from-repo: 'https://hg.mozilla.org/mozilla-central' + from-branch: 'central' + to-repo: 'https://hg.mozilla.org/releases/mozilla-beta' + to-branch: 'beta' + early-to-late-beta: + fetch-version-from: "browser/config/version.txt" + version-files: [] + replacements: + - - build/defines.sh + - EARLY_BETA_OR_EARLIER=1 + - EARLY_BETA_OR_EARLIER= + merge-old-head: false + to-repo: 'https://hg.mozilla.org/releases/mozilla-beta' + to-branch: 'beta' + beta-to-release: + fetch-version-from: "browser/config/version.txt" + version-files: + - filename: "browser/config/version_display.txt" + new-suffix: '' + replacements: [] + merge-old-head: true + base-tag: 'FIREFOX_RELEASE_{major_version}_BASE' + end-tag: 'FIREFOX_RELEASE_{major_version}_END' + from-repo: 'https://hg.mozilla.org/releases/mozilla-beta' + from-branch: 'beta' + to-repo: 'https://hg.mozilla.org/releases/mozilla-release' + to-branch: 'release' + release-to-esr: + fetch-version-from: "browser/config/version.txt" + version-files: + - filename: "browser/config/version_display.txt" + new-suffix: 'esr' + replacements: [] + merge-old-head: false + end-tag: "FIREFOX_ESR_{major_version}_BASE" + to-repo: 'https://hg.mozilla.org/releases/mozilla-esr115' + to-branch: 'esr115' + bump-central: + fetch-version-from: "browser/config/version.txt" + version-files: + - filename: "config/milestone.txt" + version-bump: "major" + new-suffix: 'a1' + - filename: "browser/config/version.txt" + version-bump: "major" + new-suffix: 'a1' + - filename: "browser/config/version_display.txt" + version-bump: "major" + new-suffix: 'a1' + replacements: + - - "services/sync/modules/constants.sys.mjs" + - 'WEAVE_VERSION = "1.{current_weave_version}.0"' + - 'WEAVE_VERSION = "1.{next_weave_version}.0"' + merge-old-head: false + end-tag: 'FIREFOX_NIGHTLY_{major_version}_END' + to-repo: 'https://hg.mozilla.org/mozilla-central' + to-branch: 'central' + bump-esr115: + fetch-version-from: "browser/config/version.txt" + version-files: + - filename: "config/milestone.txt" + version-bump: "minor" + - filename: "browser/config/version.txt" + version-bump: "minor" + - filename: "browser/config/version_display.txt" + version-bump: "minor" + replacements: [] + merge-old-head: false + to-repo: 'https://hg.mozilla.org/releases/mozilla-esr115' + to-branch: 'esr115' + +scriptworker: + # See additional configuration in taskcluster/gecko_taskgraph/util/scriptworker.py + scope-prefix: 'project:releng' + +partner-urls: + release-partner-repack: + by-release-product: + default: null + firefox: + by-release-type: + default: null + beta|release.*: + by-release-level: + production: 'git@github.com:mozilla-partners/repack-manifests.git' + staging: 'git@github.com:moz-releng-automation-stage/repack-manifests.git' + esr.*: + by-release-level: + production: 'git@github.com:mozilla-partners/esr-repack-manifests.git' + staging: 'git@github.com:moz-releng-automation-stage/esr-repack-manifests.git' + release-partner-attribution: + by-release-product: + default: null + firefox: + by-release-type: + default: null + beta|release.*: + by-release-level: + production: 'git@github.com:mozilla-partners/repack-manifests.git' + staging: 'git@github.com:moz-releng-automation-stage/repack-manifests.git' + esr.*: + by-release-level: + production: 'git@github.com:mozilla-partners/esr-repack-manifests.git' + staging: 'git@github.com:moz-releng-automation-stage/esr-repack-manifests.git' + release-eme-free-repack: + by-release-product: + default: null + firefox: + by-release-type: + default: null + beta|release.*: + by-release-level: + production: 'git@github.com:mozilla-partners/mozilla-EME-free-manifest.git' + staging: 'git@github.com:moz-releng-automation-stage/mozilla-EME-free-manifest.git' + + +task-priority: + by-project: + 'mozilla-release': 'highest' + 'mozilla-esr.*': 'very-high' + 'mozilla-beta': 'high' + 'mozilla-central': 'medium' + 'autoland': 'low' + 'mozilla-inbound': 'low' + 'default': 'very-low' + +taskgraph: + register: gecko_taskgraph:register + +workers: + aliases: + b-linux.*: + provisioner: '{trust-domain}-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + b-win2012: + provisioner: '{trust-domain}-{level}' + implementation: generic-worker + os: windows + worker-type: '{alias}-azure' + b-win2022: + provisioner: '{trust-domain}-{level}' + implementation: generic-worker + os: windows + worker-type: '{alias}' + image: + provisioner: '{trust-domain}-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + images: + provisioner: '{trust-domain}-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + images-gcp: + provisioner: '{trust-domain}-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + addon: + provisioner: scriptworker-k8s + implementation: push-addons + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-addon' + staging: '{trust-domain}-1-addon' + balrog: + provisioner: scriptworker-k8s + implementation: balrog + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-balrog' + staging: '{trust-domain}-1-balrog' + bouncer: + provisioner: scriptworker-k8s + # Note that this implementation doesn't correspond with an + # payload_builder, there are several `bouncer-*` implemenations. + implementation: bouncer + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-bouncer' + staging: '{trust-domain}-1-bouncer' + beetmover: + provisioner: scriptworker-k8s + implementation: beetmover + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-beetmover' + staging: '{trust-domain}-1-beetmover' + shipit: + provisioner: scriptworker-k8s + implementation: shipit + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-shipit' + staging: '{trust-domain}-1-shipit' + linux-depsigning: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: linux + worker-type: '{trust-domain}-t-signing' + linux-signing: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: linux + worker-type: + by-release-level: + production: '{trust-domain}-3-signing' + staging: '{trust-domain}-t-signing' + mac-depsigning: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: macosx + worker-type: depsigning-mac-v1 + mac-signing: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: macosx + worker-type: + by-release-level: + production: signing-mac-v1 + staging: depsigning-mac-v1 + tree: + provisioner: scriptworker-k8s + implementation: treescript + os: scriptworker + worker-type: + by-release-level: + production: '{trust-domain}-3-tree' + staging: + by-project: + autoland: '{trust-domain}-3-tree' + default: '{trust-domain}-1-tree' + tree-dev: + provisioner: scriptworker-k8s + implementation: treescript + os: scriptworker + worker-type: '{trust-domain}-1-tree-dev' + t-bitbar-gw.*: + provisioner: proj-autophone + implementation: generic-worker + os: linux-bitbar + worker-type: 'gecko-{alias}' + t-linux(-large|-xlarge|-xlarge-source): + provisioner: '{trust-domain}-t' + implementation: docker-worker + os: linux + worker-type: '{alias}-gcp' + t-linux-kvm: + provisioner: '{trust-domain}-t' + implementation: docker-worker + os: linux + worker-type: 't-linux-kvm-gcp' + t-linux-talos: + provisioner: releng-hardware + implementation: generic-worker + os: linux + worker-type: 'gecko-{alias}' + t-linux-talos-1804: + provisioner: releng-hardware + implementation: generic-worker + os: linux + worker-type: 'gecko-{alias}' + t-linux-wayland: + provisioner: '{trust-domain}-t' + implementation: generic-worker + os: linux + worker-type: 't-linux-vm-2204-wayland' + t-osx-1015-r8: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + t-osx-1100-m1: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + t-osx-1400-m2: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + t-osx-1015-power: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + t-linux-xlarge-pgo: + provisioner: + by-level: + '3': '{trust-domain}-{level}' + default: '{trust-domain}-t' + implementation: docker-worker + os: linux + worker-type: 't-linux-xlarge-gcp' + b-osx-1015: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: + by-level: + '3': 'gecko-3-b-osx-1015' + default: 'gecko-1-b-osx-1015' + b-osx-arm64: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: + by-level: + '3': 'gecko-3-b-osx-arm64' + default: 'gecko-1-b-osx-arm64' + t-win10-64(|-gpu-s|-source): + provisioner: '{trust-domain}-t' + implementation: generic-worker + os: windows + worker-type: '{alias}' + t-win10-64(-hw|-ref-hw|-1803-hw): + provisioner: releng-hardware + implementation: generic-worker + os: windows + worker-type: 'gecko-{alias}' + win11-64-2009(-hw-ref): + provisioner: releng-hardware + implementation: generic-worker + os: windows + worker-type: '{alias}' + win10-64-2009(|-gpu|-ssd|-source|-ssd-gpu): + provisioner: 'gecko-t' + implementation: generic-worker + os: windows + worker-type: '{alias}' + t-win11-64(|-gpu-s|-source): + provisioner: '{trust-domain}-t' + implementation: generic-worker + os: windows + worker-type: '{alias}' + win11-64-2009(|-gpu|-ssd|-source|-ssd-gpu): + provisioner: 'gecko-t' + implementation: generic-worker + os: windows + worker-type: '{alias}' + t-win7-32-gpu: + provisioner: '{trust-domain}-t' + implementation: generic-worker + os: windows + worker-type: '{alias}' + t-win7-32: + provisioner: releng-hardware + implementation: generic-worker + os: windows + worker-type: 'gecko-{alias}-hw' + t-win64-aarch64-laptop: + provisioner: bitbar + implementation: generic-worker + os: windows + worker-type: 'gecko-{alias}' + succeed: + provisioner: built-in + implementation: succeed + os: none + worker-type: succeed + misc: + provisioner: '{trust-domain}-t' + implementation: docker-worker + os: linux + worker-type: misc-gcp + + +mac-notarization: + mac-entitlements: + by-platform: + macosx64.*: + by-release-level: + production: security/mac/hardenedruntime/v1/production/browser.xml + default: security/mac/hardenedruntime/v1/developer/browser.xml + default: '' + mac-requirements: + by-platform: + macosx64.*: build/package/mac_osx/requirements.plist + default: '' + +mac-signing: + hardened-sign-config: + by-hardened-signing-type: + production: + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/production/plugin-container.xml + globs: + - "/Contents/MacOS/plugin-container.app" + + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/production/media-plugin-helper.xml + globs: + - "/Contents/MacOS/media-plugin-helper.app" + + - deep: false + runtime: true + force: true + # These files are signed wihtout entitlements + globs: + - "/Contents/MacOS/crashreporter.app" + - "/Contents/MacOS/updater.app" + - "/Contents/Library/LaunchServices/org.mozilla.updater" + - "/Contents/MacOS/XUL" + - "/Contents/MacOS/pingsender" + - "/Contents/MacOS/minidump-analyzer" + - "/Contents/MacOS/*.dylib" + - "/Contents/Resources/gmp-clearkey/*/*.dylib" + + - deep: false + runtime: true + force: true + entitlements: + by-build-platform: + .*devedition.*: security/mac/hardenedruntime/v2/production/firefoxdeveloperedition.browser.xml + default: + by-project: + mozilla-central: security/mac/hardenedruntime/v2/production/nightly.browser.xml + default: security/mac/hardenedruntime/v2/production/firefox.browser.xml + globs: + - "/" # The .app + + default: + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/developer/plugin-container.xml + globs: + - "/Contents/MacOS/plugin-container.app" + + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/developer/media-plugin-helper.xml + globs: + - "/Contents/MacOS/media-plugin-helper.app" + + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/developer/utility.xml + globs: + - "/Contents/MacOS/crashreporter.app" + - "/Contents/MacOS/updater.app" + - "/Contents/Library/LaunchServices/org.mozilla.updater" + - "/Contents/MacOS/pingsender" + - "/Contents/MacOS/minidump-analyzer" + + - deep: false + runtime: true + force: true + # These files are signed without entitlements + globs: + - "/Contents/MacOS/XUL" + - "/Contents/MacOS/*.dylib" + - "/Contents/Resources/gmp-clearkey/*/*.dylib" + + - deep: false + runtime: true + force: true + entitlements: security/mac/hardenedruntime/v2/developer/browser.xml + globs: + - "/" # The .app + +expiration-policy: + by-project: + try: + default: 28 days + shortest: 7 days + short: 14 days + medium: 28 days + long: 28 days + autoland: + default: 1 year + shortest: 14 days + short: 3 months + medium: 1 year + # To avoid keeping shippable builds for over a year + long: 1 year + default: + default: 3 months + shortest: 7 days + short: 1 month + medium: 1 year + long: 1 year diff --git a/taskcluster/ci/cron-bouncer-check/kind.yml b/taskcluster/ci/cron-bouncer-check/kind.yml new file mode 100644 index 0000000000..62d2c80960 --- /dev/null +++ b/taskcluster/ci/cron-bouncer-check/kind.yml @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.bouncer_check:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: bouncer-check + description: bouncer check + worker-type: b-linux-gcp + worker: + max-run-time: 1200 + docker-image: {in-tree: "update-verify"} + run: + sparse-profile: mozharness + extra-config: + bouncer_prefix: "https://download.mozilla.org/" + attributes: + build_platform: linux64 + build_type: opt + cron: true + treeherder: + symbol: Rel(ckbouncer) + kind: test + tier: 1 + +jobs: + firefox: + shipping-product: firefox + index: + product: firefox + job-name: firefox-bouncer-check + run: + config: + by-release-type: + nightly: + - releases/bouncer_firefox_nightly.py + beta: + - releases/bouncer_firefox_beta.py + release: + - releases/bouncer_firefox_release.py + esr.*: + - releases/bouncer_firefox_esr.py + default: + - releases/bouncer_firefox_beta.py + product-field: + by-project: + mozilla-central: FIREFOX_NIGHTLY + mozilla-beta: LATEST_FIREFOX_RELEASED_DEVEL_VERSION + mozilla-release: LATEST_FIREFOX_VERSION + mozilla-esr115: FIREFOX_ESR + # mozilla-esrXX: FIREFOX_ESR_NEXT + default: LATEST_FIREFOX_DEVEL_VERSION + products-url: https://product-details.mozilla.org/1.0/firefox_versions.json + treeherder: + platform: firefox-release/opt + + devedition: + shipping-product: devedition + run-on-projects: [mozilla-beta] + index: + product: devedition + job-name: devedition-bouncer-check + run: + config: + by-project: + mozilla-beta: + - releases/bouncer_firefox_devedition.py + default: + - releases/dev_bouncer_firefox_devedition.py + product-field: + by-project: + mozilla-beta: LATEST_FIREFOX_RELEASED_DEVEL_VERSION + default: LATEST_FIREFOX_DEVEL_VERSION + products-url: https://product-details.mozilla.org/1.0/firefox_versions.json + treeherder: + platform: devedition-release/opt diff --git a/taskcluster/ci/diffoscope/artifacts.yml b/taskcluster/ci/diffoscope/artifacts.yml new file mode 100644 index 0000000000..0e901f50fd --- /dev/null +++ b/taskcluster/ci/diffoscope/artifacts.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + # Artifact builds always build with nightly branding, so these checks don't + # work on non-trunk branches. + run-on-projects: ['trunk'] + unpack: true + fail-on-diff: true + pre-diff-commands: + # jogfile.json is only created in artifact builds. + - rm b/"$RESOURCE_DIR"/jogfile.json + # Remove noise from differences in line numbers in preprocessor output + # due to #if/#else branches. + - sed -i '/^\/\/@line /d' {a,b}/"$RESOURCE_DIR"/browser/defaults/preferences/firefox.js + # bug 1825141 - telemetry.fog.artifact_build isn't stable between + # artifact and not-artifact builds, so remove its line if present. + - sed -i '/telemetry\.fog\.artifact\_build/d' b/"$RESOURCE_DIR"/browser/defaults/preferences/firefox.js + # artifact builds are not fed with accepted-mar-channel-ids + - sed -i '/ACCEPTED_MAR_CHANNEL_IDS/d' {a,b}/"$RESOURCE_DIR"/update-settings.ini + # Extra diffoscope arguments to account for: + # - about:buildconfig being expectedly different. + extra-args: >- + --exclude b/"$RESOURCE_DIR"/chrome/toolkit/content/global/buildconfig.html + # This is necessary to avoid building the dependencies on every push on autoland + # A more robust fix for this is https://bugzilla.mozilla.org/show_bug.cgi?id=1643346 + optimization: + skip-unless-expanded: null + +artifact-linux64-validation: + symbol: Linux64a + new: artifact-build-linux64-artifact/opt + original: build-linux64/opt + +artifact-macosx64-validation: + symbol: Macosx64a + new: artifact-build-macosx64-artifact/opt + original: build-macosx64/opt + +artifact-win64-validation: + symbol: Win64a + new: artifact-build-win64-artifact/opt + original: build-win64/opt + # Extra diffoscope arguments to account for: + # - about:buildconfig being expectedly different. + # - There are some differences in PE metadata in helper.exe because + # it's (re)built in the EME build, and that part of the build is + # not reproducible. + extra-args: >- + --exclude b/"$RESOURCE_DIR"/chrome/toolkit/content/global/buildconfig.html + --exclude b/firefox/uninstall/helper.exe diff --git a/taskcluster/ci/diffoscope/kind.yml b/taskcluster/ci/diffoscope/kind.yml new file mode 100644 index 0000000000..89b38fc665 --- /dev/null +++ b/taskcluster/ci/diffoscope/kind.yml @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - artifact-build + - build + - toolchain + +transforms: + - gecko_taskgraph.transforms.diffoscope:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +# Note: --exclude-command .--line-numbers is because of +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879003 +# That skips `objdump --disassemble --line-numbers` and falls back to +# `objdump --disassemble` +# Note: the .chk excludes are for files that are known to differ between +# builds because they are signed with an ephemeral private key that is +# generated for each build. +job-defaults: + tier: 2 + args: >- + --output-empty + --no-default-limits + --max-page-size 100000000 + --max-page-diff-block-lines 10000 + --exclude-directory-metadata=yes + --exclude-command .--line-numbers + +jobs-from: + - artifacts.yml + - reproducible.yml + +# Make a task for each diff we might want. The following are just examples, +# Both original and new can point to builds from the full set of tasks or +# from other sets through an index-search. Other kinds than `build` can be +# compared (for example, static-analysis), provided you adjust the +# kind-dependencies above. +# jobs: +# android-build-vs-previous-try: +# symbol: A +# new: build-android-arm/opt +# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.mobile.android-arm-opt} +# linux64-build-vs-previous-try: +# symbol: L +# new: build-linux64/opt +# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.linux64-opt} +# extra-args: >- +# --exclude-command .--hex-dump=.gnu_debuglink +# macosx-build-vs-previous-try: +# symbol: M +# new: build-macosx64/opt +# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.macosx64-opt} +# win32-build-vs-previous-try: +# symbol: W +# new: build-win32/opt +# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.win32-opt} diff --git a/taskcluster/ci/diffoscope/reproducible.yml b/taskcluster/ci/diffoscope/reproducible.yml new file mode 100644 index 0000000000..4f89bfca4e --- /dev/null +++ b/taskcluster/ci/diffoscope/reproducible.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + fail-on-diff: true + run-on-projects: ['mozilla-central'] + +reproducible-linux32: + symbol: Linux32 + new: build-linux-reproduced/opt + original: build-linux-shippable/opt + +reproducible-linux32-generated-files: + symbol: Linux32g + artifact: target.generated-files.tar.gz + new: build-linux-reproduced/opt + original: build-linux-shippable/opt + +reproducible-linux64: + symbol: Linux64 + new: build-linux64-reproduced/opt + original: build-linux64-shippable/opt + +reproducible-linux64-generated-files: + symbol: Linux64g + artifact: target.generated-files.tar.gz + new: build-linux64-reproduced/opt + original: build-linux64-shippable/opt diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 0000000000..149186fb3b --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,247 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - packages + +transforms: + - gecko_taskgraph.transforms.docker_image:transforms + - gecko_taskgraph.transforms.cached_tasks:transforms + - gecko_taskgraph.transforms.task:transforms + +# make a task for each docker-image we might want. For the moment, since we +# write artifacts for each, these are whitelisted, but ideally that will change +# (to use subdirectory clones of the proper directory), at which point we can +# generate tasks for every docker image in the directory, secure in the +# knowledge that unnecessary images will be omitted from the target task graph +jobs: + image_builder: + symbol: I(ib) + # Neither the ubuntu1804-*raw nor the ubuntu1804-*packages images can have + # packages dependencies. + ubuntu1804-raw: + symbol: I(ub18-raw) + definition: debian-raw + args: + BASE_IMAGE: ubuntu:bionic + ubuntu1804-packages: + symbol: I(ub18-pkg) + definition: debian-packages + parent: ubuntu1804-raw + ubuntu1804-i386-raw: + symbol: I(ub18-32-raw) + definition: debian-raw + args: + BASE_IMAGE: i386/ubuntu:bionic + ubuntu1804-i386-packages: + symbol: I(ub18-32-pkg) + definition: debian-packages + parent: ubuntu1804-i386-raw + ubuntu1804-base: + symbol: I(ub18-base) + parent: ubuntu1804-raw + packages: + - ub18-mercurial + - ub18-python-zstandard + - ub18-python-psutil + ubuntu1804-test-base: + symbol: I(ub18-test-base) + parent: ubuntu1804-base + packages: + - ub18-libc6 + - ub18-pygobject + - ub18-pyatspi + - ub18-32-libc6 + ubuntu1804-test: + symbol: I(ub18-test) + parent: ubuntu1804-test-base + ubuntu1804-build-python: + symbol: I(ub18-build-python) + parent: ubuntu1804-base + # Neither the ubuntu1804-*raw nor the ubuntu1804-*packages images can have + # packages dependencies. + ubuntu2004-raw: + symbol: I(ub20-raw) + definition: debian-raw + args: + BASE_IMAGE: ubuntu:focal + ubuntu2204-raw: + symbol: I(ub22-raw) + definition: debian-raw + args: + BASE_IMAGE: ubuntu:jammy + ubuntu2004-packages: + symbol: I(ub20-pkg) + definition: debian-packages + parent: ubuntu2004-raw + ubuntu2204-packages: + symbol: I(ub22-pkg) + definition: debian-packages + parent: ubuntu2204-raw + ubuntu2004-base: + symbol: I(ub20-base) + definition: debian-base + parent: ubuntu2004-raw + packages: + - ub20-mercurial + - ub20-python-zstandard + ubuntu2204-base: + symbol: I(ub22-base) + definition: debian-base + parent: ubuntu2204-raw + packages: + - ub22-mercurial + - ub22-python-zstandard + # Neither the debian8-*raw nor the debian8-*packages images can have + # packages dependencies. + debian8-raw: + symbol: I(deb8-raw) + definition: debian-raw + args: + BASE_IMAGE: debian:jessie-20210208 + DIST: jessie + debian8-packages: + symbol: I(deb8-pkg) + definition: debian-packages + parent: debian8-raw + debian8-i386-raw: + symbol: I(deb8-32-raw) + definition: debian-raw + args: + BASE_IMAGE: i386/debian:jessie-20210208 + DIST: jessie + debian8-i386-packages: + symbol: I(deb8-32-pkg) + definition: debian-packages + parent: debian8-i386-raw + custom-v8: + symbol: I(custom-v8) + parent: debian12-base + # Chromium-as-Release + custom-car-linux: + symbol: I(custom-car-linux) + parent: debian12-base + # Chromium-as-Release Android + custom-car-android: + symbol: I(custom-car-android) + parent: debian12-base + valgrind-build: + symbol: I(vb) + parent: debian12-amd64-build + packages: + - deb12-valgrind + lint: + symbol: I(lnt) + parent: debian12-base + # Neither the debian12-raw nor the debian12-packages images can have + # packages dependencies. + debian12-raw: + symbol: I(deb12-raw) + definition: debian-raw + args: + BASE_IMAGE: debian:bookworm-20230612 + DIST: bookworm + SNAPSHOT: '20230611T210420Z' + debian12-packages: + symbol: I(deb12-pkg) + definition: debian-packages + parent: debian12-raw + debian12-base: + symbol: I(deb12-base) + definition: debian-base + parent: debian12-raw + packages: + - deb12-mercurial + - deb12-python-zstandard + debian12-amd64-build: + symbol: I(deb12) + parent: debian12-base + definition: debian-build + debian12-repackage: + symbol: I(deb12-rpk) + parent: debian12-base + definition: debian-repackage + packages: + - deb12-mercurial + deb12-toolchain-build: + symbol: I(deb12-toolchain) + parent: debian12-base + definition: toolchain-build + android-build: + symbol: I(agb) + parent: debian12-base + fetch: + symbol: I(fetch) + parent: debian12-raw + packages: + - deb12-python-zstandard + static-analysis-build: + symbol: I(static-analysis-build) + parent: debian12-base + gdb-test: + symbol: I(gdb) + parent: debian12-amd64-build + index-task: + symbol: I(idx) + funsize-update-generator: + symbol: I(pg) + update-verify: + symbol: I(uv) + parent: ubuntu2004-base + diffoscope: + symbol: I(diff) + parent: debian12-base + partner-repack: + symbol: I(PR) + parent: debian12-base + definition: partner-repack + updatebot: + symbol: I(3rdp) + parent: push-to-try + periodic-updates: + symbol: I(file) + parent: debian12-base + firefox-snap: + symbol: I(snap) + firefox-flatpak: + symbol: I(flatpak) + webrender: + symbol: I(webrender) + parent: debian12-base + condprof: + symbol: I(condprof) + parent: ubuntu1804-test + github-sync: + symbol: I(github-sync) + parent: debian12-base + sentry: + symbol: I(sentry) + parent: debian12-base + system-symbols-mac: + symbol: I(system-symbols-mac) + parent: debian12-base + system-symbols-win: + symbol: I(system-symbols-win) + parent: debian12-base + system-symbols-win-gfx: + symbol: I(system-symbols-win-gfx) + parent: debian12-base + system-symbols-linux-scraper: + symbol: I(system-symbols-linux) + parent: debian12-base + push-to-try: + symbol: I(push-to-try) + parent: debian12-base + decision: + symbol: I(decision) + parent: ubuntu2004-base + snap-build-core22: + symbol: I(snap22-build) + parent: ubuntu2204-base + definition: snap-coreXX-build + args: + SNAP_BASE: core22 + SNAP_LIST: "gnome-42-2204 gtk-common-themes gnome-42-2204-sdk" diff --git a/taskcluster/ci/fetch/benchmarks.yml b/taskcluster/ci/fetch/benchmarks.yml new file mode 100644 index 0000000000..7e31470baf --- /dev/null +++ b/taskcluster/ci/fetch/benchmarks.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +octane: + description: Octane benchmark + fetch: + type: static-url + url: https://github.com/mozilla/perf-automation/releases/download/octane-v1/octane-0f5b8d48d4a9.zip + sha256: 38425ee1abfc5feca178b2f60fbd82b5873897c345112a85359be00024402f9f + size: 1816138 + +web-tooling-benchmark: + description: Web Tooling Benchmark + fetch: + type: static-url + url: https://github.com/mozilla/perf-automation/releases/download/V1/web-tooling-benchmark-b2ac25c897c9.zip + sha256: 93b0b51df0cec3ca9bfa0bdf81d782306dcf18532e39b3ff3180409125daaff1 + size: 5444135 diff --git a/taskcluster/ci/fetch/browsertime.yml b/taskcluster/ci/fetch/browsertime.yml new file mode 100644 index 0000000000..614ddce593 --- /dev/null +++ b/taskcluster/ci/fetch/browsertime.yml @@ -0,0 +1,180 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +win64-ffmpeg-4.4.1: + description: 'Win64 ffmpeg' + fetch: + type: static-url + # url: https://github.com/GyanD/codexffmpeg/releases/tag/4.4.1 + url: https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip + sha256: f0f407041f82905c310b79f4f76973815700d1e438a10c3cbce66a345ff4f457 + size: 134760541 + +linux64-ffmpeg-4.4.1: + description: 'Linux64 ffmpeg' + fetch: + type: static-url + # url: https://www.johnvansickle.com/ffmpeg/old-releases/ + url: https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz + sha256: 1f38d1c99b6eb3d36f709a481f5a5526448567a800ee9cddcfe4be54f8b68e62 + size: 17592088 + +mac64-ffmpeg-4.4.1: + description: 'Mac64 ffmpeg' + fetch: + type: static-url + # url: https://evermeet.cx/pub/ffmpeg/ + url: https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip + sha256: 9d07411bce176f56314cc5624f5df51f9cffa76a59e3e81abe2175d4ebaf87ae + size: 49100208 + +linux64-chromedriver: + description: 'Linux64 chromedriver' + fetch: + type: static-url + url: https://chromedriver.storage.googleapis.com/2.46/chromedriver_linux64.zip + sha256: 461919e080e19335a34224e2d353b96b07c7d068621aa940f9c136e86d090047 + size: 5404417 + +win32-chromedriver-120: + description: "Win32 chromedriver v120" + fetch: + type: static-url + artifact-name: chromedriver_win32_120.tar.zst + add-prefix: "120" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/120.0.6099.18/win32/chromedriver-win32.zip + sha256: f07b320a3e739c619b9bda0c62169d6173350e7415317125fba76c8b002458ce + size: 7360109 + +linux64-chromedriver-120: + description: "Linux64 chromedriver v120" + fetch: + type: static-url + artifact-name: chromedriver_linux64_120.tar.zst + add-prefix: "120" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/120.0.6099.18/linux64/chromedriver-linux64.zip + sha256: 3990b819b9b8866784dcd634e284e6e02f2b92f81ba89fb7b6c98061ad2b3c6e + size: 8575905 + +mac64-chromedriver-120: + description: "Mac64 chromedriver v120" + fetch: + type: static-url + artifact-name: chromedriver_mac64_120.tar.zst + add-prefix: "120" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/120.0.6099.18/mac-x64/chromedriver-mac-x64.zip + sha256: f645302e046e4314477b72244335c60098563d202fc0c9467df7d71b850c21f1 + size: 9235668 + +mac-arm-chromedriver-120: + description: "Mac64 arm chromedriver v120" + fetch: + type: static-url + artifact-name: chromedriver_mac-arm_120.tar.zst + add-prefix: "120" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/120.0.6099.18/mac-arm64/chromedriver-mac-arm64.zip + sha256: 141e0abedfe2a1df6c3eb334becdf1993dce8b4fc0f906457606470c6542f236 + size: 8293703 + +win32-chromedriver-121: + description: "Win32 chromedriver v121" + fetch: + type: static-url + artifact-name: chromedriver_win32_121.tar.zst + add-prefix: "121" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6167.57/win32/chromedriver-win32.zip + sha256: a6618777d23e2d65a4938c150df1c7516be6d8e52748f635c7bf1df36e0a143f + size: 7989797 + + +linux64-chromedriver-121: + description: "Linux64 chromedriver v121" + fetch: + type: static-url + artifact-name: chromedriver_linux64_121.tar.zst + add-prefix: "121" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6167.57/linux64/chromedriver-linux64.zip + sha256: f4a3fe88694cd67484041d01dfc33e9769b9709335e97e3283f11da0e95a73f8 + size: 8625832 + + +mac64-chromedriver-121: + description: "Mac64 chromedriver v121" + fetch: + type: static-url + artifact-name: chromedriver_mac64_121.tar.zst + add-prefix: "121" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6167.57/mac-x64/chromedriver-mac-x64.zip + sha256: c89fa84060d18ed03f3f68de67400051691836b85e8b7d77b386d6239f9b8ced + size: 9233941 + + +mac-arm-chromedriver-121: + description: "Mac-arm chromedriver v121" + fetch: + type: static-url + artifact-name: chromedriver_mac-arm_121.tar.zst + add-prefix: "121" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6167.57/mac-arm64/chromedriver-mac-arm64.zip + sha256: e162f11cebcb7d255daf3f7af2e8f6046389f2b5264c0a991022957af0524afc + size: 8318525 + +win32-chromedriver-122: + description: "Win32 chromedriver v122" + fetch: + type: static-url + artifact-name: chromedriver_win32_122.tar.zst + add-prefix: "122" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/win32/chrome-win32.zip + sha256: 4778e327a30972fa1d39a9aabc60c04e82553b7cafa2f8f768eaec9ee0935eab + size: 134566218 + +linux64-chromedriver-122: + description: "Linux64 chromedriver v122" + fetch: + type: static-url + artifact-name: chromedriver_linux64_122.tar.zst + add-prefix: "122" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/linux64/chrome-linux64.zip + sha256: 5514382fc80e092f76a6a7ea85bdfcafacd7e62e1c831a7842f0eca003383c5d + size: 149627735 + +mac64-chromedriver-122: + description: "Mac64 chromedriver v122" + fetch: + type: static-url + artifact-name: chromedriver_mac64_122.tar.zst + add-prefix: "122" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/mac-x64/chrome-mac-x64.zip + sha256: 753d2814bb82d723289eea76628e9b043bc36666d7cf285876dc4f19ca811c04 + size: 155446914 + +mac-arm-chromedriver-122: + description: "Mac64 arm chromedriver v122" + fetch: + type: static-url + artifact-name: chromedriver_mac-arm_122.tar.zst + add-prefix: "122" + url: https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/122.0.6261.18/mac-arm64/chrome-mac-arm64.zip + sha256: 1edd4096771669e6c429eff5a459f520defd43afb798fc86d58608d95355be6e + size: 146946870 + +upload-speed-test-file: + description: "File used for the upload speed performance test" + fetch: + type: static-url + artifact-name: upload-test-32MB.dat + url: https://github.com/mozilla/perf-automation/raw/master/test_files/upload-test-32MB.dat + sha256: 72caa47a951e9f37d1e1be2119a64b1fbc6348c4f37d574bee4587e02b8f96a5 + size: 32000000 + +firefox-addons: + description: "Download a tar archive of real Firefox extensions xpi files available on AMO (used to run perftests with real extensions installed)" + fetch: + type: static-url + artifact-name: firefox-addons.tar.zst + add-prefix: firefox-addons/ + url: https://github.com/mozilla/perf-automation/raw/c9e497eaa86f45538e3509120947215c6f97c95b/test_files/firefox-addons.tar + sha256: 20372ff1d58fc33d1568f8922fe66e2e2e01c77663820344d2a364a8ddd68281 + size: 3584000 diff --git a/taskcluster/ci/fetch/chromium-fetch.yml b/taskcluster/ci/fetch/chromium-fetch.yml new file mode 100644 index 0000000000..88f7e4fc62 --- /dev/null +++ b/taskcluster/ci/fetch/chromium-fetch.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + fetch: + type: chromium-fetch + script: /builds/worker/bin/fetch-chromium.py + +linux64-chromium: + description: 'Linux64 Chromium Fetch' + fetch: + platform: linux + artifact-name: chromium-linux.tar.bz2 + +win32-chromium: + description: 'Windows32 Chromium Fetch' + fetch: + platform: win32 + artifact-name: chromium-win32.tar.bz2 + +win64-chromium: + description: 'Windows64 Chromium Fetch' + fetch: + platform: win64 + artifact-name: chromium-win64.tar.bz2 + +mac-chromium: + description: 'MacOSX Chromium Fetch' + fetch: + platform: mac + artifact-name: chromium-mac.tar.bz2 + +mac-chromium-arm: + description: 'MacOSX ARM Chromium Fetch' + fetch: + platform: mac-arm + artifact-name: chromium-mac-arm.tar.bz2 diff --git a/taskcluster/ci/fetch/kind.yml b/taskcluster/ci/fetch/kind.yml new file mode 100644 index 0000000000..027ab7f1eb --- /dev/null +++ b/taskcluster/ci/fetch/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.fetch:transforms + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - benchmarks.yml + - browsertime.yml + - chromium-fetch.yml + - resource-monitor.yml + - toolchain-clang-tidy-external.yml + - toolchains.yml + - updatebot.yml diff --git a/taskcluster/ci/fetch/moz.build b/taskcluster/ci/fetch/moz.build new file mode 100644 index 0000000000..28b8003edf --- /dev/null +++ b/taskcluster/ci/fetch/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("updatebot.yml"): + BUG_COMPONENT = ("Developer Infrastructure", "Mach Vendor & Updatebot") diff --git a/taskcluster/ci/fetch/resource-monitor.yml b/taskcluster/ci/fetch/resource-monitor.yml new file mode 100644 index 0000000000..2fe94cb4f9 --- /dev/null +++ b/taskcluster/ci/fetch/resource-monitor.yml @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +resource-monitor: + description: Resource monitoring tools + fetch: + type: git + repo: https://github.com/mozilla-releng/resource-monitor + revision: 17371502a3b04579375d707844e6bf08dee95d22 diff --git a/taskcluster/ci/fetch/toolchain-clang-tidy-external.yml b/taskcluster/ci/fetch/toolchain-clang-tidy-external.yml new file mode 100644 index 0000000000..d71058c06b --- /dev/null +++ b/taskcluster/ci/fetch/toolchain-clang-tidy-external.yml @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +civet-source: + description: Repository for external static analysis checks + artifact-prefix: project/civet + fetch: + type: git + repo: git@github.com:mozilla-services/civet.git + revision: fa787d2047a5c3759e67f0df58415a984dcc8777 + ssh-key: project/civet/github-deploy-key + artifact-name: civet-source.tar.zst diff --git a/taskcluster/ci/fetch/toolchains.yml b/taskcluster/ci/fetch/toolchains.yml new file mode 100644 index 0000000000..50ffb065b0 --- /dev/null +++ b/taskcluster/ci/fetch/toolchains.yml @@ -0,0 +1,714 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +binutils-2.31.1: + description: binutils 2.31.1 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/binutils/binutils-2.31.1.tar.xz + sha256: 5d20086ecf5752cc7d9134246e9588fa201740d540f7eb84d795b1f7a93bca86 + size: 20467996 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-binutils/3A24BC1E8FB409FA9F14371813FCEF89DD9E3C4F.key + artifact-name: binutils-source.tar.zst + strip-components: 1 + add-prefix: binutils-source/ + +binutils-2.41: + description: binutils 2.41 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz + sha256: ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450 + size: 26765692 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-binutils/3A24BC1E8FB409FA9F14371813FCEF89DD9E3C4F.key + artifact-name: binutils-source.tar.zst + strip-components: 1 + add-prefix: binutils-source/ + +gcc-8.5.0: + description: GCC 8.5.0 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/gcc/gcc-8.5.0/gcc-8.5.0.tar.xz + sha256: d308841a511bb830a6100397b0042db24ce11f642dab6ea6ee44842e5325ed50 + size: 63841008 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-gcc/D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62.key + artifact-name: gcc-source.tar.zst + strip-components: 1 + add-prefix: gcc-source/ + +gcc-9.5.0: + description: GCC 9.5.0 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/gcc/gcc-9.5.0/gcc-9.5.0.tar.xz + sha256: 27769f64ef1d4cd5e2be8682c0c93f9887983e6cfd1a927ce5a0a2915a95cf8f + size: 72462752 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-gcc/7F74F97C103468EE5D750B583AB00996FC26A641.key + artifact-name: gcc-source.tar.zst + strip-components: 1 + add-prefix: gcc-source/ + +gcc-11.4.0: + description: GCC 11.4.0 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/gcc/gcc-11.4.0/gcc-11.4.0.tar.xz + sha256: 3f2db222b007e8a4a23cd5ba56726ef08e8b1f1eb2055ee72c1402cea73a8dd9 + size: 79837032 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-gcc/D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62.key + artifact-name: gcc-source.tar.zst + strip-components: 1 + add-prefix: gcc-source/ + +gmp-6.1.0: + description: GMP 6.1.0 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/gmp/gmp-6.1.0.tar.bz2 + sha256: 498449a994efeba527885c10405993427995d3f86b8768d8cdf8d9dd7c6b73e8 + size: 2383840 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-gcc/343C2FF0FBEE5EC2EDBEF399F3599FF828C67298.key + artifact-name: gmp-source.tar.zst + strip-components: 1 + add-prefix: gmp-source/ + +go-1.14.4: + description: Golang 1.14.4 build tools + fetch: + type: static-url + url: https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz + sha256: aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067 + size: 123711003 + +isl-0.16.1: + description: ISL 0.16.1 source code + fetch: + type: static-url + url: https://gcc.gnu.org/pub/gcc/infrastructure/isl-0.16.1.tar.bz2 + sha256: 412538bb65c799ac98e17e8cfcdacbb257a57362acfaaff254b0fcae970126d2 + size: 1626446 + artifact-name: isl-source.tar.zst + strip-components: 1 + add-prefix: isl-source/ + +mpc-1.0.3: + description: mpc 1.0.3 source code + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/mpc/mpc-1.0.3.tar.gz + sha256: 617decc6ea09889fb08ede330917a00b16809b8db88c29c31bfbb49cbf88ecc3 + size: 669925 + gpg-signature: + sig-url: "{url}.sig" + key-path: build/unix/build-gcc/AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3.key + artifact-name: mpc-source.tar.zst + strip-components: 1 + add-prefix: mpc-source/ + +mpfr-3.1.4: + description: mpfr 3.1.4 source code + fetch: + type: static-url + url: http://www.mpfr.org/mpfr-3.1.4/mpfr-3.1.4.tar.bz2 + sha256: d3103a80cdad2407ed581f3618c4bed04e0c92d1cf771a65ead662cc397f7775 + size: 1279284 + gpg-signature: + sig-url: "{url}.asc" + key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key + artifact-name: mpfr-source.tar.zst + strip-components: 1 + add-prefix: mpfr-source/ + +nasm-2.14.02: + description: nasm 2.14.02 source code + fetch: + type: static-url + url: https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.xz + sha256: e24ade3e928f7253aa8c14aa44726d1edf3f98643f87c9d72ec1df44b26be8f5 + size: 827620 + +nasm-2.15.05: + description: nasm 2.15.05 source code + fetch: + type: static-url + url: https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.xz + sha256: 3caf6729c1073bf96629b57cee31eeb54f4f8129b01902c73428836550b30a3f + size: 995732 + +wix-3.14.0: + description: wix 3.14.0 binary release + # WIX doesn't use a wrapping toplevel DIR, so we should really repackage it when + # Bug 1479533 and related code is ready. + fetch: + type: static-url + url: https://github.com/wixtoolset/wix3/releases/download/wix314rtm/wix314-binaries.zip + sha256: 13f067f38969faf163d93a804b48ea0576790a202c8f10291f2000f0e356e934 + size: 41282726 + +zlib-1.3.1: + description: zlib 1.3.1 source code + fetch: + type: static-url + url: https://zlib.net/fossils/zlib-1.3.1.tar.gz + sha256: 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 + size: 1512791 + +nsis-3.07: + description: nsis 3.07 source code + fetch: + type: static-url + url: https://downloads.sourceforge.net/project/nsis/NSIS%203/3.07/nsis-3.07-src.tar.bz2 + sha256: 4dfad3388589985b4cd91d20e18e1458aa31e7d139b5b8adf25c3a9c1015efba + size: 1784292 + +nsis-3.07-win: + description: nsis 3.07 for windows + fetch: + type: static-url + # Temporary: pull down our own repackage of NSIS, with its plugin DLLs signed by Mozilla + # See https://bugzilla.mozilla.org/show_bug.cgi?id=1751450#c38 for more details + # bug 1771192 tracks adding a signing task downstream of the NSIS toolchain task + # to fix this more robustly. + url: https://github.com/mozilla-releng/adhoc-signing-blobs/raw/nsis/nsis-3.07-signed.zip + sha256: 5856cd49b52870caa41814d71ce30c4304d5001b80c04f4eb7028ffedd91df7e + size: 2618390 + artifact-name: nsis.tar.zst + strip-components: 1 + add-prefix: nsis/ + +upx-3.95-win: + description: upx 3.95 for windows + fetch: + type: static-url + url: https://hg.mozilla.org/mozilla-build/raw-file/3b8c537ca3c879551956ad47ca9f089583f647c5/upx-3.95-win64.zip + sha256: 5c076f87ba64d82f11513f4af0ceb07246a3540aa3c72ca3ffc2d53971fa56e3 + size: 462820 + +wine: + description: wine source code + fetch: + type: static-url + url: https://dl.winehq.org/wine/source/8.0/wine-8.0.2.tar.xz + sha256: 6ec8fb6f2c72d576cb11f52b2f8d59af64404802154651d122b98466d91dc847 + size: 29060452 + gpg-signature: + sig-url: "{url}.sign" + key-path: build/unix/build-gcc/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key + artifact-name: wine.tar.zst + strip-components: 1 + add-prefix: wine-source/ + +cbindgen-0.26.0: + description: cbindgen source code + fetch: + type: git + repo: https://github.com/mozilla/cbindgen + revision: 703b53c06f9fe2dbc0193d67626558cfa84a0f62 + +cctools-port: + description: cctools-port source code + fetch: + type: git + repo: https://github.com/tpoechtrager/cctools-port + revision: 30518813875aed656aa7f18b6d485feee25f8f87 + +libtapi: + description: libtapi source code + fetch: + type: git + repo: https://github.com/tpoechtrager/apple-libtapi + revision: b4f141a16c7c82f75973429c92242a8c7de3ddb1 + +ldid: + description: ldid source code + fetch: + type: git + repo: https://github.com/tpoechtrager/ldid + revision: 4bf8f4d60384a0693dbbe2084ce62a35bfeb87ab + +libdmg-hfsplus: + description: libdmg-hfsplug source code + fetch: + type: git + repo: https://github.com/mozilla/libdmg-hfsplus/ + revision: a0a959bd25370c1c0a00c9ec525e3e78285adbf9 + +hfsplus-tools: + description: hfstools-plus source code + fetch: + type: static-url + # Most-upstream is https://opensource.apple.com/source/diskdev_cmds/ + url: https://src.fedoraproject.org/repo/pkgs/hfsplus-tools/diskdev_cmds-540.1.linux3.tar.gz/0435afc389b919027b69616ad1b05709/diskdev_cmds-540.1.linux3.tar.gz + sha256: b01b203a97f9a3bf36a027c13ddfc59292730552e62722d690d33bd5c24f5497 + size: 411205 + +xar: + description: xar source code + fetch: + type: git + repo: https://github.com/gabrielesvelto/xar + revision: 199ff76a7463f6b6a2548b4db29cd3dd5026df69 + +bomutils: + description: bomutils (mkbom) source code + fetch: + type: git + repo: https://github.com/hogliux/bomutils + revision: 0e7dc7eefefb3a0ba2ef3a432cdc1089ac40740c + +rust-size: + description: rust-size source code + fetch: + type: git + repo: https://github.com/luser/rust-size + revision: ab659d93b1faba95307df952aefe3fbed3583669 + +gn: + description: GN source code + fetch: + type: git + repo: https://gn.googlesource.com/gn + revision: 7e05276308f3d49f8514ef02afe4f94487b1efb2 + +grcov: + description: grcov source code + fetch: + type: git + repo: https://github.com/mozilla/grcov + revision: 322fc39acacd75aca0ff1c0a1ec2a3e91f04011c + +sccache: + description: sccache source code + fetch: + type: git + repo: https://github.com/mozilla/sccache + # 0.7.6 release + revision: 794c987b7843f404b9e50513dae978f807ba0a60 + +fxc2: + description: fxc2 source code + fetch: + type: git + repo: https://github.com/mozilla/fxc2 + revision: 63ad74b7faa7033f2c1be9cc1cd0225241a1a9a5 + +mingw-w64: + description: mingw-w64 source code + fetch: + type: git + repo: https://git.code.sf.net/p/mingw-w64/mingw-w64 + revision: 95a2cf4b7260200c4a60f4f1d1127e72799a3541 + +mingw-w64-trunk: + description: mingw-w64 master branch source code + attributes: + cached_task: false + fetch: + type: git + repo: https://git.code.sf.net/p/mingw-w64/mingw-w64 + branch: master + +llvm-mingw: + description: llvm-mingw source code + fetch: + type: git + repo: https://github.com/mstorsjo/llvm-mingw + revision: 9f8e5cebd6dbbb7546e8917e6c59284699a48d26 + +android-ndk-rs: + description: android-ndk-rs source code + fetch: + type: git + # Until there's a version of cargo-apk published on crates.io that has + # https://github.com/rust-windowing/android-ndk-rs/pull/236, we need to use + # an unpublished version. Additionally, until we update the NDK version used + # in gecko we must use our own patched version. See bug 1615148. + repo: https://github.com/jamienicol/android-ndk-rs + revision: 595f4e14a78371e3ab59b12f7bd8131f2ec5b4a1 + +clang-8.0: + description: clang 8.0.0 source code + fetch: + type: git + repo: https://github.com/llvm/llvm-project + revision: d2298e74235598f15594fe2c99bbac870a507c59 + +clang-14: + description: clang 14.0.5 source code + fetch: + type: git + repo: https://github.com/llvm/llvm-project + revision: 4bc1d0b51c8e488d78ab69c8b19cfbcd1f7db6a4 + +clang-17: + description: clang 17.0.6 source code + fetch: + type: git + repo: https://github.com/llvm/llvm-project + revision: 6009708b4367171ccdbf4b5905cb6a803753fe18 + +clang-18: + description: clang 18.1.0-rc2 source code + fetch: + type: git + repo: https://github.com/llvm/llvm-project + revision: c6c86965d967886ca5a2ed8968c949cd42cdd49a + +clang-trunk: + description: clang main branch source code + attributes: + cached_task: false + fetch: + type: git + repo: https://github.com/llvm/llvm-project + branch: main + +# We build stable rust from source so the resulting compiler acts as a nightly +# rust compiler, allowing to use unstable features like -Zbuild-std and +# sanitizers. +rust-1.76.0: + description: Rust 1.76.0 source code + fetch: + type: git + include-dot-git: true + repo: https://github.com/rust-lang/rust/ + revision: 07dca489ac2d933c78d3c5158e3f43beefeb02ce + +wasi-sdk-11: + description: wasi-sdk-11 source code + fetch: + type: git + repo: https://github.com/WebAssembly/wasi-sdk + revision: ceabbfe181599bca83d81e087a229797e472c09c + +wasi-sdk: + description: wasi-sdk-20 source code + fetch: + type: git + repo: https://github.com/WebAssembly/wasi-sdk + revision: 935fe1acd2fcd7ea4aed2d5ee4527482862b6344 + +ninja: + description: ninja 1.9.0 + fetch: + type: static-url + url: https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip + sha256: 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9 + size: 254497 + artifact-name: ninja.tar.zst + add-prefix: ninja/bin/ + +cmake: + description: cmake 3.25.1 + fetch: + type: static-url + url: https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-windows-x86_64.zip + sha256: d93958d87cc9b91983489f0b37a268b03a3c891894d11f5437fa2a5ce94aab24 + size: 41460154 + artifact-name: cmake.tar.zst + strip-components: 1 + add-prefix: cmake/ + +dump-syms: + description: dump_syms source code + fetch: + type: git + repo: https://github.com/mozilla/dump_syms/ + revision: b1951e4ccec48a2e97460a7ec40be35aa79e0037 + +rust-minidump: + description: rust-minidump 0.19.1 source code (for minidump-stackwalk) + fetch: + type: git + repo: https://github.com/rust-minidump/rust-minidump/ + revision: c3de84b061339c686a572fb9f059e7ba3fad38d6 + +fix-stacks: + description: fix-stacks source code + fetch: + type: git + repo: https://github.com/mozilla/fix-stacks/ + revision: 1fb1bcc8cb4dd65fbe342853954651563a0a88e7 + +winchecksec: + description: winchecksec source code + fetch: + type: git + repo: https://github.com/trailofbits/winchecksec/ + revision: 93ffe67dbfc757bf6f440d80b8acf88e652ed60a + +sonatype-nexus: + description: Sonatype Nexus Maven repository service + fetch: + type: static-url + url: https://download.sonatype.com/nexus/oss/nexus-2.14.18-01-bundle.tar.gz + sha256: 350af8b270bec7fda5861a15b8a5b7870fd8903d0b9a210754113d5a966ce6a6 + size: 83493896 + artifact-name: sonatype-nexus.tar.zst + strip-components: 1 + add-prefix: sonatype-nexus/ + +jdk-8-linux64: + description: Java 8 JDK + fetch: + type: static-url + url: https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u312-b07/OpenJDK8U-jdk_x64_linux_hotspot_8u312b07.tar.gz + sha256: 699981083983b60a7eeb511ad640fae3ae4b879de5a3980fe837e8ade9c34a08 + size: 103016010 + artifact-name: jdk-8.tar.zst + strip-components: 1 + add-prefix: jdk-8/ + +makecab: + description: makecab source code + fetch: + type: git + repo: https://github.com/luser/rust-makecab/ + revision: 236cd02baefa754fc7a61b161c71d2aa13a87c2a + +nodejs-12-linux64: + description: nodejs 12 for x64 linux + fetch: + type: static-url + url: https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.xz + sha256: e6d052364bfa2c17da92cf31794100cfd709ba147415ddaeed2222eec9ca1469 + size: 14584388 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-16-linux64: + description: nodejs 16 for x64 linux + fetch: + type: static-url + url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz + sha256: c88b52497ab38a3ddf526e5b46a41270320409109c3f74171b241132984fd08f + size: 22492000 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-16-macosx64: + description: nodejs 16 for x64 macOS + fetch: + type: static-url + url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-darwin-x64.tar.xz + sha256: 601d1d632b9815dbb647f5cd440a95fa651110b5b4bcaf07ed3aa7886525716b + size: 20975524 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-18-macosx64: + description: nodejs 18 for x64 macOS + fetch: + type: static-url + url: https://nodejs.org/dist/v18.19.0/node-v18.19.0-darwin-x64.tar.xz + sha256: 4f60b40071c6a67482b99d22a5265839177b8706a2b13b493396f368f0b42b00 + size: 22496376 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-16-macosx64-arm64: + description: nodejs 16 for arm64 macOS + fetch: + type: static-url + url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-darwin-arm64.tar.xz + sha256: 7313c9db5f5140ece847a97f366032a10bddb6d87fa28a361ada918addcd5c73 + size: 19363804 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-18-macosx64-arm64: + description: nodejs 18 for arm64 macOS + fetch: + type: static-url + url: https://nodejs.org/dist/v18.19.0/node-v18.19.0-darwin-arm64.tar.xz + sha256: 857885257c586fb13c93d33d326ca82056ba8b7445ce27cf34dd406add04158e + size: 20768464 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-16-win32: + description: nodejs 16 for x86 Windows + fetch: + type: static-url + url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-win-x86.zip + sha256: 17e2bf8ed00b3d15321743104104c8b6ef55677dfc18ee2ed44de64fbf2c4def + size: 25424808 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-18-win32: + description: nodejs 18 for x86 Windows + fetch: + type: static-url + url: https://nodejs.org/dist/v18.19.0/node-v18.19.0-win-x86.zip + sha256: 3171148f735954bdca8adccb2200379e61142e0f5a2bdb4ca6432afb0d9951df + size: 27504941 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-16-win64: + description: nodejs 16 for x64 Windows + fetch: + type: static-url + url: https://nodejs.org/dist/v16.19.0/node-v16.19.0-win-x64.zip + sha256: 534ca7a24e999c81cec847a498cc43d47e2bb158f6edf639e5297f2718350e96 + size: 27072850 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-18-win64: + description: nodejs 18 for x64 Windows + fetch: + type: static-url + url: https://nodejs.org/dist/v18.19.0/node-v18.19.0-win-x64.zip + sha256: 5311913d45e1fcc3643c58d1e3926eb85437b180f025fe5857413c9f02403645 + size: 29332158 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +nodejs-18-source: + description: nodejs 18 source + fetch: + type: static-url + url: https://nodejs.org/dist/v18.19.0/node-v18.19.0.tar.xz + sha256: f52b41af20596a9abd8ed75241837ec43945468221448bbf841361e2091819b6 + size: 41248748 + artifact-name: nodejs.tar.zst + strip-components: 1 + add-prefix: node/ + +gnumake: + description: GNU make source + fetch: + type: static-url + url: https://ftp.gnu.org/gnu/make/make-4.3.tar.gz + sha256: e05fdde47c5f7ca45cb697e973894ff4f5d79e13b750ed57d7b66d8defc78e19 + size: 2317073 + artifact-name: make.tar.zst + strip-components: 1 + add-prefix: make/ + +msix-packaging: + description: Microsoft msix-packaging + fetch: + type: git + repo: https://github.com/mozilla/msix-packaging + revision: 8164b7ac5a67688e18fa2497fb31ba13ce9bfb15 + +pkgconf: + description: pkgconf 1.8.0 + fetch: + type: static-url + url: https://distfiles.dereferenced.org/pkgconf/pkgconf-1.8.0.tar.xz + sha256: ef9c7e61822b7cb8356e6e9e1dca58d9556f3200d78acab35e4347e9d4c2bbaf + size: 296304 + artifact-name: pkgconf.tar.zst + strip-components: 1 + add-prefix: pkgconf/ + +afl-plus-plus-4.0: + description: AFL++ 4.0x source + fetch: + type: git + repo: https://github.com/AFLplusplus/AFLplusplus + revision: 0d164e4c1811c4d05f940f78e90fc56b661fb3b4 + +cargo-vet: + description: cargo-vet source + fetch: + type: git + repo: https://github.com/mozilla/cargo-vet + revision: 5bd670061a22074eb5ee9d69feccc5900df00ec3 + +symbol-scrapers: + description: Linux distribution symbol scraper source + fetch: + type: git + repo: https://github.com/mozilla/symbol-scrapers + revision: 95ea2e4c5f82469cb394620c8d332e3457466e45 + +xz-5.4.4: + description: xz 5.4.4 source code + fetch: + type: static-url + url: https://tukaani.org/xz/xz-5.4.4.tar.gz + sha256: aae39544e254cfd27e942d35a048d592959bd7a79f9a624afb0498bb5613bdf8 + size: 2874706 + artifact-name: xz-source.tar.zst + strip-components: 1 + add-prefix: xz-source/ + +cpython-3.8.10: + description: cpython 3.8.10 source code + fetch: + type: static-url + url: https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tgz + sha256: b37ac74d2cbad2590e7cd0dd2b3826c29afe89a734090a87bf8c03c45066cb65 + size: 24720640 + artifact-name: cpython-source.tar.zst + strip-components: 1 + add-prefix: cpython-source/ + +cpython-3.8.10.exe: + description: cpython 3.8.10 installer + fetch: + type: static-url + url: https://www.python.org/ftp/python/3.8.10/python-3.8.10-amd64.exe + sha256: 7628244cb53408b50639d2c1287c659f4e29d3dfdb9084b11aed5870c0c6a48a + size: 28296784 + artifact-name: python-3.8.10-amd64.exe + +cpython-3.11.7: + description: cpython 3.11.7 source code + fetch: + type: static-url + url: https://www.python.org/ftp/python/3.11.7/Python-3.11.7.tgz + sha256: 068c05f82262e57641bd93458dfa883128858f5f4997aad7a36fd25b13b29209 + size: 26601929 + artifact-name: cpython-source.tar.zst + strip-components: 1 + add-prefix: cpython-source/ + +cpython-3.11.7.exe: + description: cpython 3.11.7 installer + fetch: + type: static-url + url: https://www.python.org/ftp/python/3.11.7/python-3.11.7-amd64.exe + sha256: c117c6444494bbe4cc937e8a5a61899d53f7f5c5bc573c5d130304e457d54024 + size: 26009544 + artifact-name: python-3.11.7-amd64.exe + +windows-rs: + description: Source of the windows rust crate + fetch: + type: static-url + url: https://crates.io/api/v1/crates/windows/0.52.0/download + sha256: e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be + size: 11843715 + artifact-name: windows-rs.tar.zst + strip-components: 1 + add-prefix: windows-rs/ diff --git a/taskcluster/ci/fetch/updatebot.yml b/taskcluster/ci/fetch/updatebot.yml new file mode 100644 index 0000000000..b757a383f8 --- /dev/null +++ b/taskcluster/ci/fetch/updatebot.yml @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +cloud-sql-proxy-1.25.0: + description: Windows executable for Google's Cloud SQL Proxy + fetch: + type: static-url + url: https://storage.googleapis.com/cloudsql-proxy/v1.25.0/cloud_sql_proxy_x64.exe + sha256: b922a8d3924499b829bfe53a25382f9b340ea804a738e2a7d8b34957a329a2dd + size: 16195072 + artifact-name: cloud_sql_proxy.exe +php-windows-7.4.24: + description: VC15 x64 Non Thread Safe (2021-Sep-21 16:58:34) + fetch: + type: static-url + url: https://windows.php.net/downloads/releases/archives/php-7.4.24-nts-Win32-vc15-x64.zip + sha256: 1e93fb390ab2761a47d27c97225a518759731c632c0dd87436cbf25f19582436 + size: 26067094 + artifact-name: php.tar.zst + add-prefix: php-win/ +arcanist: + description: checkout of arcanist for running on Windows + fetch: + type: git + repo: https://github.com/phacility/arcanist + revision: a028291f8e5e79b6446ad67ed8b0be9c5ec8f029 +depot_tools: + description: Google's depot_tools package + fetch: + type: git + repo: https://chromium.googlesource.com/chromium/tools/depot_tools.git + revision: 7b3351443198f4e7d068999b169385b55a43679a diff --git a/taskcluster/ci/fuzzing/kind.yml b/taskcluster/ci/fuzzing/kind.yml new file mode 100644 index 0000000000..68e212bb9c --- /dev/null +++ b/taskcluster/ci/fuzzing/kind.yml @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - fetch + - toolchain + - build + +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + docker-image: {in-tree: debian12-amd64-build} + use-system-python: false + treeherder: + kind: test + tier: 3 + task-context: + from-object: {} + substitution-fields: [] + if-dependencies: [build] + +jobs: + simple: + platform: linux64/opt + require-build: + linux64/opt: build-linux64/opt + description: Simple Fuzzing Test + treeherder: + symbol: simple-fuzzing + fetches: + build: + - target.jsshell.zip + - target.fuzztest.tests.tar.gz + worker: + env: + LD_LIBRARY_PATH: /builds/worker/fetches + JSSHELL: ./js + run: + using: run-task + command: >- + cd $MOZ_FETCHES_DIR && + python3 tools/fuzzing/smoke/smoke.py + + python: + platform: linux64-asan-fuzzing/opt + require-build: + linux64-asan-fuzzing/opt: build-linux64-asan-fuzzing/opt + description: Python Fuzzing Smoke Tests + treeherder: + symbol: fuzzing-python + fetches: + build: + - target.tar.bz2 + run: + using: run-task + cwd: '{checkout}' + command: > + export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox && + ./mach python-test --subsuite fuzzing diff --git a/taskcluster/ci/fxrecord/kind.yml b/taskcluster/ci/fxrecord/kind.yml new file mode 100644 index 0000000000..7ff7816e7e --- /dev/null +++ b/taskcluster/ci/fxrecord/kind.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - build + +only-for-build-platforms: + - win64-shippable/opt + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.fxrecord + - gecko_taskgraph.transforms.release_notifications + - gecko_taskgraph.transforms.task + +jobs: + fxrecord: + name: notify-fxrecord-failure + description: "Desktop Startup Visual Metrics" + from-deps: + group-by: single-with-filters + run-on-projects: + - "mozilla-central" + worker-type: performance-hardware/gecko-t-fxrecorder + worker: + implementation: generic-worker + max-run-time: 1800 + os: windows + command: ["C:\\fxrecorder\\fxrecorder.exe --config C:\\fxrecorder\\fxrecord.toml record"] + treeherder: + symbol: fxrec(1st) + tier: 2 + kind: other + notifications: + status-types: + - "on-failed" + - "on-exception" + subject: fxrecord run failed + message: fxrecord run failed + emails: + - "barret@mozilla.com" + - "perftest-alerts@mozilla.com" diff --git a/taskcluster/ci/geckodriver-mac-notarization/kind.yml b/taskcluster/ci/geckodriver-mac-notarization/kind.yml new file mode 100644 index 0000000000..a90e6faa2e --- /dev/null +++ b/taskcluster/ci/geckodriver-mac-notarization/kind.yml @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.mac_notarization + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.geckodriver_mac_notarization + - gecko_taskgraph.transforms.task + +kind-dependencies: + - geckodriver-signing + +only-for-build-platforms: + - macosx64-geckodriver/opt + - macosx64-aarch64-geckodriver/opt + +jobs: + geckodriver-mac-notarization: + from-deps: + group-by: single-with-filters + worker-type: linux-signing + treeherder: + symbol: TM(gd-n) + worker: + implementation: scriptworker-signing + max-run-time: 3600 + upstream-artifacts: + - taskType: signing + paths: + - public/build/geckodriver.tar.gz + formats: ["apple_notarization_geckodriver"] + taskId: + task-reference: diff --git a/taskcluster/ci/geckodriver-signing/kind.yml b/taskcluster/ci/geckodriver-signing/kind.yml new file mode 100644 index 0000000000..08007495e5 --- /dev/null +++ b/taskcluster/ci/geckodriver-signing/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.geckodriver_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - toolchain + +jobs: + geckodriver-signing: + from-deps: + with-attributes: + geckodriver: [true] diff --git a/taskcluster/ci/generate-profile/kind.yml b/taskcluster/ci/generate-profile/kind.yml new file mode 100644 index 0000000000..839c5aa7fe --- /dev/null +++ b/taskcluster/ci/generate-profile/kind.yml @@ -0,0 +1,324 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - instrumented-build + +transforms: + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.run_pgo_profile:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + + +job-defaults: + treeherder: + symbol: Bpgo(run) + kind: build + tier: 1 + run-on-projects: ['release'] + use-system-python: false + +jobs: + linux-shippable/opt: + description: "Linux Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux-profile + treeherder: + platform: linux32-shippable/opt + worker-type: b-linux-gcp + worker: + chain-of-trust: true + env: + NEED_XVFB: "true" + max-run-time: 1200 + docker-image: {in-tree: ubuntu1804-test} + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: run-task + cwd: '{checkout}' + command: > + ./taskcluster/scripts/misc/run-profileserver.sh + run-on-projects: ['release'] + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + + linux64-shippable/opt: + description: "Linux64 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: linux64-profile + treeherder: + platform: linux64-shippable/opt + worker-type: b-linux-gcp + worker: + chain-of-trust: true + env: + NEED_XVFB: "true" + max-run-time: 1200 + docker-image: {in-tree: ubuntu1804-test} + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: run-task + cwd: '{checkout}' + command: > + ./taskcluster/scripts/misc/run-profileserver.sh + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + + android-x86-shippable/opt: + description: "Android 5.0 x86 Profile Generation" + shipping-phase: build + shipping-product: fennec + index: + product: mobile + job-name: android-x86-profile + treeherder: + platform: android-5-0-x86/pgo + worker-type: t-linux-xlarge-pgo + worker: + chain-of-trust: true + max-run-time: 2700 + docker-image: {in-tree: ubuntu1804-test} + env: + WORKING_DIR: "/builds/worker" + MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness" + PYTHON: "/builds/worker/checkouts/gecko/mach python" + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: mozharness + job-script: taskcluster/scripts/tester/test-linux.sh + script: android_emulator_pgo.py + tooltool-downloads: internal + options: [installer-path=/builds/worker/fetches/geckoview-test_runner.apk] + config: + - android/android_common.py + - android/android-x86_64-profile-generation.py + - android/android_pgo.py + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + + android-x86_64-shippable/opt: + description: "Android 5.0 x86_64 Profile Generation" + shipping-phase: build + shipping-product: fennec + index: + product: mobile + job-name: android-x86_64-profile + treeherder: + platform: android-5-0-x86_64/pgo + worker-type: t-linux-xlarge-pgo + worker: + chain-of-trust: true + max-run-time: 2700 + docker-image: {in-tree: ubuntu1804-test} + env: + WORKING_DIR: "/builds/worker" + MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness" + PYTHON: "/builds/worker/checkouts/gecko/mach python" + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: mozharness + job-script: taskcluster/scripts/tester/test-linux.sh + script: android_emulator_pgo.py + tooltool-downloads: internal + options: [installer-path=/builds/worker/fetches/geckoview-test_runner.apk] + config: + - android/android_common.py + - android/android-x86_64-profile-generation.py + - android/android_pgo.py + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + + android-aarch64-shippable/opt: + description: "Android 5.0 Aarch64 Profile Generation" + shipping-phase: build + shipping-product: fennec + index: + product: mobile + job-name: android-aarch64-profile + treeherder: + platform: android-5-0-aarch64/pgo + worker-type: b-linux-kvm-gcp + worker: + chain-of-trust: true + max-run-time: 2700 + kvm: true # for access to /dev/kvm for hw accel in the emulator + docker-image: {in-tree: ubuntu1804-test} + env: + WORKING_DIR: "/builds/worker" + MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness" + PYTHON: "/builds/worker/checkouts/gecko/mach python" + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: mozharness + job-script: taskcluster/scripts/tester/test-linux.sh + script: android_emulator_pgo.py + tooltool-downloads: internal + options: [installer-path=/builds/worker/fetches/geckoview-test_runner.apk] + config: + - android/android_common.py + - android/android-aarch64-profile-generation.py + - android/android_pgo.py + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + - android-sdk-linux + - android31-system-image-x86_64-linux + - android31-avd-x86_64-linux + + win32-shippable/opt: + description: "Win32 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: win32-profile + treeherder: + platform: windows2012-32-shippable/opt + worker-type: b-win2022 + worker: + chain-of-trust: true + max-run-time: 2400 + env: + UPLOAD_PATH: "artifacts" + artifacts: + - type: directory + name: public/build + path: build/src/artifacts/ + run: + using: run-task + sparse-profile: profile-generate + cwd: '{checkout}' + command: ./taskcluster/scripts/misc/run-profileserver.sh + fetches: + toolchain: + - win64-clang-toolchain + - win32-minidump-stackwalk + + win64-shippable/opt: + description: "Win64 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: win64-profile + treeherder: + platform: windows2012-64-shippable/opt + worker-type: b-win2022 + worker: + chain-of-trust: true + max-run-time: 2400 + env: + UPLOAD_PATH: "artifacts" + artifacts: + - type: directory + name: public/build + path: build/src/artifacts/ + run: + using: run-task + sparse-profile: profile-generate + cwd: '{checkout}' + command: ./taskcluster/scripts/misc/run-profileserver.sh + fetches: + toolchain: + - win64-clang-toolchain + - win32-minidump-stackwalk + + macosx64-shippable/opt: + description: "macosx64 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: macosx64-profile + treeherder: + platform: macosx1015-64-shippable/opt + worker-type: b-osx-1015 + worker: + chain-of-trust: true + max-run-time: 7200 + env: + SHELL: "/bin/bash" + artifacts: + - type: directory + name: public/build + path: artifacts/ + run: + using: run-task + sparse-profile: profile-generate + cwd: '{checkout}' + command: ./taskcluster/scripts/misc/run-profileserver-macos.sh + fetches: + toolchain: + - macosx64-clang + - macosx64-minidump-stackwalk + + macosx64-aarch64-shippable/opt: + description: "macosx64 aarch64 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: macosx64-aarch64-profile + treeherder: + platform: macosx1100-64-shippable/opt + worker-type: b-osx-arm64 + worker: + chain-of-trust: true + max-run-time: 7200 + env: + SHELL: "/bin/bash" + artifacts: + - type: directory + name: public/build + path: artifacts/ + run: + using: run-task + sparse-profile: profile-generate + cwd: '{checkout}' + command: ./taskcluster/scripts/misc/run-profileserver-macos.sh + fetches: + toolchain: + - macosx64-clang + - macosx64-minidump-stackwalk diff --git a/taskcluster/ci/github-sync/kind.yml b/taskcluster/ci/github-sync/kind.yml new file mode 100644 index 0000000000..7c0b8136c4 --- /dev/null +++ b/taskcluster/ci/github-sync/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - gecko_taskgraph.transforms.github_sync:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + run-on-projects: ['mozilla-central-only'] + worker-type: b-linux-gcp + treeherder: + platform: other/opt + tier: 2 + kind: other + worker: + max-run-time: 3600 + docker-image: {in-tree: github-sync} + run: + sparse-profile: github-sync + using: run-task + +jobs: + webrender: + description: Sync webrender to github mirror + secret: gecko/gfx-github-sync/token + run: + command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh webrender gfx/wr servo/webrender @bors-servo' + when: + files-changed: + - 'gfx/wr/**' + treeherder: + symbol: GhS(wr) + + qcms: + description: Sync qcms to github mirror + secret: gecko/gfx-github-sync/token + run: + command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh qcms gfx/qcms FirefoxGraphics/qcms @jrmuizel' + when: + files-changed: + - 'gfx/qcms/**' + treeherder: + symbol: GhS(qcms) diff --git a/taskcluster/ci/hazard/kind.yml b/taskcluster/ci/hazard/kind.yml new file mode 100644 index 0000000000..569141e8c8 --- /dev/null +++ b/taskcluster/ci/hazard/kind.yml @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + treeherder: + kind: build + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + docker-image: {in-tree: debian12-amd64-build} + use-system-python: false + +jobs: + linux64-shell-haz/debug: + description: "JS Shell Hazard Analysis Linux" + index: + product: firefox + job-name: shell-haz-debug + treeherder: + platform: linux64/debug + symbol: SM(H) + run: + using: hazard + command: > + cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder + && ./build-haz-linux.sh --project js $HOME/workspace + when: + files-changed: + - js/public/** + - js/src/** + fetches: + toolchain: + - linux64-clang + - linux64-gcc-9 + - linux64-gcc-sixgill + - linux64-rust + - linux64-cbindgen + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + + linux64-haz/debug: + description: "Browser Hazard Analysis Linux" + index: + product: firefox + job-name: browser-haz-debug + treeherder: + platform: linux64/debug + symbol: H + run: + using: hazard + command: > + cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder + && ./build-haz-linux.sh --project browser $HOME/workspace + fetches: + toolchain: + - linux64-clang + - linux64-gcc-9 + - linux64-gcc-sixgill + - linux64-rust + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-backstop: null diff --git a/taskcluster/ci/instrumented-build/kind.yml b/taskcluster/ci/instrumented-build/kind.yml new file mode 100644 index 0000000000..0350bc51af --- /dev/null +++ b/taskcluster/ci/instrumented-build/kind.yml @@ -0,0 +1,409 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + shipping-phase: build + treeherder: + symbol: Bpgo(instr) + tier: 1 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: instrumented + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + use-system-python: false + run: + using: mozharness + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + use-sccache: + by-release-level: + production: false + staging: true + fetches: + toolchain: + - linux64-sccache + run-on-projects: ['release'] + +jobs: + linux-shippable/opt: + description: "Linux Instrumented" + index: + product: firefox + job-name: instrumented-linux + shipping-product: firefox + treeherder: + platform: linux32-shippable/opt + worker-type: b-linux-gcp + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_32_builds.py + mozconfig-variant: profile-generate + tooltool-downloads: public + run-on-projects: ['release'] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + linux64-shippable/opt: + description: "Linux64 Instrumented" + index: + product: firefox + job-name: instrumented-linux64 + shipping-product: firefox + treeherder: + platform: linux64-shippable/opt + worker-type: b-linux-gcp + run: + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + mozconfig-variant: profile-generate + tooltool-downloads: public + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + macosx64-shippable/opt: + description: "MacOS X x64 Cross-compile Instrumented" + shipping-phase: build + shipping-product: firefox + treeherder: + platform: osx-shippable/opt + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: instrumented + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: profile-generate + fetches: + toolchain: + - linux64-clang + - linux64-hfsplus + - linux64-libdmg + - linux64-rust-macos + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - macosx64-sdk + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + macosx64-aarch64-shippable/opt: + description: "MacOS X aarch64 Cross-compile Instrumented" + shipping-phase: build + shipping-product: firefox + treeherder: + platform: osx-aarch64-shippable/opt + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: instrumented + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: profile-generate + extra-config: + mozconfig_platform: macosx64-aarch64 + fetches: + toolchain: + - linux64-clang + - linux64-hfsplus + - linux64-libdmg + - linux64-rust-macos + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - macosx64-sdk + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + android-x86-shippable/opt: + description: "Android 5.0 x86 PGO instrumented" + index: + product: mobile + job-name: instrumented-android-x86 + shipping-product: fennec + treeherder: + platform: android-5-0-x86/pgo + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + artifacts: + - name: public/build + path: /builds/worker/artifacts/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview-test_runner.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/bundle/withGeckoBinariesDebug/test_runner-withGeckoBinaries-debug.aab + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/bundle/withGeckoBinariesDebug/geckoview_example-withGeckoBinaries-debug.aab + type: file + run: + config: + - builds/releng_base_android_64_builds.py + custom-build-variant-cfg: x86-profile-generate + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-clang + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + android-x86_64-shippable/opt: + description: "Android 5.0 x86_64 PGO instrumented" + index: + product: mobile + job-name: instrumented-android-x86_64 + shipping-product: fennec + treeherder: + platform: android-5-0-x86_64/pgo + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + artifacts: + - name: public/build + path: /builds/worker/artifacts/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview-test_runner.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/bundle/withGeckoBinariesDebug/test_runner-withGeckoBinaries-debug.aab + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/bundle/withGeckoBinariesDebug/geckoview_example-withGeckoBinaries-debug.aab + type: file + run: + config: + - builds/releng_base_android_64_builds.py + custom-build-variant-cfg: x86_64-profile-generate + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-clang + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + android-aarch64-shippable/opt: + description: "Android 5.0 AArch64 PGO instrumented" + index: + product: mobile + job-name: instrumented-android-aarch64 + shipping-product: fennec + treeherder: + platform: android-5-0-aarch64/pgo + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + artifacts: + - name: public/build + path: /builds/worker/artifacts/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview-test_runner.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/apk/withGeckoBinaries/debug/test_runner-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview-test_runner.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/test_runner/outputs/bundle/withGeckoBinariesDebug/test_runner-withGeckoBinaries-debug.aab + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + - name: public/build/geckoview_example.aab + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/bundle/withGeckoBinariesDebug/geckoview_example-withGeckoBinaries-debug.aab + type: file + run: + config: + - builds/releng_base_android_64_builds.py + custom-build-variant-cfg: aarch64-profile-generate + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-clang + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + win32-shippable/opt: + description: "Win32 Shippable PGO Instrumented" + index: + product: firefox + job-name: instrumented-win32 + shipping-product: firefox + treeherder: + platform: windows2012-32-shippable/opt + worker-type: b-linux-gcp + run: + options: [append-env-variables-from-configs] + mozconfig-variant: profile-generate + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + - windows-rs + fetch: + - upx-3.95-win + + win64-shippable/opt: + description: "Win64 Shippable PGO Instrumented" + index: + product: firefox + job-name: instrumented-win64 + shipping-product: firefox + treeherder: + platform: windows2012-64-shippable/opt + worker-type: b-linux-gcp + run: + options: [append-env-variables-from-configs] + mozconfig-variant: profile-generate + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + - windows-rs + fetch: + - upx-3.95-win diff --git a/taskcluster/ci/l10n-bump/kind.yml b/taskcluster/ci/l10n-bump/kind.yml new file mode 100644 index 0000000000..2dd21e4068 --- /dev/null +++ b/taskcluster/ci/l10n-bump/kind.yml @@ -0,0 +1,131 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_version_bump:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + worker-type: tree + worker: + implementation: treescript + tags: [] + bump: false + dontbuild: + by-project: + mozilla-beta: false + default: true + ignore-closed-tree: + by-project: + autoland: true + default: false + push: + by-project: + mozilla-(central|beta): true + autoland: true + default: false + source-repo: + by-release-type: + beta: https://hg.mozilla.org/releases/mozilla-beta + nightly: https://hg.mozilla.org/mozilla-central + default: https://hg.mozilla.org/integration/autoland + l10n-bump-info: + by-release-type: + # XXX whenever the `beta` config changes, make sure to make + # the corresponding change in merge-automation/kind.yml + beta: + name: Firefox l10n changesets + path: browser/locales/l10n-changesets.json + version-path: browser/config/version.txt + l10n-repo-url: "https://hg.mozilla.org/l10n-central/%(locale)s/json-pushes?version=2&tipsonly=1" + ignore-config: + ja: + - macosx64 + - macosx64-devedition + ja-JP-mac: + - linux + - linux-devedition + - linux64 + - linux64-devedition + - win32 + - win32-devedition + - win64 + - win64-devedition + - win64-aarch64 + - win64-aarch64-devedition + platform-configs: + [ + { + "platforms": + [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/shipped-locales", + }, + ] + default: + name: Firefox l10n changesets + path: browser/locales/l10n-changesets.json + l10n-repo-url: "https://hg.mozilla.org/l10n-central/%(locale)s/json-pushes?version=2&tipsonly=1" + version-path: browser/config/version.txt + ignore-config: + ja: + - macosx64 + - macosx64-devedition + ja-JP-mac: + - linux + - linux-devedition + - linux64 + - linux64-devedition + - win32 + - win32-devedition + - win64 + - win64-devedition + - win64-aarch64 + - win64-aarch64-devedition + platform-configs: + [ + { + "platforms": + [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/all-locales", + }, + ] + +jobs: + l10n-bumper: + name: l10n_bumper + description: l10n changesets bumper + run-on-projects: [] + treeherder: + kind: build + platform: linux64/opt + symbol: l10n-bump(lb) + tier: 1 diff --git a/taskcluster/ci/l10n-cross-channel/kind.yml b/taskcluster/ci/l10n-cross-channel/kind.yml new file mode 100644 index 0000000000..da30306599 --- /dev/null +++ b/taskcluster/ci/l10n-cross-channel/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.cross_channel:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + quarantine: + description: Push strings from all shipping trains to the quarantine strings repo + run-on-projects: [] + worker-type: b-linux-gcp + ssh-key-secret: + by-level: + "3": project/releng/gecko/build/level-3/l10n-cross-channel-quarantine-ssh + default: null + worker: + docker-image: {in-tree: push-to-try} + max-run-time: 3600 + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts + env: + TASK_ID: {"task-reference": ""} + TASKCLUSTER_PROXY_URL: http://taskcluster + treeherder: + platform: firefox-release/opt + tier: 1 + kind: build + symbol: Rel(l10n-cross-channel) + run: + using: mach + actions: + by-level: + "3": ["prep", "create", "push", "clean"] + default: ["prep", "create", "clean"] diff --git a/taskcluster/ci/l10n/kind.yml b/taskcluster/ci/l10n/kind.yml new file mode 100644 index 0000000000..d1effef2e6 --- /dev/null +++ b/taskcluster/ci/l10n/kind.yml @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.l10n + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + - build-signing + - fetch + - repackage + - toolchain + +only-for-build-platforms: + - linux64-shippable/opt + - linux-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +jobs: + l10n: + description: Localization + from-deps: + group-by: platform + locales-file: browser/locales/l10n-onchange-changesets.json + ignore-locales: + by-build-platform: + # OSX has a special locale for japanese + macosx64.*: [ja] + default: [ja-JP-mac] + run-time: + by-build-platform: + default: 900 + win.*: 1800 + secrets: false + fetches: + toolchain: + by-build-platform: + default: [] + macosx64.*: + - linux64-libdmg + - linux64-hfsplus + win.*: + - linux64-wine + - nsis + fetch: + by-build-platform: + default: [] + win.*: + - upx-3.95-win + tooltool: + by-build-platform: + default: internal + linux.*: public + index: + type: l10n + product: firefox + job-name: + by-build-platform: + linux(?!64).*: linux-opt + linux64.*: linux64-opt + macosx64.*: macosx64-opt + win32.*: win32-opt + win64.*: win64-opt + run-on-projects: + by-build-platform: + linux-shippable.*: ['release'] # Do not run on autoland - Bug 1599197 + default: ['all'] + worker-type: b-linux-gcp + treeherder: + symbol: L10n + tier: 1 + platform: + by-build-platform: + linux64: linux64/opt + linux64-shippable: linux64-shippable/opt + linux: linux32/opt + linux-shippable: linux32-shippable/opt + macosx64: osx-cross/opt + macosx64-shippable: osx-shippable/opt + win32: windows2012-32/opt + win32-shippable: windows2012-32-shippable/opt + win64: windows2012-64/opt + win64-shippable: windows2012-64-shippable/opt + env: + by-build-platform: + linux.*: # linux64 and 32 get same treatment here + EN_US_PACKAGE_NAME: target.tar.bz2 + EN_US_BINARY_URL: + artifact-reference: + macosx64.*: + EN_US_PACKAGE_NAME: target.dmg + EN_US_BINARY_URL: + artifact-reference: + win.*: + EN_US_PACKAGE_NAME: target.zip + EN_US_BINARY_URL: + artifact-reference: + mozharness: + config: + by-build-platform: + linux(?!64).*: + - single_locale/firefox.py + - single_locale/linux32.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + linux64.*: + - single_locale/firefox.py + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + macosx64.*: + - single_locale/firefox.py + - single_locale/macosx64.py + - single_locale/tc_common.py + - single_locale/tc_macosx64.py + win32.*: + - single_locale/firefox.py + - single_locale/win32.py + - single_locale/tc_common.py + - single_locale/tc_win32.py + win64.*: + - single_locale/firefox.py + - single_locale/win64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + # no default, so we fail on new entries + actions: ['clone-locales', 'list-locales', 'setup', 'repack', 'summary'] + script: mozharness/scripts/desktop_l10n.py + when: + files-changed: + - browser/locales/l10n-onchange-changesets.json + - testing/mozharness/configs/single_locale/** + - testing/mozharness/mozharness/mozilla/l10n/locales.py + - testing/mozharness/scripts/desktop_l10n.py + - third_party/python/compare-locales/** + - toolkit/locales/** + - toolkit/mozapps/installer/** diff --git a/taskcluster/ci/mar-signing-autograph-stage/kind.yml b/taskcluster/ci/mar-signing-autograph-stage/kind.yml new file mode 100644 index 0000000000..711c28c44b --- /dev/null +++ b/taskcluster/ci/mar-signing-autograph-stage/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.mar_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage + +only-for-build-platforms: + - linux64-shippable/opt + +jobs: + mar-signing-autograph-stage: + from-deps: + group-by: single-with-filters + shipping-phase: null + treeherder-group: ms-stage + treeherder: + tier: 3 + description-suffix: 'autograph-stage mar signing test' + required_signoffs: + - mar-signing + run-on-projects: [] + shippable: false + attributes: + retrigger: true + worker-type: linux-depsigning diff --git a/taskcluster/ci/mar-signing-l10n/kind.yml b/taskcluster/ci/mar-signing-l10n/kind.yml new file mode 100644 index 0000000000..ad470915ef --- /dev/null +++ b/taskcluster/ci/mar-signing-l10n/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.mar_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-l10n + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - linux64-asan-reporter-shippable/opt + - linux-devedition/opt + - linux64-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + +jobs: + mar-signing-l10n: + from-deps: + group-by: single-with-filters + shipping-phase: promote + treeherder-group: ms + description-suffix: 'mar signing' + required_signoffs: + - mar-signing diff --git a/taskcluster/ci/mar-signing/kind.yml b/taskcluster/ci/mar-signing/kind.yml new file mode 100644 index 0000000000..f1337058d0 --- /dev/null +++ b/taskcluster/ci/mar-signing/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.mar_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - linux-devedition/opt + - linux64-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + - linux64-asan-reporter-shippable/opt + - win64-asan-reporter-shippable/opt + +jobs: + mar-signing: + from-deps: + group-by: single-with-filters + shipping-phase: promote + treeherder-group: ms + description-suffix: 'mar signing' + required_signoffs: + - mar-signing diff --git a/taskcluster/ci/maybe-release/kind.yml b/taskcluster/ci/maybe-release/kind.yml new file mode 100644 index 0000000000..8664e52168 --- /dev/null +++ b/taskcluster/ci/maybe-release/kind.yml @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.maybe_release:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: Check recent releases and trigger a new ship-it phase if worthwhile. + worker-type: shipit + worker: + implementation: shipit-maybe-release + scopes: + by-release-level: + production: + - project:releng:ship-it:server:production + - project:releng:ship-it:action:create-new-release + staging: + - project:releng:ship-it:server:staging + - project:releng:ship-it:action:create-new-release + run-on-projects: [] + treeherder: + symbol: Rel(maybe-release) + tier: 1 + kind: build + +jobs: + firefox: + name: maybe-release-firefox + shipping-product: firefox + worker: + phase: ship_firefox + treeherder: + platform: firefox-release/opt + + devedition: + name: maybe-release-devedition + shipping-product: devedition + worker: + phase: ship_devedition + treeherder: + platform: devedition-release/opt diff --git a/taskcluster/ci/merge-automation/kind.yml b/taskcluster/ci/merge-automation/kind.yml new file mode 100644 index 0000000000..8e2b34147e --- /dev/null +++ b/taskcluster/ci/merge-automation/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.merge_automation:transforms + - gecko_taskgraph.transforms.task:transforms + + +jobs: + merge-automation: + name: merge-automation + label: merge-automation + treeherder: + symbol: Rel(merge-automation) + tier: 1 + kind: build + platform: firefox-release/opt + description: Merge repositories for release management. + worker-type: tree + scopes: [] + worker: + implementation: treescript + tags: [] + bump: false + dontbuild: false + ignore-closed-tree: true + push: false + l10n-bump-info: + by-behavior: + central-to-beta: + name: Firefox l10n changesets + path: browser/locales/l10n-changesets.json + version-path: browser/config/version.txt + l10n-repo-url: "https://hg.mozilla.org/l10n-central/%(locale)s/json-pushes?version=2&tipsonly=1" + ignore-config: + ja: + - macosx64 + - macosx64-devedition + ja-JP-mac: + - linux + - linux-devedition + - linux64 + - linux64-devedition + - win32 + - win32-devedition + - win64 + - win64-devedition + - win64-aarch64 + - win64-aarch64-devedition + platform-configs: [{ + "platforms": [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/shipped-locales", + }] + default: null + run-on-projects: [] + expiration-policy: long diff --git a/taskcluster/ci/openh264-plugin/kind.yml b/taskcluster/ci/openh264-plugin/kind.yml new file mode 100644 index 0000000000..993489e3f6 --- /dev/null +++ b/taskcluster/ci/openh264-plugin/kind.yml @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.openh264:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: "Build OpenH264 plugin" + treeherder: + kind: build + symbol: h264 + tier: 2 + run-on-projects: [] + repo: 'https://github.com/cisco/openh264.git' + revision: '31c4d2e4a037526fd30d4e5c39f60885986cf865' + use-system-python: false + worker: + chain-of-trust: true + +jobs: + linux32/opt: + attributes: + build_platform: linux32 + build_type: opt + treeherder: + platform: linux32/opt + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: /builds/worker/workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/linux32.py + fetches: + toolchain: + - linux64-binutils + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - sysroot-i686-linux-gnu + linux64/opt: + attributes: + build_platform: linux64 + build_type: opt + treeherder: + platform: linux64/opt + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: /builds/worker/workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/linux64.py + fetches: + toolchain: + - linux64-binutils + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - sysroot-x86_64-linux-gnu + linux64-aarch64/opt: + attributes: + build_platform: linux64-aarch64 + build_type: opt + treeherder: + platform: linux64-aarch64/opt + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: /builds/worker/workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/linux64-aarch64.py + fetches: + toolchain: + - linux64-binutils + - linux64-clang + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + - sysroot-aarch64-linux-gnu + macosx64/opt: + attributes: + build_platform: macosx64 + build_type: opt + treeherder: + platform: macosx64/opt + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: /builds/worker/workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/macosx64.py + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - macosx64-sdk-13.3 + macosx64-aarch64/opt: + attributes: + build_platform: macosx64-aarch64 + build_type: opt + treeherder: + platform: macosx64-aarch64/opt + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: /builds/worker/workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/macosx64-aarch64.py + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang + - linux64-dump_syms + - macosx64-sdk-13.3 + win32/opt: + attributes: + build_platform: win32 + build_type: opt + treeherder: + platform: win32/opt + worker-type: b-win2022 + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/win32.py + tooltool-downloads: internal + fetches: + toolchain: + - win64-clang + - win64-dump_syms + - win64-mozmake + - win64-nasm + win64/opt: + attributes: + build_platform: win64 + build_type: opt + treeherder: + platform: win64/opt + worker-type: b-win2022 + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/win64.py + tooltool-downloads: internal + fetches: + toolchain: + - win64-clang + - win64-dump_syms + - win64-mozmake + - win64-nasm + win64-aarch64/opt: + attributes: + build_platform: win64-aarch64 + build_type: opt + treeherder: + platform: win64-aarch64/opt + worker-type: b-win2022 + worker: + max-run-time: 1800 + artifacts: + - name: private/openh264 + path: workspace/openh264/artifacts + type: directory + run: + using: mozharness + script: mozharness/scripts/openh264_build.py + config: + - openh264/win64-aarch64.py + tooltool-downloads: internal + fetches: + toolchain: + - win64-clang + - win64-dump_syms + - win64-mozmake diff --git a/taskcluster/ci/openh264-signing/kind.yml b/taskcluster/ci/openh264-signing/kind.yml new file mode 100644 index 0000000000..e4dc174cc7 --- /dev/null +++ b/taskcluster/ci/openh264-signing/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.openh264_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - openh264-plugin + +jobs: + openh264-signing: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/packages/debian.yml b/taskcluster/ci/packages/debian.yml new file mode 100644 index 0000000000..0a87ef6782 --- /dev/null +++ b/taskcluster/ci/packages/debian.yml @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: debian-package + dist: jessie + +deb12-mercurial: + description: "Modern Mercurial for Debian bookworm" + treeherder: + symbol: Deb12(hg) + run: + using: debian-package + dist: bookworm + tarball: + url: https://www.mercurial-scm.org/release/mercurial-6.4.3.tar.gz + sha256: a77fb62d325934a83a00b2fc6db109f44c67b86768fc40490366f513640c7f64 + pre-build-command: >- + cp -r contrib/packaging/debian debian && + sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.deb12moz1/" \ + -e "s/__DATE__/$(date --rfc-2822)/" \ + -e "s/__CODENAME__/bookworm/" debian/changelog + +deb12-valgrind: + description: "Valgrind for Debian bookworm" + treeherder: + symbol: Deb12(valgrind) + run: + dist: bookworm + dsc: + url: http://snapshot.debian.org/archive/debian/20230111T213757Z/pool/main/v/valgrind/valgrind_3.20.0-1.dsc + sha256: e32d373bf2d4f0bd0e9673c711d0e4a8eed43c22ca81714ae3e2d85b2f315493 + +deb12-python-zstandard: + description: "python-zstandard for Debian bookworm" + treeherder: + symbol: Deb12(python-zstandard) + run: + using: debian-package + dist: bookworm + tarball: + url: https://github.com/indygreg/python-zstandard/releases/download/0.22.0/zstandard-0.22.0.tar.gz + sha256: 8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70 + pre-build-command: debchange -v 0.22.0-1.deb12moz --distribution testing "Mozilla backport for bookworm" + name: python-zstandard + +deb8-gcc-8: + description: "gcc 8 for Debian jessie" + treeherder: + symbol: Deb8(gcc-8) + worker: + max-run-time: 3600 + env: + DEB_BUILD_OPTIONS: nocheck + WITH_BOOTSTRAP: 'off' + WITHOUT_LANG: ada brig cilkrts fortran d go objc obj-c++ asan lsan tsan ubsan jit nvptx + run: + using: debian-package + dist: jessie + dsc: + url: http://snapshot.debian.org/archive/debian/20210202T150504Z/pool/main/g/gcc-8/gcc-8_8.4.0-7.dsc + sha256: b93f13bc67f5a8f16e837fbb543d00d3475d5c95777a1b0b5a448fa2310a98c5 + # autoconf2.64 and quilt necessary to run the `debian/rules control` command to + # generate an updated debian/control. + # uncommenting `with_common_libs` makes the packaging create the libstdc++6 package. + # `libstdcxx_abi = new` overrides the default that disables the new ABI on jessie + # `BINUTILSBDV=2.25-3~` and `ISL_BUILD_DEP=libisl-dev,` override wrong versions + # in build dependencies for jessie. + # `OFFLOAD_BUILD_DEP=` overrides the dependency on nvptx, which is not available + # in jessie. + # `NT=` overrides the use of `:native` which is not supported by dpkg on jessie. + pre-build-command: >- + apt install autoconf2.64 quilt && + sed -i 's/#with_common_libs/with_common_libs/' debian/rules.defs && + echo libstdcxx_abi = new >> debian/rules.defs && + echo BINUTILSBDV=2.25-3~ >> debian/rules.conf && + echo ISL_BUILD_DEP=libisl-dev, >> debian/rules.conf && + echo OFFLOAD_BUILD_DEP= >> debian/rules.conf && + debian/rules control NT= + +deb8-32-gcc-8: + description: "gcc 8 for Debian jessie" + treeherder: + symbol: Deb8-32(gcc-8) + worker: + max-run-time: 3600 + env: + DEB_BUILD_OPTIONS: nocheck + WITH_BOOTSTRAP: 'off' + WITHOUT_LANG: ada biarch brig cilkrts fortran d go objc obj-c++ asan lsan tsan ubsan jit nvptx + run: + using: debian-package + dist: jessie + dsc: + url: http://snapshot.debian.org/archive/debian/20210202T150504Z/pool/main/g/gcc-8/gcc-8_8.4.0-7.dsc + sha256: b93f13bc67f5a8f16e837fbb543d00d3475d5c95777a1b0b5a448fa2310a98c5 + pre-build-command: >- + apt install autoconf2.64 quilt && + sed -i 's/#with_common_libs/with_common_libs/' debian/rules.defs && + echo libstdcxx_abi = new >> debian/rules.defs && + echo GCC_MULTILIB_BUILD_DEP='$(DPKG_BUILD_DEP)' >> debian/rules.conf && + echo BINUTILSBDV=2.25-3~ >> debian/rules.conf && + echo ISL_BUILD_DEP=libisl-dev, >> debian/rules.conf && + echo OFFLOAD_BUILD_DEP= >> debian/rules.conf && + debian/rules control NT= + arch: i386 + +deb8-gtk3: + description: "X11-only gtk+3 for Debian jessie" + treeherder: + symbol: Deb8(gtk) + worker: + max-run-time: 3600 + env: + DEB_BUILD_OPTIONS: nocheck + run: + dist: jessie + dsc: + url: http://snapshot.debian.org/archive/debian/20150819T212910Z/pool/main/g/gtk+3.0/gtk+3.0_3.14.5-1+deb8u1.dsc + sha256: b1808b03317b171c813e31185a4520df4dbde54580842a72a4c44540cd6bde17 + pre-build-command: >- + sed -i /linux-any/d debian/control* debian/libgtk-3-0.symbols && + sed -i s/--enable-wayland-backend/--disable-wayland-backend/ debian/rules && + debchange -v 3.14.5-1.deb8moz --distribution jessie-backports "Mozilla rebuild without wayland support" diff --git a/taskcluster/ci/packages/kind.yml b/taskcluster/ci/packages/kind.yml new file mode 100644 index 0000000000..5b21346de8 --- /dev/null +++ b/taskcluster/ci/packages/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.cached_tasks:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + kind: build + platform: packages/opt + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run-on-projects: [] + +jobs-from: + - debian.yml + - ubuntu.yml diff --git a/taskcluster/ci/packages/ubuntu.yml b/taskcluster/ci/packages/ubuntu.yml new file mode 100644 index 0000000000..d9bbd5f2ad --- /dev/null +++ b/taskcluster/ci/packages/ubuntu.yml @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: ubuntu-package + dist: bionic + +ub18-libc6: + description: "Older version of glibc in Ubuntu 18.04 that doesn't cause bug 1678451" + treeherder: + symbol: Ub18(libc6) + run: + dsc: + url: https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/glibc/2.27-3ubuntu1.2/glibc_2.27-3ubuntu1.2.dsc + sha256: 2ac325cce2251c710dd041f4725925cdbccefba6f6f0f0d95a2a5a09e2050c2e + patch: ubuntu-glibc.diff + +ub18-32-libc6: + description: "Older version of glibc in Ubuntu 18.04 that doesn't cause bug 1678451" + treeherder: + symbol: Ub18(libc6-32) + run: + dsc: + url: https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/glibc/2.27-3ubuntu1.2/glibc_2.27-3ubuntu1.2.dsc + sha256: 2ac325cce2251c710dd041f4725925cdbccefba6f6f0f0d95a2a5a09e2050c2e + arch: i386 + patch: ubuntu-glibc.diff + +ub18-python-psutil: + description: "python-psutil for Ubuntu bionic" + treeherder: + symbol: Ub18(python-psutil) + run: + dsc: + url: https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-psutil/5.4.2-1ubuntu0.1/python-psutil_5.4.2-1ubuntu0.1.dsc + sha256: a0ca4560d25fcfe5b7407d2cb21318f0635f2c3559898efce1679eef35b61f05 + name: python-psutil + pre-build-command: >- + apt-get install python3.8-dev python3.8-dbg && + export DEBPYTHON3_SUPPORTED="3.8" + +ub18-pygobject: + description: "pygobject for Ubuntu bionic" + treeherder: + symbol: Ub18(pygobject) + run: + dsc: + url: http://archive.ubuntu.com/ubuntu/pool/main/p/pygobject/pygobject_3.26.1-2ubuntu1.dsc + sha256: b77f84885d9b6d2c03422ecb3bf69d50dd0607d3456154943671e5a1f87946aa + name: pygobject + patch: pygobject.diff + pre-build-command: >- + apt-get install python3.8-dev python3.8-dbg && + export DEBPYTHON3_SUPPORTED="3.6,3.8" + +ub18-pyatspi: + description: "pyatspi for Ubuntu bionic" + treeherder: + symbol: Ub18(pyatspi) + run: + dsc: + url: http://archive.ubuntu.com/ubuntu/pool/main/p/pyatspi/pyatspi_2.26.0+dfsg-1.dsc + sha256: 83c57052b055c0e2ee8307e3467d069de859db6c362137be01d142e085026826 + name: pyatspi + pre-build-command: >- + sed -i s/\\basync\\b/asynchronous/ pyatspi/registry.py && + EDITOR=cat dpkg-source --commit . sync && + apt-get install python3.8-dev python3.8-dbg && + export DEBPYTHON3_SUPPORTED="3.6,3.8" + +ub18-python-zstandard: + description: "python-zstandard for Ubuntu bionic" + treeherder: + symbol: Ub18(python-zstandard) + run: + tarball: + url: https://github.com/indygreg/python-zstandard/releases/download/0.19.0/zstandard-0.19.0.tar.gz + sha256: 31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863 + name: python-zstandard + pre-build-command: >- + apt-get install python3.8-dev && + export DEBPYTHON3_SUPPORTED="3.8" && + debchange -v 0.19.0-1.ub18moz --distribution bionic "Mozilla backport for bionic" + +ub18-mercurial: + description: "Modern Mercurial for Ubuntu bionic" + treeherder: + symbol: Ub18(hg) + run: + tarball: + url: https://www.mercurial-scm.org/release/mercurial-6.4.3.tar.gz + sha256: a77fb62d325934a83a00b2fc6db109f44c67b86768fc40490366f513640c7f64 + pre-build-command: >- + apt-get install python3.8-dev && + export DEB_HG_PYTHON_VERSIONS="3.8" && + export DEBPYTHON3_SUPPORTED="3.8" && + cp -r contrib/packaging/debian debian && + sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.ub18moz1/" \ + -e "s/__DATE__/$(date --rfc-2822)/" \ + -e "s/__CODENAME__/bionic/" debian/changelog + +ub20-python-zstandard: + description: "python-zstandard for Ubuntu focal" + treeherder: + symbol: Ub20(python-zstandard) + run: + tarball: + url: https://github.com/indygreg/python-zstandard/releases/download/0.22.0/zstandard-0.22.0.tar.gz + sha256: 8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70 + name: python-zstandard + dist: focal + pre-build-command: debchange -v 0.22.0-1.ub20moz --distribution bionic "Mozilla backport for focal" + +ub20-mercurial: + description: "Modern Mercurial for Ubuntu focal" + treeherder: + symbol: Ub20(hg) + run: + tarball: + url: https://www.mercurial-scm.org/release/mercurial-6.4.3.tar.gz + sha256: a77fb62d325934a83a00b2fc6db109f44c67b86768fc40490366f513640c7f64 + dist: focal + pre-build-command: >- + cp -r contrib/packaging/debian debian && + sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.ub20moz1/" \ + -e "s/__DATE__/$(date --rfc-2822)/" \ + -e "s/__CODENAME__/focal/" debian/changelog + +ub22-python-zstandard: + description: "python-zstandard for Ubuntu jammy" + treeherder: + symbol: Ub22(python-zstandard) + run: + tarball: + url: https://github.com/indygreg/python-zstandard/releases/download/0.22.0/zstandard-0.22.0.tar.gz + sha256: 8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70 + name: python-zstandard + dist: jammy + pre-build-command: debchange -v 0.22.0-1.ub22moz --distribution jammy "Mozilla backport for jammy" + +ub22-mercurial: + description: "Modern Mercurial for Ubuntu jammy" + treeherder: + symbol: Ub22(hg) + run: + tarball: + url: https://www.mercurial-scm.org/release/mercurial-6.4.3.tar.gz + sha256: a77fb62d325934a83a00b2fc6db109f44c67b86768fc40490366f513640c7f64 + dist: jammy + pre-build-command: >- + cp -r contrib/packaging/debian debian && + sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.ub22moz1/" \ + -e "s/__DATE__/$(date --rfc-2822)/" \ + -e "s/__CODENAME__/jammy/" debian/changelog diff --git a/taskcluster/ci/partials-signing/kind.yml b/taskcluster/ci/partials-signing/kind.yml new file mode 100644 index 0000000000..2c2bf0a9a0 --- /dev/null +++ b/taskcluster/ci/partials-signing/kind.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.mar_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - partials + +jobs: + partials-signing: + from-deps: + group-by: single-with-filters + shipping-phase: promote + treeherder-group: ps + description-suffix: 'partial signing' + required_signoffs: + - mar-signing diff --git a/taskcluster/ci/partials/kind.yml b/taskcluster/ci/partials/kind.yml new file mode 100644 index 0000000000..ea72198c5c --- /dev/null +++ b/taskcluster/ci/partials/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.partials + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage + - repackage-l10n + +only-for-attributes: + - nightly + - shippable + +only-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + - win32-shippable/opt + - win32-devedition/opt + - win64-shippable/opt + - win64-devedition/opt + - win64-aarch64-shippable/opt + - win64-aarch64-devedition/opt + - linux-shippable/opt + - linux-devedition/opt + - linux64-shippable/opt + - linux64-devedition/opt + +jobs: + partials: + from-deps: + group-by: single-with-filters + shipping-phase: promote diff --git a/taskcluster/ci/perftest/android.yml b/taskcluster/ci/perftest/android.yml new file mode 100644 index 0000000000..6faebffee9 --- /dev/null +++ b/taskcluster/ci/perftest/android.yml @@ -0,0 +1,338 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +job-defaults: + fetches: + toolchain: + - linux64-node + - linux64-geckodriver + + worker: + env: + PYTHONPATH: "testing/mozbase/mozdevice:third_party/python/six" + + run: + sparse-profile: perftest + checkout: true + cwd: '{checkout}' + command: >- + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +hw-a51: + worker-type: t-bitbar-gw-perf-a51 + description: Run ./mach perftest on a Samsung Galaxy a51 + treeherder: + symbol: perftest-a51 + platform: android-hw-a51-11-0-arm7-shippable/opt + +# TODO: is this right? +hw-a51-aarch64: + worker-type: t-bitbar-gw-perf-a51 + description: Run ./mach perftest on a Samsung Galaxy a51 + treeherder: + symbol: perftest-a51 + platform: android-hw-a51-11-0-aarch64-shippable/opt + +hw-a51-startup-fenix-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(startup-fenix-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=fenix + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-fenix-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(startup-fenix-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=fenix + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-focus-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Focus on a Samsung A51 + treeherder: + symbol: perftest(startup-focus-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=focus + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-focus-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Focus on a Samsung A51 + treeherder: + symbol: perftest(startup-focus-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=focus + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-geckoview-cold-main-first-frame: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Geckoview on a Samsung A51 + treeherder: + symbol: perftest(startup-geckoview-cmff) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_main_first_frame + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=geckoview_example + --AndroidStartUp-release-channel=nightly + +hw-a51-startup-geckoview-cold-view-nav-start: + worker-type: t-bitbar-gw-perf-a51 + description: Run android startup perftest on Geckoview_example on a Samsung A51 + treeherder: + symbol: perftest(startup-geckoview-cvns) + tier: 2 + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor=mobile-browser + --AndroidStartUp + testing/performance/perftest_android_startup.js + --browsertime-cycles=0 + --AndroidStartUp-test-name=cold_view_nav_start + --perfherder + --hooks + testing/performance/hooks_android_startup.py + --AndroidStartUp-product=geckoview_example + --AndroidStartUp-release-channel=nightly + +hw-a51-perfstats-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on GeckoView on a Samsung A51 + treeherder: + symbol: perftest(perfstats-gv) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_api16 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app geckoview + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-aarch64-perfstats-gv: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on GeckoView on a Samsung A51 + treeherder: + symbol: perftest(perfstats-gv) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_aarch64 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app geckoview + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-perfstats-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(perfstats-fenix) + platform: android-hw-a51-11-0-arm7-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk fenix_nightly_armeabi_v7a + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app fenix + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-aarch64-perfstats-fenix: + worker-type: t-bitbar-gw-perf-a51 + description: Run PerfStats perftest on Fenix on a Samsung A51 + treeherder: + symbol: perftest(perfstats-fenix) + platform: android-hw-a51-11-0-aarch64-shippable/opt + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.fenix + --android-activity org.mozilla.fenix.IntentReceiverActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk fenix_nightly_arm64_v8a + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-app fenix + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +hw-a51-record-websites: + worker-type: t-bitbar-gw-perf-a51 + description: Run perftest to record mobile websites + treeherder: + symbol: perftest(rec) + platform: android-hw-a51-11-0-aarch64-shippable/opt + require-build: + android-hw-a51-11-0-aarch64-shippable/opt: build-android-aarch64-shippable/opt + attributes: + cron: false + run-on-projects: [] + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --flavor mobile-browser + --android + --android-app-name org.mozilla.geckoview_example + --android-activity org.mozilla.geckoview_example.GeckoViewActivity + --android-clear-logcat + --android-capture-logcat logcat + --android-install-apk gve_nightly_aarch64 + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --proxy + --output $MOZ_FETCHES_DIR/../artifacts + --hooks testing/performance/hooks_recording.py + testing/performance/perftest_record.js diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml new file mode 100644 index 0000000000..5d8f253653 --- /dev/null +++ b/taskcluster/ci/perftest/kind.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - gecko_taskgraph.transforms.perftest:transforms + - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - android.yml + - linux.yml + - macosx.yml + - windows.yml + +job-defaults: + attributes: + retrigger: true + run-on-projects: [] + use-system-python: false + treeherder: + kind: other + tier: 3 + task-context: + from-object: {} + substitution-fields: [] + worker: + taskcluster-proxy: true + max-run-time: 10800 + env: + USE_ARTIFACT: '1' + HOSTUTILS_MANIFEST_PATH: "testing/config/tooltool-manifests/linux64/hostutils.manifest" + artifacts: + - type: directory + name: public/build + path: artifacts + run: + run-as-root: true + using: run-task + checkout: false diff --git a/taskcluster/ci/perftest/linux.yml b/taskcluster/ci/perftest/linux.yml new file mode 100644 index 0000000000..1016469fb8 --- /dev/null +++ b/taskcluster/ci/perftest/linux.yml @@ -0,0 +1,384 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-linux-talos-1804 + fetches: + toolchain: + - linux64-node + - linux64-geckodriver + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.condprof.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.mochitest.tests.tar.gz + - artifact: target.tar.bz2 + platform: linux1804-64-shippable/opt + require-build: + linux1804-64-shippable/opt: build-linux64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +webpagetest-firefox: + description: Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites + treeherder: + symbol: perftest(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_firefox_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=firefox + --output $MOZ_FETCHES_DIR/../artifacts + +webpagetest-chrome: + description: Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites + treeherder: + symbol: perftest-chrome(linux-webpagetest) + tier: 2 + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_chrome_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --perfherder-app=chrome + --output $MOZ_FETCHES_DIR/../artifacts + +try-browsertime: + description: Run ./mach perftest on Linux + treeherder: + symbol: perftest(linux-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(linux-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +# We'd like not to have to specify each test separately here. See bug 1865852. +service-worker: + description: Run service worker tests + treeherder: + symbol: perftest(linux-sw) + tier: 2 + attributes: + batch: false + cron: false + run-on-projects: [autoland, mozilla-central] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_caching.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_fetch.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_registration.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + # Disabled due to bug 1849400 + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + # broken: bug 1879851 + # - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + # broken: bug 1879851 + # g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 10 + --visualmetrics + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(linux-perfstats) + attributes: + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js + +record-websites: + description: Run mozperftest to record desktop websites + platform: linux1804-64/opt + require-build: + linux1804-64/opt: build-linux64/opt + treeherder: + symbol: perftest(rec) + attributes: + cron: false + run-on-projects: [] + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + --flavor desktop-browser + --verbose + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --proxy + --output $MOZ_FETCHES_DIR/../artifacts + --hooks testing/performance/hooks_recording.py + testing/performance/perftest_record.js + +side-by-side: + description: Run mozperftest side-by-side + treeherder: + symbol: perftest(side-by-side) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + tools side-by-side + -t {test_name} + --base-platform {platform} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} + +mwu-change-detector: + description: Runs Mann-Whitney U-test on two revisions to detect performance changes + treeherder: + symbol: perftest-detect(mwu) + tier: 2 + run: + using: run-task + sparse-profile: perftest + checkout: true + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 python/mozperftest/mozperftest/runner.py + tools change-detector + --task-name {task_name} + --base-revision {base_revision} + --new-revision {new_revision} + --base-branch {base_branch} + --new-branch {new_branch} diff --git a/taskcluster/ci/perftest/macosx.yml b/taskcluster/ci/perftest/macosx.yml new file mode 100644 index 0000000000..d98f7524b8 --- /dev/null +++ b/taskcluster/ci/perftest/macosx.yml @@ -0,0 +1,287 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-osx-1015-r8 + fetches: + build: + - artifact: target.mozinfo.json + - artifact: target.common.tests.tar.gz + - artifact: target.condprof.tests.tar.gz + - artifact: target.mochitest.tests.tar.gz + - artifact: target.xpcshell.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.dmg + extract: false + toolchain: + - macosx64-geckodriver + - macosx64-node + platform: macosx1015-64-shippable-qr/opt + require-build: + macosx1015-64-shippable-qr/opt: build-macosx64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-xpcshell: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-xpcshell) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --flavor desktop-browser + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + +try-browsertime: + description: Run ./mach perftest on macOs + treeherder: + symbol: perftest(macos-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +# We'd like not to have to specify each test separately here. See bug 1865852. +service-worker: + description: Run service worker tests + treeherder: + symbol: perftest(macosx-sw) + tier: 2 + attributes: + batch: false + cron: false + run-on-projects: [autoland, mozilla-central] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_caching.html + --mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + bin/python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_fetch.html + --mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + bin/python3 python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_registration.html + --mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test on macOS + treeherder: + symbol: perftest(macos-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +http3: + description: Run HTTP/3 test + treeherder: + symbol: perftest(http3) + attributes: + batch: false + # Disabled due to bug 1849400 + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + xpcshell/tests/netwerk/test/unit/test_http3_perf.js + --flavor xpcshell + --perfherder + --perfherder-metrics name:speed,unit:bps + --output $MOZ_FETCHES_DIR/../artifacts + --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell + --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json + --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node + --xpcshell-cycles 13 + --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + # broken: bug 1879851 + # - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + # broken: bug 1879851 + # g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + python3 python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(macos-perfstats) + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3 -m venv . && + bin/python3 python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg + --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js diff --git a/taskcluster/ci/perftest/windows.yml b/taskcluster/ci/perftest/windows.yml new file mode 100644 index 0000000000..3e027e5741 --- /dev/null +++ b/taskcluster/ci/perftest/windows.yml @@ -0,0 +1,235 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-win10-64-1803-hw + fetches: + build: + - artifact: target.condprof.tests.tar.gz + - artifact: target.common.tests.tar.gz + - artifact: target.mochitest.tests.tar.gz + - artifact: target.perftests.tests.tar.gz + - artifact: target.zip + toolchain: + - win64-node + - win64-geckodriver + platform: windows10-64-shippable/opt + require-build: + windows10-64-shippable/opt: build-win64-shippable/opt + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles + +try-browsertime: + description: Run ./mach perftest on windows + treeherder: + symbol: perftest(win-bt) + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +# We'd like not to have to specify each test separately here. See bug 1865852. +service-worker: + description: Run service worker tests + treeherder: + symbol: perftest(win-sw) + tier: 2 + attributes: + batch: false + cron: false + run-on-projects: [autoland, mozilla-central] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_caching.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + python3.exe python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_fetch.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts && + python3.exe python/mozperftest/mozperftest/runner.py + dom/serviceworkers/test/performance/test_registration.html + --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor mochitest + --output $MOZ_FETCHES_DIR/../artifacts + +domcount: + description: Run DOM test + treeherder: + symbol: perftest(win-dom) + attributes: + batch: false + cron: true + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + browser/base/content/test/perftest_browser_xhtml_dom.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --perfherder + --perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + + +livesites: + description: Live site performance testing + variants: [http3] + treeherder: + symbol: perftest({symbol}) + attributes: + batch: false + cron: true + perftest: + - [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js] + # broken: bug 1678588 + # - [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js] + # broken: bug 1879851 + # - [g-image, netwerk/test/perf/perftest_http3_google_image.js] + # broken: bug 1799655 + # - [g-search, netwerk/test/perf/perftest_http3_google_search.js] + - [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js] + # broken: bug 1695871 + # - [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js] + # - [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js] + perftest-metrics: + by-perftest: + cloudflare: + # Example of how the specify settings for each metric + resources: + extraOptions: [] + firstPaint: {} + navigationTiming: {} + pageTimings: {} + timeToContentfulPaint: {} + # broken: bug 1678588 + # fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond] + # broken: bug 1879851 + # g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime] + # g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed] + # broken: bug 1695871 + # ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + # ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames] + default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint] + # Leave this here so people know this exists when looking + # for examples + perftest-perfherder-global: + extraOptions: [] + perftest-extra-options: + by-perftest: + cloudflare: + - --perfherder-split-by browserScripts.pageinfo.url + # g-search: + # - --perfherder-split-by browserScripts.pageinfo.url + lq-fetch: + - --perfherder-split-by browserScripts.pageinfo.url + default: [] + perftest-btime-variants: + by-perftest: + cloudflare: + - ["10s", browsertime.waitTime=10000] + - ["25s", browsertime.waitTime=25000] + - ["35s", browsertime.waitTime=35000] + - ["60s", browsertime.waitTime=60000] + # broken: bug 1678588 + # fb-scroll: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + # g-search: + # - ["10s", browsertime.waitTime=10000] + # - ["25s", browsertime.waitTime=25000] + # - ["35s", browsertime.waitTime=35000] + # - ["60s", browsertime.waitTime=60000] + lq-fetch: + - [null, "browsertime.waitTime=1000,browsertime.cycles=5"] + # broken: bug 1695871 + # ytw: + # - [null, browsertime.waitTime=20000] + # ytw-scroll: + # - [null, browsertime.waitTime=20000] + default: + - [null, browsertime.waitTime=1000] + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + {perftest_testname} + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 10 + --flavor desktop-browser + --perfherder + --perfherder-metrics {perftest_metrics} + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +controlled: + description: Controlled performance testing + treeherder: + symbol: perftest(controlled) + worker: + max-run-time: 14400 + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + netwerk/test/perf/perftest_http3_controlled.js + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --browsertime-iterations 1 + --browsertime-cycles 96 + --hooks netwerk/test/perf/hooks_throttling.py + --flavor desktop-browser + --perfherder + --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms + --perfherder-simplify-names + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + +perfstats: + description: Run PerfStats pageload test + treeherder: + symbol: perftest(win-perfstats) + attributes: + batch: false + cron: false + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.exe python/mozperftest/mozperftest/runner.py + --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe + --flavor desktop-browser + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe + --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe + --output $MOZ_FETCHES_DIR/../artifacts + --perfherder + --perfherder-metrics name:HttpChannelCompletion,unit:ms name:HttpChannelCompletion_Cache,unit:ms name:HttpChannelCompletion_Network,unit:ms name:DisplayListBuilding,unit:ms name:Reflowing,unit:ms name:Styling,unit:ms + --browsertime-iterations 10 + --hooks testing/performance/hooks_perfstats.py + testing/performance/perftest_perfstats.js diff --git a/taskcluster/ci/post-balrog-dummy/kind.yml b/taskcluster/ci/post-balrog-dummy/kind.yml new file mode 100644 index 0000000000..e1a9849bbc --- /dev/null +++ b/taskcluster/ci/post-balrog-dummy/kind.yml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.per_platform_dummy + - gecko_taskgraph.transforms.reverse_chunk_deps + - gecko_taskgraph.transforms.task + +kind-dependencies: + - balrog + +jobs: + post-balrog-dummy: + description: Dummy task to deal with max_dependencies + from-deps: {} + run-on-projects: [] + shipping-phase: promote + worker-type: succeed diff --git a/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml b/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml new file mode 100644 index 0000000000..40ff97dce8 --- /dev/null +++ b/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.reverse_chunk_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-checksums + - release-beetmover-source-checksums + - release-eme-free-repack-beetmover-checksums + - release-beetmover-signed-langpacks-checksums + +jobs: + firefox-promote: + name: post-beetmover-checksums-dummy + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + shipping-product: firefox + worker-type: succeed + devedition-promote: + name: post-beetmover-checksums-dummy + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + shipping-product: devedition + worker-type: succeed diff --git a/taskcluster/ci/post-beetmover-dummy/kind.yml b/taskcluster/ci/post-beetmover-dummy/kind.yml new file mode 100644 index 0000000000..e5d50ce256 --- /dev/null +++ b/taskcluster/ci/post-beetmover-dummy/kind.yml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.per_platform_dummy + - gecko_taskgraph.transforms.reverse_chunk_deps + - gecko_taskgraph.transforms.task + +kind-dependencies: + - beetmover-checksums + - beetmover-repackage + +jobs: + post-beetmover-dummy: + description: Dummy task to deal with max_dependencies + from-deps: {} + run-on-projects: [] + shipping-phase: promote + worker-type: succeed diff --git a/taskcluster/ci/post-langpack-dummy/kind.yml b/taskcluster/ci/post-langpack-dummy/kind.yml new file mode 100644 index 0000000000..95fe9f7684 --- /dev/null +++ b/taskcluster/ci/post-langpack-dummy/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.reverse_chunk_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-beetmover-signed-langpacks + +jobs: + firefox-promote: + name: post-langpack-dummy + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + shipping-product: firefox + worker-type: succeed + + devedition-promote: + name: post-langpack-dummy + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + shipping-product: devedition + worker-type: succeed diff --git a/taskcluster/ci/post-update-verify-dummy/kind.yml b/taskcluster/ci/post-update-verify-dummy/kind.yml new file mode 100644 index 0000000000..6d7bfb3515 --- /dev/null +++ b/taskcluster/ci/post-update-verify-dummy/kind.yml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.per_platform_dummy + - gecko_taskgraph.transforms.reverse_chunk_deps + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-update-verify + +jobs: + post-update-verify-dummy: + description: Dummy task to deal with max_dependencies + from-deps: {} + run-on-projects: [] + shipping-phase: promote + worker-type: succeed diff --git a/taskcluster/ci/release-balrog-scheduling/kind.yml b/taskcluster/ci/release-balrog-scheduling/kind.yml new file mode 100644 index 0000000000..64bd751ca3 --- /dev/null +++ b/taskcluster/ci/release-balrog-scheduling/kind.yml @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-bouncer-check + - release-push-langpacks + +job-defaults: + run-on-projects: [] + shipping-phase: ship + worker-type: balrog + worker: + implementation: balrog + balrog-action: schedule + +jobs: + firefox: + description: Schedule Firefox publishing in balrog + name: release-firefox_schedule_publishing_in_balrog + shipping-product: firefox + worker: + product: firefox + publish-rules: + by-release-level: + production: + by-release-type: + beta: [32] + release: [145] + esr115: [17824] + default: [] + staging: + by-release-type: + beta: [32] + release: [145] + esr115: [909] + default: [] + background-rate: + by-release-type: + beta: + by-beta-number: + '1': 25 + '2': 50 + '3': 100 + default: null + default: null + force-fallback-mapping-update: + by-release-type: + beta: + by-beta-number: + '1': true + default: false + default: false + pin-channels: + by-release-type: + beta: ["beta"] + release: ["release"] + esr.*: ["esr", "esr-next"] + default: [] + treeherder: + platform: firefox-release/opt + symbol: Rel(BSFx) + tier: 1 + kind: build + devedition: + description: Schedule Devedition publishing in balrog + name: release-devedition_schedule_publishing_in_balrog + shipping-product: devedition + worker: + product: devedition + publish-rules: [10] + background-rate: + by-release-type: + beta: null + default: null + force-fallback-mapping-update: + by-release-type: + beta: true + default: false + pin-channels: ["aurora"] + treeherder: + platform: devedition-release/opt + symbol: Rel(BSDev) + tier: 1 + kind: build diff --git a/taskcluster/ci/release-balrog-submit-toplevel/kind.yml b/taskcluster/ci/release-balrog-submit-toplevel/kind.yml new file mode 100644 index 0000000000..dc1155df9d --- /dev/null +++ b/taskcluster/ci/release-balrog-submit-toplevel/kind.yml @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.balrog_toplevel:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + run-on-projects: [] # to make sure this never runs in CI + shipping-phase: promote + worker-type: balrog + worker: + implementation: balrog + balrog-action: v2-submit-toplevel + require-mirrors: true + archive-domain: + by-release-level: + production: archive.mozilla.org + staging: ftp.stage.mozaws.net + download-domain: + by-release-level: + production: download.mozilla.org + staging: dev.bouncer.nonprod.webservices.mozgcp.net + whats-new-config: browser/config/whats_new_page.yml + +jobs: + firefox: + name: submit-toplevel-firefox-release-to-balrog + description: Submit toplevel Firefox release to balrog + shipping-product: firefox + worker: + product: firefox + channel-names: + by-release-type: + beta: ["beta", "beta-localtest", "beta-cdntest"] + release(-rc)?: ["release", "release-localtest", "release-cdntest"] + esr.*: ["esr", "esr-localtest", "esr-cdntest", "esr-localtest-next", "esr-cdntest-next"] + default: [] + rules-to-update: + by-release-type: + beta: ["firefox-beta-cdntest", "firefox-beta-localtest"] + release(-rc)?: ["firefox-release-cdntest", "firefox-release-localtest"] + esr115: ["firefox-esr115-cdntest", "firefox-esr115-localtest"] + default: [] + pin-channels: + by-release-type: + beta: ["beta-localtest", "beta-cdntest"] + release(-rc)?: ["release-localtest", "release-cdntest"] + esr.*: ["esr-localtest", "esr-cdntest", "esr-localtest-next", "esr-cdntest-next"] + default: [] + platforms: ["linux", "linux64", "macosx64", "win32", "win64", "win64-aarch64"] + treeherder: + platform: firefox-release/opt + symbol: Rel(BPFx) + tier: 1 + kind: build + + devedition: + name: submit-toplevel-devedition-release-to-balrog + description: submit toplevel Devedition release to balrog + shipping-product: devedition + run-on-releases: [beta] + worker: + product: devedition + channel-names: ["aurora", "aurora-localtest", "aurora-cdntest"] + rules-to-update: ["devedition-cdntest", "devedition-localtest"] + pin-channels: ["aurora-localtest", "aurora-cdntest"] + platforms: ["linux", "linux64", "macosx64", "win32", "win64", "win64-aarch64"] + treeherder: + platform: devedition-release/opt + symbol: Rel(BPDev) + tier: 1 + kind: build diff --git a/taskcluster/ci/release-beetmover-push-to-release/kind.yml b/taskcluster/ci/release-beetmover-push-to-release/kind.yml new file mode 100644 index 0000000000..affc0f2c77 --- /dev/null +++ b/taskcluster/ci/release-beetmover-push-to-release/kind.yml @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.beetmover_push_to_release:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-generate-checksums-beetmover + - post-update-verify-dummy + +job-defaults: + run-on-projects: [] + shipping-phase: push + worker: + max-run-time: 2700 + +jobs: + devedition-push-to-release: + name: devedition_push_to_release + product: devedition + shipping-product: devedition + treeherder-platform: devedition-release/opt + + firefox-push-to-release: + name: firefox_push_to_release + product: firefox + shipping-product: firefox + treeherder-platform: firefox-release/opt diff --git a/taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml b/taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml new file mode 100644 index 0000000000..473f711a2a --- /dev/null +++ b/taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_langpack_checksums + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-beetmover-signed-langpacks + +jobs: + release-beetmover-signed-langpacks-checksums: + from-deps: + group-by: single-with-filters + shipping-phase: promote + attributes: + artifact_prefix: public + artifact_map: taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml diff --git a/taskcluster/ci/release-beetmover-signed-langpacks/kind.yml b/taskcluster/ci/release-beetmover-signed-langpacks/kind.yml new file mode 100644 index 0000000000..63bdcfe921 --- /dev/null +++ b/taskcluster/ci/release-beetmover-signed-langpacks/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.release_beetmover_signed_addons + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build-signing + - shippable-l10n-signing + +only-for-attributes: + - nightly + - shippable + +only-for-build-platforms: + - linux64-shippable/opt + - linux64-devedition/opt + +jobs: + release-beetmover-signed-langpacks: + description: Beetmover submission for platform-independent langpacks {locales} in {platform} directory + from-deps: + group-by: single-with-filters + worker-type: + by-release-level: + production: scriptworker-k8s/gecko-3-beetmover + staging: scriptworker-k8s/gecko-1-beetmover + run-on-projects: [] + shipping-phase: promote + attributes: + artifact_map: + by-release-type: + beta|release.*|esr.*: taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml + default: taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml diff --git a/taskcluster/ci/release-beetmover-source-checksums/kind.yml b/taskcluster/ci/release-beetmover-source-checksums/kind.yml new file mode 100644 index 0000000000..d7a5bf828f --- /dev/null +++ b/taskcluster/ci/release-beetmover-source-checksums/kind.yml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_source_checksums + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-source-checksums-signing + +jobs: + release-beetmover-source-checksums: + from-deps: + group-by: single-with-filters + shipping-phase: promote + attributes: + artifact_prefix: public + artifact_map: taskcluster/gecko_taskgraph/manifests/source_checksums.yml diff --git a/taskcluster/ci/release-bouncer-aliases/kind.yml b/taskcluster/ci/release-bouncer-aliases/kind.yml new file mode 100644 index 0000000000..d1ce834c48 --- /dev/null +++ b/taskcluster/ci/release-bouncer-aliases/kind.yml @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.bouncer_aliases:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-bouncer-check + +job-defaults: + description: Update bouncer aliases job + worker-type: bouncer + worker: + implementation: bouncer-aliases + scopes: + by-release-level: + production: + - project:releng:bouncer:action:aliases + - project:releng:bouncer:server:production + staging: + - project:releng:bouncer:action:aliases + - project:releng:bouncer:server:staging + run-on-projects: [] + shipping-phase: ship + treeherder: + symbol: Rel(BncAlias) + kind: test + tier: 1 + +jobs: + devedition: + bouncer-products-per-alias: + firefox-devedition-latest-ssl: installer-ssl + firefox-devedition-latest: installer + firefox-devedition-stub: stub-installer + firefox-devedition-msi-latest-ssl: msi + firefox-devedition-msix-latest-ssl: msix + shipping-product: devedition + treeherder: + platform: devedition-release/opt + + firefox: + bouncer-products-per-alias: + by-release-type: + beta: + firefox-beta-latest-ssl: installer-ssl + firefox-beta-latest: installer + firefox-beta-stub: stub-installer + firefox-beta-msi-latest-ssl: msi + firefox-beta-msix-latest-ssl: msix + firefox-beta-pkg-latest-ssl: pkg + firefox-beta-langpack-latest-ssl: langpack + release.*: + firefox-latest-ssl: installer-ssl + firefox-latest: installer + firefox-stub: stub-installer + firefox-msi-latest-ssl: msi + firefox-msix-latest-ssl: msix + firefox-pkg-latest-ssl: pkg + firefox-langpack-latest-ssl: langpack + esr115: + # when there is one ESR branch both sets of aliases work + # when there are two ESR branches ... + # ... these always point to the older branch + firefox-esr-latest-ssl: installer-ssl + firefox-esr-latest: installer + firefox-esr-msi-latest-ssl: msi + firefox-esr-msix-latest-ssl: msix + firefox-esr-pkg-latest-ssl: pkg + firefox-esr-langpack-latest-ssl: langpack + esrXXX: # XXX - to change when we branch next ESR + firefox-esr-next-latest-ssl: installer-ssl + firefox-esr-next-latest: installer + firefox-esr-next-msi-latest-ssl: msi + firefox-esr-next-msix-latest-ssl: msix + firefox-esr-next-pkg-latest-ssl: pkg + firefox-esr-next-langpack-latest-ssl: langpack + default: {} + partner-bouncer-products-per-alias: + by-release-type: + beta: + partner-firefox-beta-PARTNER-latest: installer + partner-firefox-beta-PARTNER-stub: stub-installer + release.*: + partner-firefox-release-PARTNER-latest: installer + partner-firefox-release-PARTNER-stub: stub-installer + default: {} + shipping-product: firefox + treeherder: + platform: firefox-release/opt diff --git a/taskcluster/ci/release-bouncer-check/kind.yml b/taskcluster/ci/release-bouncer-check/kind.yml new file mode 100644 index 0000000000..7b7133ffd5 --- /dev/null +++ b/taskcluster/ci/release-bouncer-check/kind.yml @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - release-beetmover-push-to-release + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.bouncer_check:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: release-bouncer-check + description: release bouncer check + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: push + worker-type: b-linux-gcp + worker: + max-run-time: 1200 + docker-image: {in-tree: "update-verify"} + run: + sparse-profile: mozharness + extra-config: + by-release-level: + production: + bouncer_prefix: "https://download.mozilla.org/" + staging: + # This is the list on-change locales, excluding ja/ja-JP-mac + # which bouncer_check doesn't support. + locales: ['en-US', 'en-CA', 'he', 'it'] + bouncer_prefix: "https://dev.bouncer.nonprod.webservices.mozgcp.net/" + cdn_urls: + - ftp.stage.mozaws.net + + attributes: + build_platform: linux64 + build_type: opt + treeherder: + symbol: Rel(BncChk) + kind: test + tier: 1 + +jobs: + firefox: + shipping-product: firefox + index: + product: firefox + job-name: firefox-release-bouncer-check + run: + config: + by-release-type: + beta: + - releases/bouncer_firefox_beta.py + release: + - releases/bouncer_firefox_release.py + esr.*: + - releases/bouncer_firefox_esr.py + default: + - releases/bouncer_firefox_beta.py + treeherder: + platform: firefox-release/opt + + devedition: + shipping-product: devedition + index: + product: devedition + job-name: devedition-release-bouncer-check + run: + config: + - releases/bouncer_firefox_devedition.py + treeherder: + platform: devedition-release/opt diff --git a/taskcluster/ci/release-bouncer-sub/kind.yml b/taskcluster/ci/release-bouncer-sub/kind.yml new file mode 100644 index 0000000000..131b41b0a7 --- /dev/null +++ b/taskcluster/ci/release-bouncer-sub/kind.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.bouncer_submission:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: release bouncer submission job + worker-type: bouncer + worker: + implementation: bouncer-submission + scopes: + by-release-level: + production: + - project:releng:bouncer:action:submission + - project:releng:bouncer:server:production + staging: + - project:releng:bouncer:action:submission + - project:releng:bouncer:server:staging + run-on-projects: [] + shipping-phase: promote + locales-file: browser/locales/shipped-locales + treeherder: + symbol: Rel(BncSub) + kind: test + tier: 1 + +jobs: + devedition: + bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64', 'win64-aarch64'] + bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer', 'msi', 'msix'] + shipping-product: devedition + treeherder: + platform: devedition-release/opt + + firefox: + bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64', 'win64-aarch64'] + bouncer-products: + by-release-type: + default: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer', 'msi', 'msix', 'pkg', 'langpack'] + esr.*: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'msi', 'msix', 'pkg', 'langpack'] + shipping-product: firefox + treeherder: + platform: firefox-release/opt + + firefox-rc: + bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64', 'win64-aarch64'] + bouncer-products: ['complete-mar-candidates', 'partial-mar-candidates'] + shipping-product: firefox + run-on-releases: ['release-rc'] + treeherder: + symbol: Rel(BncSub-RC) + platform: firefox-release/opt diff --git a/taskcluster/ci/release-early-tagging/kind.yml b/taskcluster/ci/release-early-tagging/kind.yml new file mode 100644 index 0000000000..9b194fb441 --- /dev/null +++ b/taskcluster/ci/release-early-tagging/kind.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_version_bump:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: Release Promotion version tag for buildN + run-on-projects: [] + shipping-phase: promote + worker-type: tree + worker: + implementation: treescript + tags: ['buildN'] + bump: false + dontbuild: true + push: + by-project: + mozilla-(beta|release|esr.*): true + maple: true + default: false + + +jobs: + firefox: + name: firefox-tag-buildN + shipping-product: firefox + treeherder: + platform: firefox-release/opt + symbol: Rel(TagBuild) + tier: 1 + kind: build + + devedition: + name: devedition-tag-buildN + shipping-product: devedition + treeherder: + platform: devedition-release/opt + symbol: Rel(TagBuild) + tier: 1 + kind: build diff --git a/taskcluster/ci/release-eme-free-repack-beetmover-checksums/kind.yml b/taskcluster/ci/release-eme-free-repack-beetmover-checksums/kind.yml new file mode 100644 index 0000000000..c75a85ac54 --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-beetmover-checksums/kind.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_emefree_checksums + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack-beetmover + +only-for-build-platforms: + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +jobs: + release-eme-free-repack-beetmover-checksums: + from-deps: {} + shipping-phase: promote diff --git a/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml b/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml new file mode 100644 index 0000000000..7a33d0fb69 --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_repackage_partner + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack-repackage-signing + +only-for-build-platforms: + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +jobs: + release-eme-free-repack-beetmover: + from-deps: + group-by: single-with-filters + shipping-phase: promote + partner-path: "{platform}-EME-free/{locale}" diff --git a/taskcluster/ci/release-eme-free-repack-mac-notarization/kind.yml b/taskcluster/ci/release-eme-free-repack-mac-notarization/kind.yml new file mode 100644 index 0000000000..ff04daf61d --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-mac-notarization/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.mac_notarization + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.partner_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack-mac-signing + +only-for-build-platforms: + - macosx64-shippable/opt + +jobs: + release-eme-free-repack-mac-notarization: + from-deps: + group-by: partner-repack-ids + copy-attributes: true + shipping-product: firefox + shipping-phase: promote + max-run-time: 1800 + copy-repack-ids: true diff --git a/taskcluster/ci/release-eme-free-repack-mac-signing/kind.yml b/taskcluster/ci/release-eme-free-repack-mac-signing/kind.yml new file mode 100644 index 0000000000..84d0e17dd1 --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-mac-signing/kind.yml @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.partner_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.hardened_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack + +only-for-build-platforms: + - macosx64-shippable/opt + +jobs: + release-eme-free-repack-mac-signing: + from-deps: + group-by: partner-repack-ids + shipping-product: firefox + shipping-phase: promote + repacks-per-chunk: 5 diff --git a/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml b/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml new file mode 100644 index 0000000000..c369761202 --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing_partner + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack-repackage + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + - macosx64-shippable/opt + +jobs: + release-eme-free-repack-repackage-signing: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/release-eme-free-repack-repackage/kind.yml b/taskcluster/ci/release-eme-free-repack-repackage/kind.yml new file mode 100644 index 0000000000..78bc18fe6d --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack-repackage/kind.yml @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_partner + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-eme-free-repack + - release-eme-free-repack-mac-signing + - release-eme-free-repack-mac-notarization + - toolchain + +only-for-build-platforms: + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +jobs: + release-eme-free-repack-repackage: + from-deps: + group-by: single-with-filters + upstream-mac-kind: + by-build-type: + debug: release-eme-free-repack-mac-signing + default: + by-project: + mozilla-(central|beta|release|esr.*): release-eme-free-repack-mac-notarization + default: release-eme-free-repack-mac-signing + mozharness: + config: + by-build-platform: + macosx64-.*: + - repackage/base.py + - repackage/osx_partner.py + win32-.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_partner.py + win64-(?!aarch64).*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_partner.py + win64-aarch64-.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_partner.py + package-formats: + by-build-platform: + macosx64\b.*: [dmg] + win32\b.*: [installer] + win64\b.*: [installer] diff --git a/taskcluster/ci/release-eme-free-repack/kind.yml b/taskcluster/ci/release-eme-free-repack/kind.yml new file mode 100644 index 0000000000..cc3e85071b --- /dev/null +++ b/taskcluster/ci/release-eme-free-repack/kind.yml @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.partner_repack:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build-signing + - build-mac-signing + - build-mac-notarization + - shippable-l10n-signing + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + +job-defaults: + name: eme-free-repack + description: Release Promotion eme-free repacks + run-on-projects: [] # to make sure this never runs as part of CI + shipping-product: firefox + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "partner-repack" + chain-of-trust: true + max-run-time: 7200 + run: + using: mozharness + config: + - partner_repacks/release_mozilla-release_desktop.py + script: mozharness/scripts/desktop_partner_repacks.py + job-script: taskcluster/scripts/builder/repackage.sh + tooltool-downloads: false + extra: + limit-locales: + by-release-level: + staging: true + production: false + +jobs: + macosx64-shippable: + treeherder: + symbol: EME + platform: macosx64/opt + kind: test + tier: 1 + attributes: + build_platform: macosx64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + win32-shippable: + treeherder: + symbol: EME + platform: win32/opt + kind: test + tier: 1 + attributes: + build_platform: win32-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + win64-shippable: + treeherder: + symbol: EME + platform: win64/opt + kind: test + tier: 1 + attributes: + build_platform: win64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true diff --git a/taskcluster/ci/release-final-verify/kind.yml b/taskcluster/ci/release-final-verify/kind.yml new file mode 100644 index 0000000000..b1319665cd --- /dev/null +++ b/taskcluster/ci/release-final-verify/kind.yml @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - release-bouncer-check + - release-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.final_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: final-verify + run-on-projects: [] # to make sure this never runs as part of CI + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: [1] + + +jobs: + firefox: + description: final verify + shipping-phase: push + shipping-product: firefox + treeherder: + platform: firefox-release/opt + kind: test + symbol: Rel(FV) + tier: 1 + + devedition: + description: final verify + shipping-phase: push + shipping-product: devedition + treeherder: + platform: devedition-release/opt + kind: test + symbol: Rel(FV) + tier: 1 diff --git a/taskcluster/ci/release-flatpak-push/kind.yml b/taskcluster/ci/release-flatpak-push/kind.yml new file mode 100644 index 0000000000..27f5ebf3ec --- /dev/null +++ b/taskcluster/ci/release-flatpak-push/kind.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_flatpak_push:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-flatpak-repackage + +job-defaults: + description: Pushes Flatpaks onto Flathub + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [beta, release, release-rc] + shipping-phase: ship + treeherder: + platform: linux64/opt + kind: build + tier: 2 + worker-type: + by-release-level: + production: scriptworker-k8s/gecko-3-pushflatpak + staging: scriptworker-k8s/gecko-1-pushflatpak + worker: + implementation: push-flatpak + channel: + by-release-type: + beta: beta + release: stable + default: mock + +jobs: + firefox: + shipping-product: firefox + flathub-scope: "flathub:firefox" + treeherder: + symbol: Flatpak(push) diff --git a/taskcluster/ci/release-flatpak-repackage/kind.yml b/taskcluster/ci/release-flatpak-repackage/kind.yml new file mode 100644 index 0000000000..ecc86bf4ab --- /dev/null +++ b/taskcluster/ci/release-flatpak-repackage/kind.yml @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_flatpak_repackage:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + - post-langpack-dummy + +job-defaults: + description: Generates flatpak by reackaging the existing tar.bz2 + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [beta, release, release-rc] + shipping-phase: promote + scopes: [] + treeherder: + platform: linux64-shippable/opt + kind: build + tier: 2 + worker-type: b-linux-gcp + worker: + implementation: docker-worker + os: linux + max-run-time: 7200 + docker-image: {in-tree: firefox-flatpak} + artifacts: + - name: public/build + type: directory + path: /home/worker/artifacts/ + command: + - /bin/bash + - -cx + - ./runme.sh + env: + VERSION: "{release_config[version]}" + BUILD_NUMBER: "{release_config[build_number]}" + CANDIDATES_DIR: + by-release-level: + staging: + https://ftp.stage.mozaws.net/pub/{task[shipping-product]}/candidates + production: + https://archive.mozilla.org/pub/{task[shipping-product]}/candidates + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + L10N_CHANGESETS: "{config_params[head_repository]}/raw-file/{config_params[head_rev]}/browser/locales/l10n-changesets.json" + FLATPAK_BRANCH: + by-project: + mozilla-release: stable + default: beta + chain-of-trust: true + +jobs: + firefox: + shipping-product: firefox + attributes: + build_platform: linux64-shippable + build_type: opt + treeherder: + symbol: Flatpak(r) diff --git a/taskcluster/ci/release-generate-checksums-beetmover/kind.yml b/taskcluster/ci/release-generate-checksums-beetmover/kind.yml new file mode 100644 index 0000000000..ca998ea2f2 --- /dev/null +++ b/taskcluster/ci/release-generate-checksums-beetmover/kind.yml @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.release_generate_checksums_beetmover + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-generate-checksums-signing + +jobs: + release-generate-checksums-beetmover: + from-deps: + group-by: single-with-filters + shipping-phase: promote + attributes: + artifact_map: taskcluster/gecko_taskgraph/manifests/release_checksums.yml diff --git a/taskcluster/ci/release-generate-checksums-signing/kind.yml b/taskcluster/ci/release-generate-checksums-signing/kind.yml new file mode 100644 index 0000000000..a86b4c8dc5 --- /dev/null +++ b/taskcluster/ci/release-generate-checksums-signing/kind.yml @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - release-generate-checksums + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.release_generate_checksums_signing + - gecko_taskgraph.transforms.task + +jobs: + release-generate-checksums-signing: + from-deps: {} + shipping-phase: promote diff --git a/taskcluster/ci/release-generate-checksums/kind.yml b/taskcluster/ci/release-generate-checksums/kind.yml new file mode 100644 index 0000000000..71018c8cef --- /dev/null +++ b/taskcluster/ci/release-generate-checksums/kind.yml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - beetmover-source + - post-beetmover-checksums-dummy + - release-beetmover-signed-langpacks + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_generate_checksums:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: generate-checksums + description: generates checksums + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + max-run-time: 1200 + artifacts: + - name: public/build/SHA256SUMMARY + path: /builds/worker/SHA256SUMMARY + type: file + - name: public/build/SHA256SUMS + path: /builds/worker/SHA256SUMS + type: file + - name: public/build/SHA512SUMMARY + path: /builds/worker/SHA512SUMMARY + type: file + - name: public/build/SHA512SUMS + path: /builds/worker/SHA512SUMS + type: file + run: + using: mozharness + config: [] # See extra-config below + actions: [create-virtualenv, collect-individual-checksums, create-big-checksums, create-summary] + options: + - "version={version}" + - "build-number={build_number}" + script: "mozharness/scripts/release/generate-checksums.py" + treeherder: + symbol: Rel(GenChcks) + kind: test + tier: 1 + +jobs: + firefox: + shipping-product: firefox + attributes: + build_platform: firefox-release + build_type: opt + run: + extra-config: + by-release-level: + production: + stage_product: "firefox" + bucket_name: "moz-fx-productdelivery-pr-38b5-productdelivery" + staging: + stage_product: "firefox" + bucket_name: "moz-fx-productdelivery-no-7d6a-productdelivery" + treeherder: + platform: firefox-release/opt + + devedition: + shipping-product: devedition + attributes: + build_platform: devedition-release + build_type: opt + run: + extra-config: + by-release-level: + production: + stage_product: "devedition" + bucket_name: "moz-fx-productdelivery-pr-38b5-productdelivery" + staging: + stage_product: "devedition" + bucket_name: "moz-fx-productdelivery-no-7d6a-productdelivery" + treeherder: + platform: devedition-release/opt diff --git a/taskcluster/ci/release-mark-as-shipped/kind.yml b/taskcluster/ci/release-mark-as-shipped/kind.yml new file mode 100644 index 0000000000..c9f6c82973 --- /dev/null +++ b/taskcluster/ci/release-mark-as-shipped/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_mark_as_shipped:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-balrog-scheduling + - release-bouncer-aliases + - release-version-bump + +job-defaults: + description: mark release as shipped in Ship-It + worker-type: shipit + worker: + implementation: shipit-shipped + scopes: + by-release-level: + production: + - project:releng:ship-it:server:production + - project:releng:ship-it:action:mark-as-shipped + staging: + - project:releng:ship-it:server:staging + - project:releng:ship-it:action:mark-as-shipped + run-on-projects: [] + shipping-phase: ship + +jobs: + firefox: + name: release-firefox_mark_as_shipped + shipping-product: firefox + + devedition: + name: release-devedition_mark_as_shipped + shipping-product: devedition diff --git a/taskcluster/ci/release-msix-push/kind.yml b/taskcluster/ci/release-msix-push/kind.yml new file mode 100644 index 0000000000..3009c74df2 --- /dev/null +++ b/taskcluster/ci/release-msix-push/kind.yml @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_msix_push:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-shippable-l10n-msix + +job-defaults: + description: Pushes msix archives to Microsoft Store + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [beta, release, release-rc] + shipping-phase: ship # note override in transform + treeherder: + platform: win32-shippable/opt + kind: build + tier: 2 + worker-type: + by-release-level: + production: scriptworker-k8s/gecko-3-pushmsix + staging: scriptworker-k8s/gecko-1-pushmsix + worker: + implementation: push-msix + channel: + by-release-type: + beta: beta + release.*: release + default: mock + publish-mode: + by-release-type: + beta: Immediate + default: Manual + +jobs: + firefox: + shipping-product: firefox + treeherder: + symbol: MSIX(push) diff --git a/taskcluster/ci/release-notify-av-announce/kind.yml b/taskcluster/ci/release-notify-av-announce/kind.yml new file mode 100644 index 0000000000..45a0e5db53 --- /dev/null +++ b/taskcluster/ci/release-notify-av-announce/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + +job-defaults: + name: notify-av-announce + description: Sends email to antivirus vendors telling them that the release is available for scanning. + run-on-projects: [] + shipping-phase: push + worker-type: succeed + notifications: + # !t is a custom format option that calls .title() on the string. + subject: "{task[shipping-product]!t} {release_config[version]} build{release_config[build_number]} release" + message: | + Company Name: Mozilla + Contact Department: Release Engineering + Contact email address: release@mozilla.com + Product information: {task[shipping-product]} {release_config[version]} + Product location: https://releases.mozilla.org/pub/{task[shipping-product]}/candidates/{release_config[version]}-candidates/build{release_config[build_number]}/ + emails: + by-project: + mozilla-(beta|release|esr.*): ["av-vendor-release-announce@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox: + shipping-product: firefox + devedition: + shipping-product: devedition diff --git a/taskcluster/ci/release-notify-promote/kind.yml b/taskcluster/ci/release-notify-promote/kind.yml new file mode 100644 index 0000000000..9fcc8a3262 --- /dev/null +++ b/taskcluster/ci/release-notify-promote/kind.yml @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + - release-generate-checksums-beetmover + - release-bouncer-sub + +job-defaults: + name: notify-release-drivers-promote + description: Sends email to release-drivers telling release was promoted. + run-on-projects: [] + shipping-phase: promote + worker-type: succeed + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory" + emails: + by-project: + mozilla-(beta|release|esr.*): ["release-signoff@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox: + shipping-product: firefox + devedition: + shipping-product: devedition diff --git a/taskcluster/ci/release-notify-push/kind.yml b/taskcluster/ci/release-notify-push/kind.yml new file mode 100644 index 0000000000..1b641d383e --- /dev/null +++ b/taskcluster/ci/release-notify-push/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-beetmover-push-to-release + +job-defaults: + name: notify-release-drivers-push + description: Sends email to release-drivers telling release was pushed. + run-on-projects: [] + shipping-phase: push + worker-type: succeed + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest" + emails: + by-project: + mozilla-(beta|release|esr.*): ["release-signoff@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox: + shipping-product: firefox + devedition: + shipping-product: devedition diff --git a/taskcluster/ci/release-notify-ship/kind.yml b/taskcluster/ci/release-notify-ship/kind.yml new file mode 100644 index 0000000000..6f0c0f7627 --- /dev/null +++ b/taskcluster/ci/release-notify-ship/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-balrog-scheduling + - release-bouncer-aliases + - release-version-bump + - release-mark-as-shipped + +job-defaults: + name: notify-release-drivers-ship + description: Sends email to release-drivers telling release was shipped. + run-on-projects: [] + shipping-phase: ship + worker-type: succeed + notifications: + emails: + by-project: + mozilla-(beta|release|esr.*): ["release-signoff@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox: + shipping-product: firefox + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" + devedition: + shipping-product: devedition + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" diff --git a/taskcluster/ci/release-notify-started/kind.yml b/taskcluster/ci/release-notify-started/kind.yml new file mode 100644 index 0000000000..004665789d --- /dev/null +++ b/taskcluster/ci/release-notify-started/kind.yml @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_started:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: notify-release-drivers-started + description: Sends email to release-drivers telling release was started. + run-on-projects: [] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: "debian12-base"} + max-run-time: 600 + emails: + by-project: + mozilla-(beta|release|esr.*): ["release-signoff@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox: + shipping-product: firefox + devedition: + shipping-product: devedition diff --git a/taskcluster/ci/release-partner-attribution-beetmover/kind.yml b/taskcluster/ci/release-partner-attribution-beetmover/kind.yml new file mode 100644 index 0000000000..2851ff7d37 --- /dev/null +++ b/taskcluster/ci/release-partner-attribution-beetmover/kind.yml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.partner_attribution_beetmover + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-attribution + +jobs: + release-partner-attribution-beetmover: + from-deps: + group-by: single-with-filters + shipping-product: firefox + shipping-phase: promote + partner-path: "partner-repacks/{partner}/{subpartner}/v{release_partner_build_number}/{platform}/{locale}" diff --git a/taskcluster/ci/release-partner-attribution/kind.yml b/taskcluster/ci/release-partner-attribution/kind.yml new file mode 100644 index 0000000000..1da2656150 --- /dev/null +++ b/taskcluster/ci/release-partner-attribution/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.partner_attribution:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-signing + - repackage-signing-l10n + +# move this into the single job ?? +job-defaults: + name: partner-attribution + description: Release Promotion partner attribution + run-on-projects: [] # to make sure this never runs as part of CI + shipping-product: firefox + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "partner-repack" + chain-of-trust: true + max-run-time: 1800 + run: + using: mach + mach: python python/mozrelease/mozrelease/attribute_builds.py + +jobs: + partner-attribution: + attributes: + build_platform: linux-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true diff --git a/taskcluster/ci/release-partner-repack-beetmover/kind.yml b/taskcluster/ci/release-partner-repack-beetmover/kind.yml new file mode 100644 index 0000000000..164446f683 --- /dev/null +++ b/taskcluster/ci/release-partner-repack-beetmover/kind.yml @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.beetmover_repackage_partner + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack-repackage-signing + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + +jobs: + release-partner-repack-beetmover: + from-deps: + group-by: single-with-filters + shipping-phase: promote + partner-path: "partner-repacks/{partner}/{subpartner}/v{release_partner_build_number}/{platform}/{locale}" diff --git a/taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml b/taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml new file mode 100644 index 0000000000..98fa149810 --- /dev/null +++ b/taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.bouncer_submission_partners:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: release partners bouncer submission job + worker-type: bouncer + worker: + implementation: bouncer-submission + scopes: + by-release-level: + production: + - project:releng:bouncer:action:submission + - project:releng:bouncer:server:production + staging: + - project:releng:bouncer:action:submission + - project:releng:bouncer:server:staging + run-on-projects: [] + shipping-phase: promote + locales-file: browser/locales/shipped-locales + treeherder: + symbol: Rel(BncSub-P) + kind: test + tier: 1 + +jobs: + firefox: + bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64', 'win64-aarch64'] + bouncer-products: + by-release-type: + default: ['installer', 'stub-installer'] + esr.*: [] + shipping-product: firefox + treeherder: + platform: firefox-release/opt diff --git a/taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml b/taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml new file mode 100644 index 0000000000..4975931312 --- /dev/null +++ b/taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + +jobs: + release-partner-repack-chunking-dummy: + from-deps: + group-by: single-with-filters + copy-attributes: true + shipping-phase: promote + shipping-product: firefox + name: release-partner-repack-chunking-dummy + description: Dummy task to deal with fanning out Linux partner repacks + run-on-projects: [] + worker-type: succeed diff --git a/taskcluster/ci/release-partner-repack-mac-notarization/kind.yml b/taskcluster/ci/release-partner-repack-mac-notarization/kind.yml new file mode 100644 index 0000000000..ee7a78e54f --- /dev/null +++ b/taskcluster/ci/release-partner-repack-mac-notarization/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.mac_notarization + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.partner_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack-mac-signing + +only-for-build-platforms: + - macosx64-shippable/opt + +jobs: + release-partner-repack-mac-notarization: + from-deps: + group-by: partner-repack-ids + copy-attributes: true + shipping-product: firefox + shipping-phase: promote + max-run-time: 1800 + copy-repack-ids: true diff --git a/taskcluster/ci/release-partner-repack-mac-signing/kind.yml b/taskcluster/ci/release-partner-repack-mac-signing/kind.yml new file mode 100644 index 0000000000..dec5189979 --- /dev/null +++ b/taskcluster/ci/release-partner-repack-mac-signing/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.partner_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.hardened_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack + +only-for-build-platforms: + - macosx64-shippable/opt + +jobs: + release-partner-repack-mac-signing: + from-deps: + group-by: partner-repack-ids + copy-attributes: true + shipping-product: firefox + shipping-phase: promote + repacks-per-chunk: 5 diff --git a/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml b/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml new file mode 100644 index 0000000000..2242023f91 --- /dev/null +++ b/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing_partner + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack-chunking-dummy # Linux + - release-partner-repack-repackage # Windows, Mac + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + +jobs: + release-partner-repack-repackage-signing: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/release-partner-repack-repackage/kind.yml b/taskcluster/ci/release-partner-repack-repackage/kind.yml new file mode 100644 index 0000000000..099c55240f --- /dev/null +++ b/taskcluster/ci/release-partner-repack-repackage/kind.yml @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind + - gecko_taskgraph.transforms.chunk_partners + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_partner + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-partner-repack + - release-partner-repack-mac-signing + - release-partner-repack-mac-notarization + - toolchain + +only-for-build-platforms: + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + +jobs: + release-partner-repack-repackage: + from-deps: + group-by: single-with-filters + upstream-mac-kind: + by-build-type: + debug: release-partner-repack-mac-signing + default: + by-project: + mozilla-(central|beta|release|esr.*): release-partner-repack-mac-notarization + default: release-partner-repack-mac-signing + mozharness: + config: + by-build-platform: + macosx64-.*: + - repackage/base.py + - repackage/osx_partner.py + win32-.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_partner.py + win64-(?!aarch64).*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_partner.py + win64-aarch64-.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_partner.py + package-formats: + by-build-platform: + macosx64\b.*: [dmg] + win32\b.*: [installer] + win64\b.*: [installer] diff --git a/taskcluster/ci/release-partner-repack/kind.yml b/taskcluster/ci/release-partner-repack/kind.yml new file mode 100644 index 0000000000..2a5d2fc996 --- /dev/null +++ b/taskcluster/ci/release-partner-repack/kind.yml @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.partner_repack:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build-signing + - build-mac-signing + - build-mac-notarization + - shippable-l10n-signing + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + +job-defaults: + name: partner-repack + description: Release Promotion partner repacks + run-on-projects: [] # to make sure this never runs as part of CI + shipping-product: firefox + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "partner-repack" + chain-of-trust: true + max-run-time: 7200 + run: + using: mozharness + config: + - partner_repacks/release_mozilla-release_desktop.py + script: mozharness/scripts/desktop_partner_repacks.py + job-script: taskcluster/scripts/builder/repackage.sh + tooltool-downloads: false + extra: + limit-locales: + by-release-level: + staging: true + production: false + +jobs: + linux-shippable: + attributes: + build_platform: linux-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + linux64-shippable: + attributes: + build_platform: linux64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + macosx64-shippable: + attributes: + build_platform: macosx64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + win32-shippable: + attributes: + build_platform: win32-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + win64-shippable: + attributes: + build_platform: win64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true + + win64-aarch64-shippable: + attributes: + build_platform: win64-aarch64-shippable + build_type: opt + artifact_prefix: releng/partner + shippable: true diff --git a/taskcluster/ci/release-push-langpacks/kind.yml b/taskcluster/ci/release-push-langpacks/kind.yml new file mode 100644 index 0000000000..fbc5bcc8a9 --- /dev/null +++ b/taskcluster/ci/release-push-langpacks/kind.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.release_sign_and_push_langpacks + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + - shippable-l10n + +only-for-build-platforms: + - linux64-shippable/opt # addons.mozilla.org only support 1 platform per locale. That's why we use linux64 + - linux64-devedition/opt + - macosx64-shippable/opt # Although, we need the special locale "ja-JP-Mac" from this platform + - macosx64-devedition/opt + +jobs: + release-push-langpacks: + description: Sends {locales} XPIs for platform to addons.mozilla.org + from-deps: + group-by: single-with-filters + worker-type: addon + worker: + channel: + by-project: + mozilla-release: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64.*: unlisted + default: listed + mozilla-beta: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64-shippable.*: unlisted + macosx64-devedition.*: unlisted + linux.*-devedition.*: unlisted + default: listed + default: unlisted + upstream-artifacts: # See transforms + run-on-projects: [] + scopes: + by-release-level: + production: + - project:releng:addons.mozilla.org:server:production + staging: + - project:releng:addons.mozilla.org:server:staging + shipping-phase: promote diff --git a/taskcluster/ci/release-secondary-balrog-scheduling/kind.yml b/taskcluster/ci/release-secondary-balrog-scheduling/kind.yml new file mode 100644 index 0000000000..9a230639b9 --- /dev/null +++ b/taskcluster/ci/release-secondary-balrog-scheduling/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-secondary-balrog-submit-toplevel + +job-defaults: + run-on-projects: [] + shipping-phase: ship + worker-type: balrog + worker: + implementation: balrog + balrog-action: schedule + +jobs: + firefox-secondary-rc: + description: Schedule Beta Firefox publishing in balrog + name: release-firefox_schedule_publishing_in_balrog + shipping-product: firefox + worker: + product: firefox + release-eta: '' + channel-names: ["beta", "beta-localtest", "beta-cdntest"] + publish-rules: [32] + rules-to-update: ["firefox-beta-cdntest", "firefox-beta-localtest"] + pin-channels: ["beta"] + treeherder: + platform: firefox-release/opt + symbol: Rel(BSFxRC) + tier: 1 + kind: build diff --git a/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml b/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml new file mode 100644 index 0000000000..d742b4fb61 --- /dev/null +++ b/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.balrog_toplevel:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + run-on-projects: [] # to make sure this never runs in CI + shipping-phase: promote + worker-type: balrog + worker: + implementation: balrog + balrog-action: v2-submit-toplevel + require-mirrors: false + platforms: ["linux", "linux64", "macosx64", "win32", "win64", "win64-aarch64"] + archive-domain: + by-release-level: + production: archive.mozilla.org + staging: ftp.stage.mozaws.net + download-domain: + by-release-level: + production: download.mozilla.org + staging: dev.bouncer.nonprod.webservices.mozgcp.net + whats-new-config: browser/config/whats_new_page.yml + +jobs: + firefox: + name: submit-toplevel-rc-firefox-release-to-balrog + description: submit toplevel RC Firefox release to balrog + shipping-product: firefox + worker: + product: firefox + channel-names: ["beta", "beta-localtest", "beta-cdntest"] + rules-to-update: ["firefox-beta-cdntest", "firefox-beta-localtest"] + pin-channels: ["beta-localtest", "beta-cdntest"] + treeherder: + platform: firefox-release/opt + symbol: Rel(BPFxRC) + tier: 1 + kind: build diff --git a/taskcluster/ci/release-secondary-final-verify/kind.yml b/taskcluster/ci/release-secondary-final-verify/kind.yml new file mode 100644 index 0000000000..0d2e58d01d --- /dev/null +++ b/taskcluster/ci/release-secondary-final-verify/kind.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-secondary-balrog-submit-toplevel + - release-secondary-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.final_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: secondary-final-verify + run-on-projects: [] # to make sure this never runs as part of CI + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: [1] + +jobs: + firefox: + description: secondary final verify + shipping-phase: promote + shipping-product: firefox + treeherder: + platform: firefox-release/opt + kind: test + symbol: Rel(FVS) + tier: 1 diff --git a/taskcluster/ci/release-secondary-flatpak-push/kind.yml b/taskcluster/ci/release-secondary-flatpak-push/kind.yml new file mode 100644 index 0000000000..7f770d4ee5 --- /dev/null +++ b/taskcluster/ci/release-secondary-flatpak-push/kind.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_flatpak_push:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-flatpak-repackage + +job-defaults: + description: Pushes Flatpaks onto Flathub + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [release-rc] + shipping-phase: ship # ship-rc phase + treeherder: + platform: linux64/opt + kind: build + tier: 2 + worker-type: + by-release-level: + production: scriptworker-k8s/gecko-3-pushflatpak + staging: scriptworker-k8s/gecko-1-pushflatpak + worker: + implementation: push-flatpak + channel: beta + +jobs: + firefox-rc: + shipping-product: firefox + flathub-scope: "flathub:firefox" + treeherder: + symbol: Flatpak(push-beta) diff --git a/taskcluster/ci/release-secondary-notify-ship/kind.yml b/taskcluster/ci/release-secondary-notify-ship/kind.yml new file mode 100644 index 0000000000..ab9b20ed62 --- /dev/null +++ b/taskcluster/ci/release-secondary-notify-ship/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-secondary-balrog-scheduling + +job-defaults: + name: notify-release-drivers-ship + description: Sends email to release-drivers telling release was shipped. + run-on-projects: [] + shipping-phase: ship + worker-type: succeed + notifications: + emails: + by-project: + mozilla-beta: ["release-signoff@mozilla.org"] + mozilla-release: ["release-signoff@mozilla.org"] + try: ["{config[params][owner]}"] + default: [] + +jobs: + firefox-rc: + shipping-product: firefox + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} RC updates are ready!" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} RC updates are ready!" diff --git a/taskcluster/ci/release-secondary-update-verify-config/kind.yml b/taskcluster/ci/release-secondary-update-verify-config/kind.yml new file mode 100644 index 0000000000..7e28131a5e --- /dev/null +++ b/taskcluster/ci/release-secondary-update-verify-config/kind.yml @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.update_verify_config:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: secondary-update-verify-config + run-on-projects: [] # to make sure this never runs as part of CI + shipping-product: firefox + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 3600 + artifacts: + - name: public/build/update-verify.cfg + path: /builds/worker/checkouts/gecko/update-verify.cfg + type: file + run: + sparse-profile: mozharness + extra: + app-name: browser + branch-prefix: mozilla + product: firefox + archive-prefix: + by-release-level: + staging: "http://ftp.stage.mozaws.net/pub" + production: "https://archive.mozilla.org/pub" + previous-archive-prefix: + by-release-level: + staging: "https://archive.mozilla.org/pub" + production: null + aus-server: + by-release-level: + staging: "https://stage.balrog.nonprod.cloudops.mozgcp.net" + production: "https://aus5.mozilla.org" + override-certs: + by-release-level: + staging: dep + production: null + include-version: beta + last-watershed: "72.0.2" + mar-channel-id-override: beta + channel: "beta-localtest" + +jobs: + firefox-secondary-linux: + treeherder: + symbol: UVCS + platform: linux-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux-shippable + extra: + platform: linux-i686 + updater-platform: linux-x86_64 + + firefox-secondary-linux64: + treeherder: + symbol: UVCS + platform: linux64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux64-shippable + extra: + platform: linux-x86_64 + updater-platform: linux-x86_64 + + firefox-secondary-macosx64: + treeherder: + symbol: UVCS + platform: macosx64/opt + kind: test + tier: 1 + attributes: + build_platform: macosx64-shippable + extra: + platform: mac + updater-platform: linux-x86_64 + + firefox-secondary-win32: + treeherder: + symbol: UVCS + platform: windows2012-32-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win32-shippable + extra: + platform: win32 + updater-platform: linux-x86_64 + + firefox-secondary-win64: + treeherder: + symbol: UVCS + platform: windows2012-64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win64-shippable + extra: + platform: win64 + updater-platform: linux-x86_64 + + firefox-secondary-win64-aarch64: + treeherder: + symbol: UVCS + platform: windows2012-aarch64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win64-aarch64-shippable + extra: + platform: win64-aarch64 + updater-platform: linux-x86_64 diff --git a/taskcluster/ci/release-secondary-update-verify/kind.yml b/taskcluster/ci/release-secondary-update-verify/kind.yml new file mode 100644 index 0000000000..f567cf7ae3 --- /dev/null +++ b/taskcluster/ci/release-secondary-update-verify/kind.yml @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-secondary-balrog-submit-toplevel + - release-secondary-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.update_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: secondary-update-verify + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + artifacts: + - name: 'public/build/diff-summary.log' + path: '/builds/worker/tools/release/updates/diff-summary.log' + type: file + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: + - 255 + treeherder: + symbol: UV(UVS) + kind: test + extra: + chunks: 16 + +jobs: + firefox-secondary-linux64: + description: linux64 secondary channel update verify + shipping-product: firefox + attributes: + build_platform: linux64-shippable + + firefox-secondary-linux: + description: linux secondary channel update verify + shipping-product: firefox + attributes: + build_platform: linux-shippable + + firefox-secondary-win64: + description: win64 secondary channel update verify + shipping-product: firefox + attributes: + build_platform: win64-shippable + + firefox-secondary-win64-aarch64: + description: win64 secondary channel update verify + shipping-product: firefox + attributes: + build_platform: win64-aarch64-shippable + + firefox-secondary-win32: + description: win32 secondary channel update verify + shipping-product: firefox + attributes: + build_platform: win32-shippable + + firefox-secondary-macosx64: + description: macosx64 secondary channel update verify + shipping-product: firefox + attributes: + build_platform: macosx64-shippable + extra: + chunks: 30 diff --git a/taskcluster/ci/release-snap-repackage/kind.yml b/taskcluster/ci/release-snap-repackage/kind.yml new file mode 100644 index 0000000000..73d9d8a410 --- /dev/null +++ b/taskcluster/ci/release-snap-repackage/kind.yml @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_snap_repackage:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + - post-langpack-dummy + +job-defaults: + description: Generates snap image + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + scopes: [] + treeherder: + platform: linux64-shippable/opt + kind: build + tier: 2 + worker-type: b-linux-gcp + worker: + implementation: docker-worker + os: linux + max-run-time: 7200 + docker-image: {in-tree: firefox-snap} + artifacts: + - name: public/build + type: directory + path: /home/worker/artifacts/ + command: + - /bin/bash + - -cx + - ./runme.sh + env: + VERSION: "{release_config[version]}" + BUILD_NUMBER: "{release_config[build_number]}" + CANDIDATES_DIR: + by-release-level: + staging: + https://ftp.stage.mozaws.net/pub/{task[shipping-product]}/candidates + production: + https://archive.mozilla.org/pub/{task[shipping-product]}/candidates + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + L10N_CHANGESETS: "{config_params[head_repository]}/raw-file/{config_params[head_rev]}/browser/locales/l10n-changesets.json" + chain-of-trust: true + +jobs: + firefox: + shipping-product: firefox + attributes: + build_platform: linux64-shippable + build_type: opt + treeherder: + symbol: Snap(r) diff --git a/taskcluster/ci/release-source-checksums-signing/kind.yml b/taskcluster/ci/release-source-checksums-signing/kind.yml new file mode 100644 index 0000000000..de9f98acc0 --- /dev/null +++ b/taskcluster/ci/release-source-checksums-signing/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.source_checksums_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - beetmover-source + +jobs: + release-source-checksums-signing: + shipping-phase: promote + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/release-source-signing/kind.yml b/taskcluster/ci/release-source-signing/kind.yml new file mode 100644 index 0000000000..9ab9e37d88 --- /dev/null +++ b/taskcluster/ci/release-source-signing/kind.yml @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.build_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - release-source + +jobs: + release-source-signing: + from-deps: + copy-attributes: true + shipping-phase: promote diff --git a/taskcluster/ci/release-source/kind.yml b/taskcluster/ci/release-source/kind.yml new file mode 100644 index 0000000000..6b550e9d7e --- /dev/null +++ b/taskcluster/ci/release-source/kind.yml @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + + +job-defaults: + shipping-phase: promote + treeherder: + symbol: Src + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + env: + PKG_SRCPACK_BASENAME: source + attributes: + shippable: true + run: + using: mozharness + actions: [package-source] + script: "mozharness/scripts/fx_desktop_build.py" + config: + - builds/releng_sub_linux_configs/64_source.py + options: [] + tooltool-downloads: public + run-on-projects: [] + # The build_attrs transform defaults to setting a `skip-unlles-schedules' + # optimization based on the task label. The tasks here don't have corresponding + # schedules defined in mozbuild. + optimization: null + +jobs: + firefox-source/opt: + description: "Firefox source builder" + treeherder: + platform: firefox-release/opt + shipping-product: firefox + + devedition-source/opt: + description: "Devedition source builder" + treeherder: + platform: devedition-release/opt + shipping-product: devedition diff --git a/taskcluster/ci/release-update-verify-config-next/kind.yml b/taskcluster/ci/release-update-verify-config-next/kind.yml new file mode 100644 index 0000000000..959d0afb9a --- /dev/null +++ b/taskcluster/ci/release-update-verify-config-next/kind.yml @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.update_verify_config:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-config-next + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [esr115] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 3600 + artifacts: + - name: public/build/update-verify.cfg + path: /builds/worker/checkouts/gecko/update-verify.cfg + type: file + run: + sparse-profile: mozharness + treeherder: + symbol: UVCnext + kind: test + tier: 1 + extra: + app-name: browser + branch-prefix: mozilla + archive-prefix: + by-release-level: + staging: "http://ftp.stage.mozaws.net/pub" + production: "https://archive.mozilla.org/pub" + previous-archive-prefix: + by-release-level: + staging: "https://archive.mozilla.org/pub" + production: null + aus-server: + by-release-level: + staging: "https://stage.balrog.nonprod.cloudops.mozgcp.net" + production: "https://aus5.mozilla.org" + override-certs: + by-release-level: + staging: dep + production: null + updater-platform: linux-x86_64 + product: firefox + channel: "esr-localtest-next" + include-version: esr115-next + last-watershed: "68.0esr" + +jobs: + firefox-next-linux: + shipping-product: firefox + treeherder: + platform: linux32-shippable/opt + attributes: + build_platform: linux-shippable + extra: + platform: linux-i686 + firefox-next-linux64: + shipping-product: firefox + treeherder: + platform: linux64-shippable/opt + attributes: + build_platform: linux64-shippable + extra: + platform: linux-x86_64 + firefox-next-macosx64: + shipping-product: firefox + treeherder: + platform: osx-shippable/opt + attributes: + build_platform: macosx64-shippable + extra: + platform: mac + firefox-next-win32: + shipping-product: firefox + treeherder: + platform: windows2012-32-shippable/opt + attributes: + build_platform: win32-shippable + extra: + platform: win32 + firefox-next-win64: + shipping-product: firefox + treeherder: + platform: windows2012-64-shippable/opt + attributes: + build_platform: win64-shippable + extra: + platform: win64 diff --git a/taskcluster/ci/release-update-verify-config/kind.yml b/taskcluster/ci/release-update-verify-config/kind.yml new file mode 100644 index 0000000000..d4e40cda12 --- /dev/null +++ b/taskcluster/ci/release-update-verify-config/kind.yml @@ -0,0 +1,292 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.update_verify_config:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-config + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 3600 + artifacts: + - name: public/build/update-verify.cfg + path: /builds/worker/checkouts/gecko/update-verify.cfg + type: file + run: + sparse-profile: mozharness + extra: + app-name: browser + branch-prefix: mozilla + archive-prefix: + by-release-level: + staging: "http://ftp.stage.mozaws.net/pub" + production: "https://archive.mozilla.org/pub" + previous-archive-prefix: + by-release-level: + staging: "https://archive.mozilla.org/pub" + production: null + aus-server: + by-release-level: + staging: "https://stage.balrog.nonprod.cloudops.mozgcp.net" + production: "https://aus5.mozilla.org" + override-certs: + by-release-level: + staging: dep + production: null + include-version: + by-release-type: + beta: beta + release(-rc)?: nonbeta + esr.*: esr + default: beta + last-watershed: + by-release-type: + beta: "72.0" + release(-rc)?: "72.0.2" + esr115: "68.0esr" + default: "default" + +jobs: + firefox-linux: + shipping-product: firefox + treeherder: + symbol: UVC + platform: linux32-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux-shippable + extra: + product: firefox + platform: linux-i686 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + firefox-linux64: + shipping-product: firefox + treeherder: + symbol: UVC + platform: linux64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux64-shippable + extra: + product: firefox + platform: linux-x86_64 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + firefox-macosx64: + shipping-product: firefox + treeherder: + symbol: UVC + platform: osx-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: macosx64-shippable + extra: + product: firefox + platform: mac + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + firefox-win32: + shipping-product: firefox + treeherder: + symbol: UVC + platform: windows2012-32-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win32-shippable + extra: + product: firefox + platform: win32 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + firefox-win64: + shipping-product: firefox + treeherder: + symbol: UVC + platform: windows2012-64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win64-shippable + extra: + product: firefox + platform: win64 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + firefox-win64-aarch64: + shipping-product: firefox + treeherder: + symbol: UVC + platform: windows2012-aarch64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win64-aarch64-shippable + extra: + product: firefox + platform: win64-aarch64 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release(-rc)?: "release-localtest" + esr.*: "esr-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + devedition-linux: + shipping-product: devedition + treeherder: + symbol: UVC + platform: linux32-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: linux-devedition + extra: + product: firefox + platform: linux-i686 + updater-platform: linux-x86_64 + channel: "aurora-localtest" + + devedition-linux64: + shipping-product: devedition + treeherder: + symbol: UVC + platform: linux64-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: linux64-devedition + extra: + product: firefox + platform: linux-x86_64 + updater-platform: linux-x86_64 + channel: "aurora-localtest" + + devedition-macosx64: + shipping-product: devedition + treeherder: + symbol: UVC + platform: osx-cross-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: macosx64-devedition + extra: + product: firefox + platform: mac + updater-platform: linux-x86_64 + channel: "aurora-localtest" + + devedition-win32: + shipping-product: devedition + treeherder: + symbol: UVC + platform: windows2012-32-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: win32-devedition + extra: + product: firefox + platform: win32 + updater-platform: linux-x86_64 + channel: "aurora-localtest" + + devedition-win64: + shipping-product: devedition + treeherder: + symbol: UVC + platform: windows2012-64-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: win64-devedition + extra: + product: firefox + platform: win64 + updater-platform: linux-x86_64 + channel: "aurora-localtest" + + devedition-win64-aarch64: + shipping-product: devedition + treeherder: + symbol: UVC + platform: windows2012-aarch64-devedition/opt + kind: test + tier: 1 + attributes: + build_platform: win64-aarch64-devedition + extra: + product: firefox + platform: win64-aarch64 + updater-platform: linux-x86_64 + channel: "aurora-localtest" diff --git a/taskcluster/ci/release-update-verify-next/kind.yml b/taskcluster/ci/release-update-verify-next/kind.yml new file mode 100644 index 0000000000..ad72e9b118 --- /dev/null +++ b/taskcluster/ci/release-update-verify-next/kind.yml @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-balrog-submit-toplevel + - release-update-verify-config-next + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.update_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-next + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [esr115] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + artifacts: + - name: 'public/build/diff-summary.log' + path: '/builds/worker/tools/release/updates/diff-summary.log' + type: file + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: + - 255 + env: + CHANNEL: "esr-localtest-next" + treeherder: + symbol: UVnext() + kind: test + extra: + chunks: 12 + +jobs: + firefox-next-linux64: + description: linux64 esr-next update verify + shipping-product: firefox + attributes: + build_platform: linux64-shippable + + firefox-next-linux: + description: linux esr-next update verify + shipping-product: firefox + attributes: + build_platform: linux-shippable + + firefox-next-win64: + description: win64 esr-next update verify + shipping-product: firefox + attributes: + build_platform: win64-shippable + + firefox-next-win32: + description: win32 esr-next update verify + shipping-product: firefox + attributes: + build_platform: win32-shippable + + firefox-next-macosx64: + description: macosx64 esr-next update verify + shipping-product: firefox + attributes: + build_platform: macosx64-shippable diff --git a/taskcluster/ci/release-update-verify/kind.yml b/taskcluster/ci/release-update-verify/kind.yml new file mode 100644 index 0000000000..ec7f9874da --- /dev/null +++ b/taskcluster/ci/release-update-verify/kind.yml @@ -0,0 +1,115 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-balrog-submit-toplevel + - release-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.update_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + artifacts: + - name: 'public/build/diff-summary.log' + path: '/builds/worker/checkouts/gecko/diff-summary.log' + type: file + docker-image: + in-tree: "update-verify" + max-run-time: 5400 + retry-exit-status: + - 255 + treeherder: + symbol: UV() + kind: test + extra: + chunks: 16 + +jobs: + firefox-linux64: + description: linux64 update verify + shipping-product: firefox + attributes: + build_platform: linux64-shippable + + firefox-linux: + description: linux update verify + shipping-product: firefox + attributes: + build_platform: linux-shippable + + firefox-win64: + description: win64 update verify + shipping-product: firefox + attributes: + build_platform: win64-shippable + + firefox-win64-aarch64: + description: win64-aarch64 update verify + shipping-product: firefox + attributes: + build_platform: win64-aarch64-shippable + + firefox-win32: + description: win32 update verify + shipping-product: firefox + attributes: + build_platform: win32-shippable + + firefox-macosx64: + description: macosx64 update verify + shipping-product: firefox + attributes: + build_platform: macosx64-shippable + extra: + chunks: 30 + + devedition-linux64: + description: linux64 update verify + shipping-product: devedition + attributes: + build_platform: linux64-devedition + + devedition-linux: + description: linux update verify + shipping-product: devedition + attributes: + build_platform: linux-devedition + + devedition-win64: + description: win64 update verify + shipping-product: devedition + attributes: + build_platform: win64-devedition + + devedition-win64-aarch64: + description: win64-aarch64 update verify + shipping-product: devedition + attributes: + build_platform: win64-aarch64-devedition + + devedition-win32: + description: win32 update verify + shipping-product: devedition + attributes: + build_platform: win32-devedition + + devedition-macosx64: + description: macosx64 update verify + shipping-product: devedition + attributes: + build_platform: macosx64-devedition + extra: + chunks: 30 diff --git a/taskcluster/ci/release-version-bump/kind.yml b/taskcluster/ci/release-version-bump/kind.yml new file mode 100644 index 0000000000..be1d9cd85f --- /dev/null +++ b/taskcluster/ci/release-version-bump/kind.yml @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_version_bump:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-beetmover-push-to-release + +job-defaults: + description: Release Promotion version bump/tag + run-on-projects: [] + shipping-phase: ship + worker-type: tree + worker: + implementation: treescript + dontbuild: true + tags: ['release'] + bump: true + bump-files: + by-project: + default: ["browser/config/version_display.txt"] + mozilla-(release|esr.*): + - "browser/config/version.txt" + - "browser/config/version_display.txt" + - "config/milestone.txt" + maple: + - "browser/config/version.txt" + - "browser/config/version_display.txt" + - "config/milestone.txt" + push: + by-project: + mozilla-(beta|release|esr.*): true + maple: true + default: false + +jobs: + firefox: + name: firefox-version-bump + shipping-product: firefox + treeherder: + platform: firefox-release/opt + symbol: Rel(VerBump) + tier: 1 + kind: build + + devedition: + name: devedition-version-bump + shipping-product: devedition + treeherder: + platform: devedition-release/opt + symbol: Rel(VerBump) + tier: 1 + kind: build diff --git a/taskcluster/ci/repackage-deb-l10n/kind.yml b/taskcluster/ci/repackage-deb-l10n/kind.yml new file mode 100644 index 0000000000..a22e50787a --- /dev/null +++ b/taskcluster/ci/repackage-deb-l10n/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.repackage_l10n + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.repackage_routes + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - shippable-l10n-signing + +only-for-build-platforms: + # Langpacks support all CPU architectures. We just need to pick one + - linux64-devedition/opt + - linux64-shippable/opt + +jobs: + repackage-deb-l10n: + from-deps: + group-by: single-with-filters + worker-type: b-linux-gcp + worker: + docker-image: {"in-tree": "debian12-repackage"} + max-run-time: 600 + mozharness: + run-as-root: true + use-caches: false + config: + by-build-platform: + linux64\b.*: + - repackage/base.py + - repackage/linux64_signed.py + package-formats: + by-build-platform: + linux.*: [deb-l10n] + treeherder-group: deb-L10n diff --git a/taskcluster/ci/repackage-deb/kind.yml b/taskcluster/ci/repackage-deb/kind.yml new file mode 100644 index 0000000000..aa6381bb4a --- /dev/null +++ b/taskcluster/ci/repackage-deb/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.repackage_routes + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build-signing + - toolchain + +only-for-build-platforms: + - linux-shippable/opt + - linux-devedition/opt + - linux64-shippable/opt + - linux64-devedition/opt + +only-for-attributes: + - nightly + - shippable + +jobs: + repackage-deb: + from-deps: + group-by: single-with-filters + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: debian12-repackage + mozharness: + run-as-root: true + use-caches: false + config: + by-build-platform: + linux\b.*: + - repackage/base.py + - repackage/linux32_signed.py + linux64\b.*: + - repackage/base.py + - repackage/linux64_signed.py + treeherder: + symbol: Rpk-deb + package-formats: + by-build-platform: + linux.*: [deb] diff --git a/taskcluster/ci/repackage-l10n/kind.yml b/taskcluster/ci/repackage-l10n/kind.yml new file mode 100644 index 0000000000..c78b52de2b --- /dev/null +++ b/taskcluster/ci/repackage-l10n/kind.yml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind + - gecko_taskgraph.transforms.repackage_l10n + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - shippable-l10n-signing + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + - toolchain + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - linux-devedition/opt + - linux64-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + +jobs: + repackage-l10n: + from-deps: + group-by: single-with-filters + upstream-mac-kind: + by-build-type: + debug: shippable-l10n-mac-signing + default: + by-project: + mozilla-(central|beta|release|esr.*): shippable-l10n-mac-notarization + default: shippable-l10n-mac-signing + worker-type: b-linux-gcp + mozharness: + config: + by-build-platform: + linux-.*: + - repackage/base.py + - repackage/linux32_signed.py + linux64-.*: + - repackage/base.py + - repackage/linux64_signed.py + macosx64-.*: + - repackage/base.py + - repackage/osx_signed.py + win32\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_signed.py + win64(?!-aarch64)\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_signed.py + win64-aarch64\b.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_signed.py + package-formats: + by-build-platform: + macosx64\b.*: + by-build-type: + # Debug builds can't be attributable + debug: [dmg, mar, pkg] + default: [dmg-attrib, mar, pkg] + linux.*: [mar] + win32\b.*: [mar, installer] + win64\b.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx + treeherder-group: L10n-Rpk diff --git a/taskcluster/ci/repackage-msi/kind.yml b/taskcluster/ci/repackage-msi/kind.yml new file mode 100644 index 0000000000..fa2fb3ada3 --- /dev/null +++ b/taskcluster/ci/repackage-msi/kind.yml @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-signing + - repackage-signing-l10n + - fetch + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + - win32-devedition/opt + - win64-devedition/opt + +jobs: + repackage-msi: + worker-type: 'b-win2022' + from-deps: + group-by: single-with-filters + mozharness: + use-magic-mh-args: false + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_signed.py + win64(?!-aarch64)\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_signed.py + win64-aarch64\b.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msi] + fetches: + fetch: + - wix-3.14.0 diff --git a/taskcluster/ci/repackage-msix/kind.yml b/taskcluster/ci/repackage-msix/kind.yml new file mode 100644 index 0000000000..90e3506f54 --- /dev/null +++ b/taskcluster/ci/repackage-msix/kind.yml @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + # We're going to generate tasks based on this. We `depend` on and fetch the package from this. + - build-signing + - toolchain + +only-for-build-platforms: + - win32/opt + - win64/opt + - win64/debug + - win32-shippable/opt + - win64-shippable/opt + - win32-devedition/opt + - win64-devedition/opt + +jobs: + repackage-msix: + from-deps: + group-by: platform + worker-type: b-linux-gcp + worker: + max-run-time: 2700 + # This is for fast try iteration; `repackage-shippable-l10n-msix` is the "real one". + run-on-projects: [] + mozharness: + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_signed.py + win64(?!-aarch64)\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_signed.py + win64-aarch64\b.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msix, msix-store] + fetches: + toolchain: + - linux64-msix-packaging + msix: + vendor: Mozilla + channel: + by-shipping-product: + devedition: aurora + default: + by-release-type: + beta.*: beta + release.*: official + esr.*: official + default: nightly + identity-name: + by-shipping-product: + devedition: + by-package-format: + msix: MozillaFirefoxDevedition + msix-store: FirefoxDevedition + default: + by-package-format: + msix: + by-release-type: + beta.*: MozillaFirefoxBeta + release.*: MozillaFirefox + esr.*: MozillaFirefox + default: MozillaFirefoxNightly + msix-store: + by-release-type: + beta.*: FirefoxBeta + release.*: Firefox + esr.*: Firefox + default: FirefoxNightly + publisher: + by-package-format: + msix-store: "CN=082E9164-EE6C-4EC8-B62C-441FAE7BEFA1" + msix: + by-level: + "1": "CN=Mozilla Fake SPC" + "2": "CN=Mozilla Fake SPC" + default: + # This value needs to line up with the signing certificate + # we're going to sign with. We only sign potential release + # or nightly builds with proper certificates, so even on + # level 3 repositories, some build types are expected to + # be signed with our fake certificate. + by-build-platform: + .*-shippable: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + .*-devedition: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + default: "CN=Mozilla Fake SPC" + publisher-display-name: + by-package-format: + msix: "Mozilla Corporation" + msix-store: "Mozilla" diff --git a/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml b/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml new file mode 100644 index 0000000000..6be2a95bf0 --- /dev/null +++ b/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + # We're going to generate tasks based on this. We `depend` on and fetch the package from this. + - build-signing + # We're not going to generate tasks based on these. But we are going to `depend` on and fetch + # langpacks from these. + - shippable-l10n-signing + - toolchain + +not-for-kind-dependencies: + - shippable-l10n-signing + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + - win32-devedition/opt + - win64-devedition/opt + +jobs: + repackage-shippable-l10n-msix: + from-deps: + group-by: platform + unique-kinds: false + worker-type: b-linux-gcp + worker: + max-run-time: 2700 + mozharness: + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_signed.py + win64(?!-aarch64)\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_signed.py + win64-aarch64\b.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msix, msix-store] + fetches: + toolchain: + - linux64-msix-packaging + msix: + vendor: Mozilla + channel: + by-shipping-product: + devedition: aurora + default: + by-release-type: + beta.*: beta + release.*: official + esr.*: official + default: nightly + identity-name: + by-shipping-product: + devedition: + by-package-format: + msix: MozillaFirefoxDevedition + msix-store: FirefoxDevedition + default: + by-package-format: + msix: + by-release-type: + beta.*: MozillaFirefoxBeta + release.*: MozillaFirefox + esr.*: MozillaFirefox + default: MozillaFirefoxNightly + msix-store: + by-release-type: + beta.*: FirefoxBeta + release.*: Firefox + esr.*: Firefox + default: FirefoxNightly + publisher: + by-package-format: + msix-store: "CN=082E9164-EE6C-4EC8-B62C-441FAE7BEFA1" + msix: + by-level: + "1": "CN=Mozilla Fake SPC" + "2": "CN=Mozilla Fake SPC" + default: + # This value needs to line up with the signing certificate + # we're going to sign with. We only sign potential release + # or nightly builds with proper certificates, so even on + # level 3 repositories, some build types are expected to + # be signed with our fake certificate. + by-build-platform: + .*-shippable: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + .*-devedition: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + default: "CN=Mozilla Fake SPC" + publisher-display-name: + by-package-format: + msix: "Mozilla Corporation" + msix-store: "Mozilla" diff --git a/taskcluster/ci/repackage-signing-l10n/kind.yml b/taskcluster/ci/repackage-signing-l10n/kind.yml new file mode 100644 index 0000000000..9d274b1ee3 --- /dev/null +++ b/taskcluster/ci/repackage-signing-l10n/kind.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing + - gecko_taskgraph.transforms.repackage_routes + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-l10n + +only-for-build-platforms: + - win32-shippable/opt + - win32-devedition/opt + - win32/opt + - win64-shippable/opt + - win64-devedition/opt + - win64/opt + - win64-aarch64-shippable/opt + - win64-aarch64-devedition/opt + - win64-asan-reporter-shippable/opt + +jobs: + repackage-signing-l10n: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/repackage-signing-msi/kind.yml b/taskcluster/ci/repackage-signing-msi/kind.yml new file mode 100644 index 0000000000..8a6691413a --- /dev/null +++ b/taskcluster/ci/repackage-signing-msi/kind.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-msi + +only-for-build-platforms: + - win32-shippable/opt + - win32-devedition/opt + - win32/opt + - win64-shippable/opt + - win64-devedition/opt + - win64/opt + +jobs: + repackage-signing-msi: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/repackage-signing-msix/kind.yml b/taskcluster/ci/repackage-signing-msix/kind.yml new file mode 100644 index 0000000000..4976f08366 --- /dev/null +++ b/taskcluster/ci/repackage-signing-msix/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-msix + +jobs: + repackage-signing-msix: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml b/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml new file mode 100644 index 0000000000..0f6f7787ee --- /dev/null +++ b/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage-shippable-l10n-msix + +jobs: + repackage-signing-shippable-l10n-msix: + from-deps: {} diff --git a/taskcluster/ci/repackage-signing/kind.yml b/taskcluster/ci/repackage-signing/kind.yml new file mode 100644 index 0000000000..49ea244a9e --- /dev/null +++ b/taskcluster/ci/repackage-signing/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage_signing + - gecko_taskgraph.transforms.repackage_routes + - gecko_taskgraph.transforms.task + +kind-dependencies: + - repackage + +only-for-build-platforms: + - win32-devedition/opt + - win32-shippable/opt + - win64-devedition/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - win64-aarch64-devedition/opt + - win64-asan-reporter-shippable/opt + +jobs: + repackage-signing: + from-deps: + group-by: single-with-filters diff --git a/taskcluster/ci/repackage/kind.yml b/taskcluster/ci/repackage/kind.yml new file mode 100644 index 0000000000..e67ce5566b --- /dev/null +++ b/taskcluster/ci/repackage/kind.yml @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.repackage + - gecko_taskgraph.transforms.repackage_routes + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build-signing + - build-mac-signing + - build-mac-notarization + - toolchain + +only-for-build-platforms: + - linux-shippable/opt + - linux-devedition/opt + - linux/opt + - linux64-shippable/opt + - linux64-devedition/opt + - linux64-asan-reporter-shippable/opt + - linux64/opt + - macosx64-shippable/opt + - macosx64-devedition/opt + - macosx64/opt + - macosx64/debug + - macosx64-aarch64/debug + - win32-shippable/opt + - win32-devedition/opt + - win32/opt + - win64-shippable/opt + - win64-devedition/opt + - win64-aarch64-shippable/opt + - win64-aarch64-devedition/opt + - win64/opt + - win64-asan-reporter-shippable/opt + +jobs: + repackage: + from-deps: + group-by: single-with-filters + upstream-mac-kind: + by-build-type: + debug: build-mac-signing + default: + by-project: + mozilla-(central|beta|release|esr.*): build-mac-notarization + default: build-mac-signing + worker-type: b-linux-gcp + worker: + max-run-time: + by-build-platform: + .*asan.*: 2700 + default: 900 + mozharness: + config: + by-build-platform: + linux\b.*: + - repackage/base.py + - repackage/linux32_signed.py + linux64\b.*: + - repackage/base.py + - repackage/linux64_signed.py + macosx64\b.*: + - repackage/base.py + - repackage/osx_signed.py + win32\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win32_signed.py + win64(?!-aarch64)\b.*: + - repackage/base.py + - repackage/win32_sfx_stub.py + - repackage/win64_signed.py + win64-aarch64\b.*: + - repackage/base.py + - repackage/win64-aarch64_sfx_stub.py + - repackage/win64_signed.py + package-formats: + by-build-platform: + macosx64\b.*: + by-build-type: + # Debug builds can't be attributable + debug: [dmg, mar, pkg] + default: [dmg-attrib, mar, pkg] + linux.*: [mar] + win32\b.*: [mar, installer] + win64\b.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx diff --git a/taskcluster/ci/repo-update/kind.yml b/taskcluster/ci/repo-update/kind.yml new file mode 100644 index 0000000000..414c3bd4f8 --- /dev/null +++ b/taskcluster/ci/repo-update/kind.yml @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.repo_update:transforms + - gecko_taskgraph.transforms.task:transforms + + +job-defaults: + worker: + env: + DO_HSTS: + by-project: + mozilla-(central|beta|esr.*): "1" + default: "" + DO_HPKP: + by-project: + mozilla-(central|beta|esr.*): "1" + default: "" + DO_REMOTE_SETTINGS: + by-project: + mozilla-(central|beta|release|esr.*): "1" + default: "" + DO_SUFFIX_LIST: + by-project: + mozilla-(central|beta|esr.*): "1" + default: "" + DO_MOBILE_EXPERIMENTS: + by-project: + mozilla-central: "" # TODO: enable after firefox-android merge + default: "" + USE_MOZILLA_CENTRAL: + by-project: + mozilla-central: "1" + default: "" + + +jobs: + hsts-hpkp-blocklist: + name: periodic_file_update + description: HSTS, HPKP and remote settings update + run-on-projects: [] # Only run via cron + treeherder: + kind: build + platform: linux64/opt + symbol: pfu + tier: 1 + worker-type: b-linux-xlarge-gcp + worker: + implementation: docker-worker + os: linux + docker-image: {in-tree: periodic-updates} + max-run-time: 14400 # Takes up to 4 hours + env: + PRODUCT: firefox + REVIEWERS: "#release-managers" + command: + - /runme.sh + taskcluster-proxy: true + artifacts: + - name: 'public/build/nsSTSPreloadList.diff' + path: '/home/worker/artifacts/nsSTSPreloadList.diff' + type: file + - name: 'public/build/StaticHPKPins.h.diff' + path: '/home/worker/artifacts/StaticHPKPins.h.diff' + type: file + - name: 'public/build/remote-settings.diff' + path: '/home/worker/artifacts/remote-settings.diff' + type: file + - name: 'public/build/initial_experiments.diff' + path: '/home/worker/artifacts/initial_experiments.diff' + type: file + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token diff --git a/taskcluster/ci/scriptworker-canary/kind.yml b/taskcluster/ci/scriptworker-canary/kind.yml new file mode 100644 index 0000000000..a1ee225b89 --- /dev/null +++ b/taskcluster/ci/scriptworker-canary/kind.yml @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.scriptworker_canary:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + platform: firefox-release/opt + tier: 1 + kind: build + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: push-to-try} + max-run-time: 1800 + env: + MOZBUILD_STATE_PATH: /builds/worker/workspace + run: + sparse-profile: push-to-try + +jobs: + push: + description: Push scriptworker canary tasks. + treeherder: + symbol: Rel(push-scriptworker-canary) + run-on-projects: [mozilla-central] + addresses: ['release+scriptworker-canary@mozilla.com'] + ssh-key-secret: project/releng/scriptworker/scriptworker-canary-sshkey diff --git a/taskcluster/ci/searchfox/kind.yml b/taskcluster/ci/searchfox/kind.yml new file mode 100644 index 0000000000..98c2235fe1 --- /dev/null +++ b/taskcluster/ci/searchfox/kind.yml @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + index: + product: firefox + treeherder: + symbol: Searchfox(idx) + kind: build + tier: 2 + run-on-projects: ['mozilla-central'] + optimization: null + use-system-python: false + # These builds don't use sccache because otherwise we wouldn't index + # the files that sccache optimizes away compilation for. + use-sccache: false + +jobs: + linux64-searchfox/debug: + description: "Linux64 Debug Searchfox" + index: + job-name: linux64-searchfox-debug + treeherder: + platform: linux64/debug + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "linux" + run: + using: mozharness + actions: [build] + config: + - builds/releng_base_firefox.py + - builds/releng_sub_linux_configs/64_searchfox_and_debug.py + script: "mozharness/scripts/fx_desktop_build.py" + tooltool-downloads: public + keep-artifacts: false + fetches: + toolchain: + - linux64-clang + - linux64-nasm + - linux64-node + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + macosx64-searchfox/debug: + description: "MacOS X x64 Debug Cross-compile Searchfox" + index: + job-name: macosx64-searchfox-debug + treeherder: + platform: osx-cross/debug + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "macosx" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: cross-debug-searchfox + secrets: true + keep-artifacts: false + fetches: + toolchain: + - linux64-clang + - linux64-hfsplus + - linux64-libdmg + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-cbindgen + - linux64-dump_syms + - macosx64-sdk + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + win64-searchfox/debug: + description: "Win64 Searchfox Debug (clang-cl)" + index: + job-name: win64-searchfox-debug + treeherder: + platform: windows2012-64/debug + worker-type: b-linux-gcp + worker: + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "windows" + run: + using: mozharness + options: [append-env-variables-from-configs] + actions: [get-secrets, build] + script: mozharness/scripts/fx_desktop_build.py + secrets: true + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/searchfox_debug.py + extra-config: + mozconfig_platform: win64 + fetches: + toolchain: + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-nasm + - linux64-node + - linux64-cbindgen + - linux64-dump_syms + - linux64-wine + - linux64-winchecksec + - nsis + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + - vs + - windows-rs + fetch: + - upx-3.95-win + + android-armv7-searchfox/debug: + description: "Android ARMv7 Debug Searchfox" + index: + job-name: android-armv7-searchfox-debug + treeherder: + platform: android-4-1-armv7/debug + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + max-run-time: 10800 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "android-armv7" + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_android_64_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: arm-debug-searchfox + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - linux64-jdk + - linux64-clang + - linux64-rust-cross + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-nasm + - linux64-node + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi diff --git a/taskcluster/ci/sentry/kind.yml b/taskcluster/ci/sentry/kind.yml new file mode 100644 index 0000000000..bfc3cf07a2 --- /dev/null +++ b/taskcluster/ci/sentry/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.sentry:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + mach-release: + description: Inform Sentry of new revisions (aka Mach releases) + worker-type: b-linux-gcp + worker: + taskcluster-proxy: true + max-run-time: 3600 + docker-image: {in-tree: sentry} + env: + HG_PUSHLOG_URL: "{head_repository}/json-pushes?version=2&changeset={head_rev}" + SENTRY_SECRET: "project/engwf/gecko/{level}/tokens" + scopes: + - "secrets:get:project/engwf/gecko/{level}/tokens" + run: + using: run-task + checkout: false + command: /bin/bash /usr/bin/submit_sentry_release.sh + run-on-projects: ['mozilla-central'] + treeherder: + symbol: Sentry(mach-release) + platform: other/opt + tier: 2 + kind: other diff --git a/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml b/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml new file mode 100644 index 0000000000..461eac91ac --- /dev/null +++ b/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.mac_notarization + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.upstream_artifact_task + - gecko_taskgraph.transforms.shippable_l10n_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - shippable-l10n-mac-signing + +only-for-attributes: + - shippable + +jobs: + shippable-l10n-mac-notarization: + from-deps: + group-by: chunk-locales + copy-attributes: true + max-run-time: 1800 + treeherder: + symbol: BMN diff --git a/taskcluster/ci/shippable-l10n-mac-signing/kind.yml b/taskcluster/ci/shippable-l10n-mac-signing/kind.yml new file mode 100644 index 0000000000..21fe44b4f0 --- /dev/null +++ b/taskcluster/ci/shippable-l10n-mac-signing/kind.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.shippable_l10n_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.hardened_signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - shippable-l10n + +only-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + - macosx64/opt + - macosx64/debug + +jobs: + shippable-l10n-mac-signing: + from-deps: + group-by: chunk-locales + treeherder: + symbol: BMS + attributes: + shipping_phase: promote diff --git a/taskcluster/ci/shippable-l10n-signing/kind.yml b/taskcluster/ci/shippable-l10n-signing/kind.yml new file mode 100644 index 0000000000..fdb545381f --- /dev/null +++ b/taskcluster/ci/shippable-l10n-signing/kind.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.name_sanity + - gecko_taskgraph.transforms.upstream_artifact_task + - gecko_taskgraph.transforms.shippable_l10n_signing + - gecko_taskgraph.transforms.signing + - gecko_taskgraph.transforms.task + +kind-dependencies: + - shippable-l10n + +only-for-attributes: + - shippable + - nightly + +not-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + - macosx64/opt + - macosx64/debug + +jobs: + shippable-l10n-signing: + from-deps: + group-by: chunk-locales diff --git a/taskcluster/ci/shippable-l10n/kind.yml b/taskcluster/ci/shippable-l10n/kind.yml new file mode 100644 index 0000000000..d38edb2ee5 --- /dev/null +++ b/taskcluster/ci/shippable-l10n/kind.yml @@ -0,0 +1,199 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.l10n + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + - build-signing + - fetch + - repackage + - toolchain + +only-for-build-platforms: + - linux64-shippable/opt + - linux-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-aarch64-shippable/opt + - linux64-devedition/opt + - linux-devedition/opt + - macosx64-devedition/opt + - win32-devedition/opt + - win64-devedition/opt + - win64-aarch64-devedition/opt + +jobs: + shippable-l10n: + description: Localization + from-deps: + group-by: platform + locales-file: browser/locales/l10n-changesets.json + locales-per-chunk: 5 + run-on-projects: ['release'] + attributes: + shipping_phase: promote + ignore-locales: + by-build-platform: + # OSX has a special locale for japanese + macosx64.*: [ja] + default: [ja-JP-mac] + run-time: 1800 + fetches: + toolchain: + by-build-platform: + default: [] + macosx64.*: + - linux64-libdmg + - linux64-hfsplus + win.*: + - linux64-wine + - nsis + fetch: + by-build-platform: + default: [] + win.*: + - upx-3.95-win + tooltool: + by-build-platform: + default: internal + linux.*: public + index: + type: shippable-l10n + product: + by-build-platform: + default: firefox + .*-devedition: devedition + job-name: + by-build-platform: + linux-shippable: linux-opt + linux64-shippable: linux64-opt + macosx64-shippable: macosx64-opt + win32-shippable: win32-opt + win64-shippable: win64-opt + win64-aarch64-shippable: win64-aarch64-opt + linux-devedition: linux-devedition-opt + linux64-devedition: linux64-devedition-opt + macosx64-devedition: macosx64-devedition-opt + win32-devedition: win32-devedition-opt + win64-devedition: win64-devedition-opt + win64-aarch64-devedition: win64-aarch64-devedition-opt + worker-type: b-linux-gcp + treeherder: + symbol: L10n(B) + tier: 1 + platform: + by-build-platform: + linux64-shippable: linux64-shippable/opt + linux-shippable: linux32-shippable/opt + macosx64-shippable: osx-shippable/opt + win32-shippable: windows2012-32-shippable/opt + win64-shippable: windows2012-64-shippable/opt + win64-aarch64-shippable: windows2012-aarch64-shippable/opt + linux64-devedition: linux64-devedition/opt + linux-devedition: linux32-devedition/opt + macosx64-devedition: osx-cross-devedition/opt + win32-devedition: windows2012-32-devedition/opt + win64-devedition: windows2012-64-devedition/opt + win64-aarch64-devedition: windows2012-aarch64-devedition/opt + env: + by-build-platform: + linux.*: # linux64 and 32 get same treatment here + EN_US_PACKAGE_NAME: target.tar.bz2 + EN_US_BINARY_URL: + artifact-reference: + macosx64.*: + EN_US_PACKAGE_NAME: target.dmg + EN_US_BINARY_URL: + artifact-reference: + win.*: + EN_US_PACKAGE_NAME: target.zip + EN_US_BINARY_URL: + artifact-reference: + mozharness: + config: + by-build-platform: + linux-shippable: + - single_locale/firefox.py + - single_locale/linux32.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - taskcluster_nightly.py + linux64-shippable: + - single_locale/firefox.py + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - taskcluster_nightly.py + macosx64-shippable: + - single_locale/firefox.py + - single_locale/macosx64.py + - single_locale/tc_common.py + - single_locale/tc_macosx64.py + - taskcluster_nightly.py + win32-shippable: + - single_locale/firefox.py + - single_locale/win32.py + - single_locale/tc_common.py + - single_locale/tc_win32.py + - taskcluster_nightly.py + win64-shippable: + - single_locale/firefox.py + - single_locale/win64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - taskcluster_nightly.py + win64-aarch64-shippable: + - single_locale/firefox.py + - single_locale/win64-aarch64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - taskcluster_nightly.py + linux-devedition: + - single_locale/devedition.py + - single_locale/linux32.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - taskcluster_nightly.py + linux64-devedition: + - single_locale/devedition.py + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - taskcluster_nightly.py + macosx64-devedition: + - single_locale/devedition.py + - single_locale/macosx64.py + - single_locale/tc_common.py + - single_locale/tc_macosx64.py + - taskcluster_nightly.py + win32-devedition: + - single_locale/devedition.py + - single_locale/win32.py + - single_locale/tc_common.py + - single_locale/tc_win32.py + - taskcluster_nightly.py + win64-devedition: + - single_locale/devedition.py + - single_locale/win64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - taskcluster_nightly.py + win64-aarch64-devedition: + - single_locale/devedition.py + - single_locale/win64-aarch64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - taskcluster_nightly.py + # no default, so we fail on new entries + actions: ['clone-locales', 'list-locales', 'setup', 'repack', + 'summary'] + script: mozharness/scripts/desktop_l10n.py diff --git a/taskcluster/ci/snap-upstream-build/kind.yml b/taskcluster/ci/snap-upstream-build/kind.yml new file mode 100644 index 0000000000..7016989ba8 --- /dev/null +++ b/taskcluster/ci/snap-upstream-build/kind.yml @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + cron: true + treeherder: + kind: build + tier: 2 + symbol: B + worker-type: b-linux-gcp + run: + checkout: true + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: snap-build-core22} + max-run-time: 900 + env: + UPLOAD_DIR: artifacts + MOZ_STACKWALK_SYMBOLS_SERVER: '1' + fetches: + toolchain: + - linux64-minidump-stackwalk + +jobs: + amd64-nightly/opt: + description: Build Firefox Nightly as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-nightly + build_type: opt + index: + product: firefox + job-name: amd64-nightly + type: generic + worker: + max-run-time: 9000 + run: + using: run-task + command: >- + ./run.sh nightly + treeherder: + platform: linux64-snap-amd64-nightly/opt + + amd64-nightly/debug: + description: Build Firefox Nightly (debug) as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-nightly + build_type: debug + index: + product: firefox + job-name: amd64-nightly-debug + type: generic + worker: + max-run-time: 7200 + run: + using: run-task + command: >- + ./run.sh nightly 1 + treeherder: + platform: linux64-snap-amd64-nightly/debug + + amd64-beta/opt: + description: Build Firefox Beta as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-beta + build_type: opt + index: + product: firefox + job-name: amd64-beta + type: generic + worker: + max-run-time: 9000 + run: + using: run-task + command: >- + ./run.sh beta + treeherder: + platform: linux64-snap-amd64-beta/opt + + amd64-beta/debug: + description: Build Firefox Beta (debug) as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-beta + build_type: debug + index: + product: firefox + job-name: amd64-beta-debug + type: generic + worker: + max-run-time: 7200 + run: + using: run-task + command: >- + ./run.sh beta 1 + treeherder: + platform: linux64-snap-amd64-beta/debug + + amd64-stable/opt: + description: Build Firefox Stable as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-stable + build_type: opt + index: + product: firefox + job-name: amd64-stable + type: generic + worker: + max-run-time: 9000 + run: + using: run-task + command: >- + ./run.sh stable + treeherder: + platform: linux64-snap-amd64-stable/opt + + amd64-stable/debug: + description: Build Firefox Stable (debug) as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-stable + build_type: debug + index: + product: firefox + job-name: amd64-stable-debug + type: generic + worker: + max-run-time: 7200 + run: + using: run-task + command: >- + ./run.sh stable 1 + treeherder: + platform: linux64-snap-amd64-stable/debug + + amd64-esr/opt: + description: Build Firefox ESR as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-esr + build_type: opt + index: + product: firefox + job-name: amd64-esr + type: generic + worker: + max-run-time: 9000 + run: + using: run-task + command: >- + ./run.sh esr + treeherder: + platform: linux64-snap-amd64-esr/opt + + amd64-esr/debug: + description: Build Firefox ESR (debug) as a Snap using upstream definition + attributes: + build_platform: linux64-snap-upstream-esr + build_type: debug + index: + product: firefox + job-name: amd64-esr-debug + type: generic + worker: + max-run-time: 7200 + run: + using: run-task + command: >- + ./run.sh esr 1 + treeherder: + platform: linux64-snap-amd64-esr/debug + + amd64-try/opt: + description: Build Firefox Nightly as a Snap using upstream definition and try repo + attributes: + build_platform: linux64-snap-upstream-try + build_type: opt + index: + product: firefox + job-name: amd64-try + type: generic + worker: + max-run-time: 9000 + run: + using: run-task + command: >- + ./run.sh try + treeherder: + platform: linux64-snap-amd64-try/opt + + amd64-try/debug: + description: Build Firefox Nightly (debug) as a Snap using upstream definition and try repo + attributes: + build_platform: linux64-snap-upstream-try + build_type: debug + index: + product: firefox + job-name: amd64-try-debug + type: generic + worker: + max-run-time: 7200 + run: + using: run-task + command: >- + ./run.sh try 1 + treeherder: + platform: linux64-snap-amd64-try/debug diff --git a/taskcluster/ci/snap-upstream-test/kind.yml b/taskcluster/ci/snap-upstream-test/kind.yml new file mode 100644 index 0000000000..585a1fca1f --- /dev/null +++ b/taskcluster/ci/snap-upstream-test/kind.yml @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +only-for-attributes: + - build_platform + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + - gecko_taskgraph.transforms.snap_test + +kind-dependencies: + - snap-upstream-build + +job-defaults: + attributes: + retrigger: true + cron: true + treeherder: + kind: test + tier: 2 + run: + using: run-task + checkout: false + worker: + max-run-time: 900 + env: + WAYLAND_DISPLAY: 'wayland-0' + artifacts: + - name: public/build + type: directory + path: builds/worker/artifacts/ + +jobs: + basic: + from-deps: + group-by: single-with-filters + description: Runs basic tests on a Firefox Snap package + worker-type: t-linux-wayland + run: + command: >- + export TASKCLUSTER_ROOT_DIR=$PWD && cd $MOZ_FETCHES_DIR/ && ./tests.sh --basic + fetches: + snap-upstream-build: + - firefox.snap + - snap-tests.zip + treeherder: + platform: linux64/opt + symbol: Sel(basic) + attributes: + snap_test_type: basic + + qa: + from-deps: + group-by: single-with-filters + description: Runs some QA tests on a Firefox Snap package + worker-type: t-linux-wayland + run: + command: >- + export TASKCLUSTER_ROOT_DIR=$PWD && cd $MOZ_FETCHES_DIR/ && ./tests.sh --qa + fetches: + snap-upstream-build: + - firefox.snap + - snap-tests.zip + treeherder: + platform: linux64/opt + symbol: Sel(QA) + attributes: + snap_test_type: qa diff --git a/taskcluster/ci/source-test/clang.yml b/taskcluster/ci/source-test/clang.yml new file mode 100644 index 0000000000..d77066afe7 --- /dev/null +++ b/taskcluster/ci/source-test/clang.yml @@ -0,0 +1,117 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run-on-projects: ["trunk"] + platform: linux64/debug + worker-type: t-linux-xlarge-source + attributes: + code-review: true + worker: + docker-image: {in-tree: static-analysis-build} + max-run-time: 5400 + treeherder: + kind: other + tier: 2 + run: + using: run-task + tooltool-downloads: public + fetches: + toolchain: + - linux64-cbindgen + - linux64-nasm + - linux64-node + when: + files-changed: + - '**/*.c' + - '**/*.cpp' + - '**/*.cc' + - '**/*.cxx' + - '**/*.m' + - '**/*.mm' + +tidy: + description: Run static-analysis (clang-tidy) on C/C++ patches + treeherder: + symbol: cpp(clang-tidy) + run: + cwd: '{checkout}' + command: >- + source taskcluster/scripts/misc/source-test-clang-setup.sh && + ./mach --log-no-times static-analysis check --outgoing --output $HOME/clang-tidy.json --format json + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy + - linux64-rust + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + worker: + artifacts: + - type: file + name: public/code-review/clang-tidy.json + path: /builds/worker/clang-tidy.json + +format: + description: Run clang-format on C/C++ patches + treeherder: + symbol: cpp(clang-format) + # Run only on try and code-review tasks + # to avoid running clang format on the whole codebase + run-on-projects: [] + run: + cwd: '{checkout}' + command: >- + source taskcluster/scripts/misc/source-test-clang-setup.sh && + ./mach --log-no-times clang-format --outgoing --output $HOME/clang-format.json --format json && + ./mach --log-no-times clang-format --outgoing --output $HOME/clang-format.diff --format diff + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy + - linux64-rust + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + when: + files-changed: + - '**/*.c' + - '**/*.cpp' + - '**/*.cc' + - '**/*.cxx' + - '**/*.m' + - '**/*.mm' + - '**/*.h' + - '**/*.hh' + - '**/*.hpp' + - '**/*.hxx' + worker: + artifacts: + - type: file + name: public/code-review/clang-format.json + path: /builds/worker/clang-format.json + - type: file + name: public/code-review/clang-format.diff + path: /builds/worker/clang-format.diff + +external: + description: Run static-analysis (clang-tidy) on C/C++ patches with external checks + treeherder: + symbol: cpp(clang-tidy-external) + run: + cwd: '{checkout}' + command: >- + source taskcluster/scripts/misc/source-test-clang-setup.sh && + ./mach --log-no-times static-analysis check --checks="-*,mozilla-civet-*" --outgoing --output $HOME/clang-tidy.json --format json + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy-external + - linux64-rust + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + worker: + artifacts: + - type: file + name: public/code-review/clang-tidy.json + path: /builds/worker/clang-tidy.json diff --git a/taskcluster/ci/source-test/cram.yml b/taskcluster/ci/source-test/cram.yml new file mode 100644 index 0000000000..6e29e7c6a6 --- /dev/null +++ b/taskcluster/ci/source-test/cram.yml @@ -0,0 +1,21 @@ +--- +tryselect: + description: tools/tryselect integration tests + platform: linux1804-64/opt + treeherder: + symbol: cram(try) + kind: test + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + mach: cramtest tools/tryselect + always-target: true + when: + files-changed: + - testing/mach_commands.py + - third_party/python/cram/** + - tools/tryselect/** diff --git a/taskcluster/ci/source-test/doc.yml b/taskcluster/ci/source-test/doc.yml new file mode 100644 index 0000000000..3e0459f8f0 --- /dev/null +++ b/taskcluster/ci/source-test/doc.yml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +generate: + description: Generate the Sphinx documentation + platform: doc/opt + treeherder: + symbol: doc(generate) + kind: test + tier: 1 + run-on-projects: ['trunk'] + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + env: + LC_ALL: C + artifacts: + - type: file + name: public/docs.tar.gz + path: /builds/worker/checkouts/gecko/docs-out/main.tar.gz + - type: file + name: public/trees.json + path: /builds/worker/artifacts/trees.json + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + ./mach doc --outdir docs-out --no-open --no-serve --archive --fatal-warnings --check-num-warnings --dump-trees=/builds/worker/artifacts/trees.json + sparse-profile: sphinx-docs + optimization: + skip-unless-schedules: [docs] + +upload: + description: Generate and upload the Sphinx documentation + platform: doc/opt + attributes: + code-review: true + treeherder: + symbol: doc(upload) + kind: test + tier: 2 + run-on-projects: [mozilla-central-only] + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + env: + LC_ALL: C + MOZ_SOURCE_DOCS_USE_GOOGLE: "1" + taskcluster-proxy: true + artifacts: + - type: file + name: public/firefox-source-docs-url.txt + path: /builds/worker/firefox-source-docs-url.txt + - type: file + name: public/trees.json + path: /builds/worker/artifacts/trees.json + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + ./mach doc --upload --no-open --no-serve --fatal-warnings --write-url /builds/worker/firefox-source-docs-url.txt --dump-trees=/builds/worker/artifacts/trees.json + sparse-profile: sphinx-docs + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/gecko-docs-upload + optimization: + skip-unless-schedules: [docs] diff --git a/taskcluster/ci/source-test/file-metadata.yml b/taskcluster/ci/source-test/file-metadata.yml new file mode 100644 index 0000000000..b1fffaf64f --- /dev/null +++ b/taskcluster/ci/source-test/file-metadata.yml @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: lint/opt + treeherder: + kind: other + tier: 1 + worker-type: t-linux-xlarge-source + worker: + artifacts: + - type: directory + path: /builds/worker/artifacts + name: public + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + +bugzilla-components: + description: Generate metadata about source files and Bugzilla + treeherder: + symbol: Bugzilla + tier: 1 + index: + product: source + job-name: source-bugzilla-info + run: + mach: file-info bugzilla-automation /builds/worker/artifacts + worker: + max-run-time: 1800 + run-on-projects: ['autoland', 'mozilla-central'] + +test-info-xorigin: + description: Generate test manifest metadata for tests disabled on cross-origin + treeherder: + symbol: test-info(xorigin) + tier: 2 + index: + product: source + job-name: test-info-xorigin + worker: + docker-image: {in-tree: debian12-amd64-build} + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy + - linux64-rust + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + run: + using: run-task + cwd: '{checkout}' + command: >- + source taskcluster/scripts/misc/source-test-common.sh && + ./mach test-info report --show-tests --show-summary --filter-keys=fail-if,fails-if,random-if,skip-if --filter-values='(?- + source taskcluster/scripts/misc/source-test-common.sh && + ./mach test-info report --show-tests --show-summary --filter-keys=fail-if,fails-if,random-if,skip-if --filter-values='(?- + source taskcluster/scripts/misc/source-test-common.sh && + ./mach test-info testrun-report --output-file /builds/worker/artifacts/test-run-info.json && + ./mach test-info report --show-tests --show-summary --show-testruns --verbose --output-file /builds/worker/artifacts/test-info-all-tests.json --runcounts-input-file /builds/worker/artifacts/test-run-info.json && + ./mach test-info report --show-annotations --output-file /builds/worker/artifacts/test-info-manifest-conditions.json diff --git a/taskcluster/ci/source-test/jsshell.yml b/taskcluster/ci/source-test/jsshell.yml new file mode 100644 index 0000000000..2ce0a846d4 --- /dev/null +++ b/taskcluster/ci/source-test/jsshell.yml @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux64-shippable/opt + require-build: + linux64.*: build-linux64-shippable/opt + worker-type: + by-platform: + linux64.*: t-linux-talos-1804 + worker: + by-platform: + linux64.*: + env: + by-shell: + sm: + SHELL: /bin/bash + JSSHELL: js + v8: + SHELL: /bin/bash + JSSHELL: d8/release/d8 + max-run-time: 3600 + treeherder: + kind: test + tier: 2 + run: + using: mach + mach: jsshell-bench --perfherder={shell} --binary=$MOZ_FETCHES_DIR/$JSSHELL {test} + run-on-projects: + by-shell: + sm: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central', 'integration'] + default: + by-test-platform: + (linux|windows|macos)(?!.*shippable).*: [] + default: ['mozilla-central'] + fetches: + build: + - target.jsshell.zip + toolchain: + - linux64-custom-v8 + optimization: + skip-unless-expanded: null + +bench-ares6: + description: Ares6 JavaScript shell benchmark suite + shell: ['sm', 'v8'] + test: ares6 + treeherder: + symbol: ares6 + +bench-sixspeed: + description: Six-Speed JavaScript shell benchmark suite + shell: ['sm', 'v8'] + test: six-speed + treeherder: + symbol: 6speed + +bench-sunspider: + description: SunSpider JavaScript shell benchmark suite + shell: ['sm'] + test: sunspider + treeherder: + symbol: sunspider + +bench-web-tooling: + description: Web Tooling shell benchmark suite + shell: ['sm', 'v8'] + test: web-tooling-benchmark + treeherder: + symbol: webtool + fetches: + fetch: + - web-tooling-benchmark + +bench-octane: + description: Octane shell benchmark suite + shell: ['sm', 'v8'] + test: octane + treeherder: + symbol: octane + fetches: + fetch: + - octane diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml new file mode 100644 index 0000000000..bb605e939a --- /dev/null +++ b/taskcluster/ci/source-test/kind.yml @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - fetch + - toolchain + - build + +jobs-from: + - clang.yml + - cram.yml + - doc.yml + - file-metadata.yml + - jsshell.yml + - mozlint.yml + - mozlint-android.yml + - node.yml + - puppeteer.yml + - python.yml + - shadow-scheduler.yml + - taskgraph.yml + - vendor.yml + - webidl.yml + - webcompat.yml + - wpt-manifest.yml + - wpt-metadata.yml + +job-defaults: + use-system-python: false + attributes: + retrigger: true + task-context: + from-object: {} + substitution-fields: [] diff --git a/taskcluster/ci/source-test/mozlint-android.yml b/taskcluster/ci/source-test/mozlint-android.yml new file mode 100644 index 0000000000..6602d8c818 --- /dev/null +++ b/taskcluster/ci/source-test/mozlint-android.yml @@ -0,0 +1,106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +# Just one TC job that runs all the lints. There's no real advantage +# to running them separately, and this way we perform the expensive +# clone and somewhat expensive compilation of the Android +# intermediates just once. +lints: + always-target: true + attributes: + build_platform: android + build_type: opt + code-review: true + description: Android lints + platform: lint/opt + treeherder: + symbol: A(lints) + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: /builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline + MOZCONFIG: mobile/android/config/mozconfigs/android-arm/nightly-android-lints + MOZ_OBJDIR: obj-firefox + PERFHERDER_EXTRA_OPTIONS: android-lints + TINDERBOX_OUTPUT: '1' + max-run-time: 3600 + artifacts: + - type: file + name: public/code-review/mozlint.json + path: /builds/worker/mozlint.json + - name: public/android/geckoview/api.txt + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/intermediates/javac/withGeckoBinariesDebug/classes/api.txt + type: file + - name: public/android/geckoview/api.txt.map + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/intermediates/javac/withGeckoBinariesDebug/classes/api.txt.map + type: file + - name: public/android/geckoview/apilint-result.json + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/intermediates/javac/withGeckoBinariesDebug/classes/apilint-result.json + type: file + - name: public/android/geckoview/javadoc-results.json + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/javadoc-results-withGeckoBinariesDebug.json + type: file + - name: public/geckoview/unittest + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/tests + type: directory + - name: public/android/lint/lint-results-officialWithoutGeckoBinariesDebug.html + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/lint-results-withGeckoBinariesDebug.html + type: file + - name: public/android/lint/lint-results-withGeckoBinariesDebug.xml + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/lint-results-withGeckoBinariesDebug.xml + type: file + - name: public/android/checkstyle/checkstyle.html + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/checkstyle/withGeckoBinariesDebug.html + type: file + - name: public/android/checkstyle/checkstyle.xml + path: /builds/worker/checkouts/gecko/obj-firefox/gradle/build/mobile/android/geckoview/reports/checkstyle/withGeckoBinariesDebug.xml + type: file + run: + using: run-task + command: > + ln -s $MOZ_FETCHES_DIR/android-gradle-dependencies $GECKO_PATH && + ln -s $MOZ_FETCHES_DIR/android-sdk-linux $GECKO_PATH && + ln -s $MOZ_FETCHES_DIR/node $GECKO_PATH && + cd $GECKO_PATH && + ./mach --log-no-times build pre-export export && + ./mach --log-no-times lint -f treeherder -f json:/builds/worker/mozlint.json + --linter android-api-lint + --linter android-format + --linter android-javadoc + --linter android-checkstyle + --linter android-lint + --linter android-test + * + tooltool-downloads: internal # For internal toolchains. + fetches: + toolchain: + # Aliases aren't allowed for toolchains installed by fetch. + - linux64-android-gradle-dependencies + - linux64-android-sdk-linux-repack + - linux64-jdk-repack + - linux64-node + when: + files-changed: + # Source files. + - 'mobile/android/**/*.java' + - 'mobile/android/**/*.kt' + # Resources. + - 'mobile/android/**/*.jpeg' + - 'mobile/android/**/*.jpg' + - 'mobile/android/**/*.png' + - 'mobile/android/**/*.svg' + - 'mobile/android/**/*.xml' + # Build stuff. + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + # Other misc lint related files. + - 'python/mozlint/**' + - 'tools/lint/**' diff --git a/taskcluster/ci/source-test/mozlint.yml b/taskcluster/ci/source-test/mozlint.yml new file mode 100644 index 0000000000..28f2381827 --- /dev/null +++ b/taskcluster/ci/source-test/mozlint.yml @@ -0,0 +1,609 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + always-target: true + attributes: + code-review: true + treeherder: + kind: test + tier: 1 + platform: lint/opt + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + artifacts: + - type: file + name: public/code-review/mozlint.json + path: /builds/worker/mozlint.json + run: + using: mach + when: + files-changed: + # Other misc lint related files. + - 'python/mozlint/**' + - 'tools/lint/**' + +codespell: + description: Checks for misspellings in text files + treeherder: + symbol: text(spell) + # We don't want patches to be backed out because of a typo, a follow up bug will be + # good enough + tier: 2 + run: + mach: lint -v -l codespell -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.js' + - '**/*.jsm' + - '**/*.jxs' + - '**/*.mjs' + - '**/*.xml' + - '**/*.html' + - '**/*.xhtml' + - '**/*.cpp' + - '**/*.c' + - '**/*.h' + - '**/*.configure' + - '**/*.py' + - '**/*.properties' + - '**/*.rst' + - '**/*.ftl' + - 'tools/lint/codespell.yml' + + +rejected-words: + description: Reject some words we don't want to use in the code base + treeherder: + symbol: text(rejected-words) + # We don't want patches to be backed out because of this, a follow up bug will be + # good enough + tier: 2 + run: + mach: lint -v -l rejected-words -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.js' + - '**/*.jsm' + - '**/*.jxs' + - '**/*.mjs' + - '**/*.xml' + - '**/*.html' + - '**/*.xhtml' + - '**/*.cpp' + - '**/*.c' + - '**/*.h' + - '**/*.idl' + - '**/*.webidl' + - '**/*.configure' + - '**/*.py' + - '**/*.properties' + - '**/*.rst' + - '**/*.ftl' + - '**/*.java' + - '**/*.kt' + - 'tools/lint/rejected-words.yml' + +eslint: + description: JS lint check + treeherder: + symbol: js(ES) + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + ./mach lint -v -l eslint -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + # Files that are likely audited. + - '**/*.js' + - '**/*.jsm' + - '**/*.json' + - '**/*.jsx' + - '**/*.mjs' + - '**/*.sjs' + - '**/*.html' + - '**/*.xhtml' + - '**/*.xml' + - 'tools/lint/eslint.yml' + # Run when eslint policies change. + - '**/.eslintignore' + - '**/*eslintrc*' + # The plugin implementing custom checks. + - 'tools/lint/eslint/eslint-plugin-mozilla/**' + - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**' + +eslint-build: + description: ESLint checks with build data + always-target: false + # Triggered by cron task, don't run on any projects and disable optimization. + run-on-projects: [] + when: null + attributes: + code-review: false + treeherder: + symbol: js(ES-B) + tier: 2 + run: + using: run-task + cwd: '{checkout}' + command: > + export MOZ_XPT_ARTIFACTS_DIR="$MOZ_FETCHES_DIR/$MOZ_XPT_ARTIFACTS_DIRNAME" && + find "$MOZ_XPT_ARTIFACTS_DIR" && + cp -r /build/node_modules_eslint node_modules && + cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + ./mach lint -v -l eslint -f treeherder --quiet -f json:/builds/worker/mozlint.json . --rule "mozilla/valid-services-property: error" --rule "mozilla/valid-ci-uses: error" + worker: + env: + MOZ_XPT_ARTIFACTS_DIRNAME: xpt_artifacts + require-build: + .*: build-linux64/opt + fetches: + build: + - artifact: target.xpt_artifacts.zip + dest: xpt_artifacts + +stylelint: + description: CSS lint check + treeherder: + symbol: stylelint + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + ./mach lint -v -l stylelint -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + # Files that are likely audited. + - '**/*.css' + - 'tools/lint/styleint.yml' + # Run when stylelint policies change. + - '**/.stylelintignore' + - '**/*stylelintrc*' + +license: + description: Check for license blocks in source files. + treeherder: + symbol: pedantic(license) + run: + mach: lint -v -l license -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.css' + - '**/*.dtd' + - '**/*.ftl' + - '**/*.h' + - '**/*.html' + - '**/*.idl' + - '**/*.js' + - '**/*.jsm' + - '**/*.jsx' + - '**/*.m' + - '**/*.mjs' + - '**/*.mm' + - '**/*.properties' + - '**/*.py' + - '**/*.rs' + - '**/*.svg' + - '**/*.webidl' + - '**/*.xhtml' + - '**/*.xml' + - 'tools/lint/license.yml' + +lintpref: + description: Check for duplicates between all.js and StaticPrefList.yaml. + treeherder: + symbol: misc(pref) + run: + mach: lint -v -l lintpref -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - 'modules/libpref/init/all.js' + - 'modules/libpref/init/StaticPrefList.yaml' + - 'browser/app/profile/firefox.js' + - 'devtools/client/preferences/debugger.js' + - 'mobile/android/app/geckoview-prefs.js' + - 'browser/branding/official/pref/firefox-branding.js' + - 'browser/branding/nightly/pref/firefox-branding.js' + - 'browser/branding/unofficial/pref/firefox-branding.js' + - 'browser/branding/aurora/pref/firefox-branding.js' + - 'browser/app/profile/channel-prefs.js' + - 'mobile/android/installer/mobile-l10n.js' + - 'mobile/android/locales/en-US/mobile-l10n.js' + - 'tools/lint/lintpref.yml' + +mingw-cap: + description: lint for MinGW Capitalization issues + treeherder: + symbol: misc(mingw) + run: + mach: lint -v -l mingw-capitalization -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mingw-capitalization.yml' + +updatebot: + description: Ensure moz.yaml files are valid per schema + treeherder: + symbol: misc(mozyaml) + run: + mach: lint -v -l updatebot -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/moz.yaml' + +mscom-init: + description: Lint for Microsoft COM initialization + treeherder: + symbol: misc(mscom) + run: + mach: lint -v -l mscom-init -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mscom-init.yml' + +py-black: + description: black run over the gecko codebase + treeherder: + symbol: py(black) + run: + mach: lint -v -l black -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + # The list of extensions should match tools/lint/black.yml + - '**/*.py' + - '**/moz.build' + - '**/*.configure' + - '**/*.mozbuild' + - 'pyproject.toml' + - 'tools/lint/black.yml' + +py-ruff: + description: Run ruff over the gecko codebase + treeherder: + symbol: py(ruff) + run: + mach: lint -v -l ruff -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.py' + - '**/*.configure' + - '**/.ruff.toml' + - 'pyproject.toml' + - 'tools/lint/ruff.yml' + - 'tools/lint/python/ruff.py' + - 'tools/lint/python/ruff_requirements.txt' + +test-manifest: + description: lint test manifests + treeherder: + symbol: misc(tm) + run: + mach: lint -v -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.toml' + - 'python/mozlint/**' + - 'tools/lint/**' + - 'tools/lint/test-disable.yml' + +wptlint-gecko: + description: web-platform-tests linter + treeherder: + symbol: misc(W) + run: + mach: lint -v -l wpt -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - 'testing/web-platform/tests/**' + - 'testing/web-platform/mozilla/tests/**' + - 'tools/lint/wpt.yml' + +yaml: + description: yamllint run over the gecko codebase + treeherder: + symbol: pedantic(yaml) + run: + mach: lint -v -l yaml -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.yml' + - '**/*.yaml' + - '**/.ymllint' + +rst: + description: rst lint run over the gecko codebase + treeherder: + symbol: text(rst) + run: + mach: lint -v -l rst -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.rst' + - 'tools/lint/rst.yml' + +shellcheck: + description: shellcheck run over the gecko codebase + treeherder: + symbol: misc(shell) + run: + mach: lint -v -l shellcheck -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.sh' + - 'tools/lint/shellcheck.yml' + + +localization: + description: l10n tests for strings with errors + treeherder: + symbol: l10n(l1nt) + run: + mach: lint -v -l l10n -f treeherder * + attributes: + code-review: false + when: + files-changed: + - '**/locales/en-US/**' + - '**/l10n.toml' + - 'third_party/python/compare-locales/**' + - 'third_party/python/fluent/**' + - 'tools/lint/l10n.yml' + + +l10n-conflicts: + description: l10n tests for strings with errors and conflicts with cross-channel + treeherder: + symbol: l10n(l1nt-conflicts) + always-target: false + run-on-projects: [] + run: + mach: lint -W -l l10n -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/locales/en-US/**' + - '**/l10n.toml' + - 'third_party/python/compare-locales/**' + - 'third_party/python/fluent/**' + - 'tools/lint/l10n.yml' + + +clang-format: + description: clang-format on the C/C++ code + treeherder: + symbol: cpp(clang-format) + run: + mach: lint -v -l clang-format -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.cpp' + - '**/*.c' + - '**/*.cc' + - '**/*.h' + - '**/*.m' + - '**/*.mm' + - 'tools/lint/clang-format.yml' + dependencies: + linux64-clang-tidy: toolchain-linux64-clang-tidy + fetches: + linux64-clang-tidy: + - artifact: clang-tidy.tar.zst + dest: clang-tools + + +rustfmt: + description: rustfmt on the Rust code + treeherder: + symbol: rust(rustfmt) + run: + mach: lint -v -l rustfmt -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.rs' + - 'tools/lint/rustfmt.yml' + fetches: + toolchain: + - linux64-rust + + +clippy: + description: clippy on the Rust code + treeherder: + symbol: rust(clippy) + run: + mach: lint -v -l clippy -f treeherder -f json:/builds/worker/mozlint.json -j 1 . + when: + files-changed: + - '**/*.rs' + - '**/Cargo.toml' + - 'tools/lint/clippy.yml' + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-rust-size + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + +file-whitespace: + description: Check for trailing whitespaces and Windows CR + platform: lint/opt + treeherder: + symbol: pedantic(file-whitespace) + # We don't want patches to be backed out because of this, a follow up bug will be + # good enough + tier: 2 + run: + mach: lint -v -l file-whitespace -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.css' + - '**/*.dtd' + - '**/*.idl' + - '**/*.ftl' + - '**/*.h' + - '**/*.html' + - '**/*.json' + - '**/*.md' + - '**/*.mn' + - '**/*.properties' + - '**/*.py' + - '**/*.rs' + - '**/*.rst' + - '**/*.toml' + - '**/*.webidl' + - '**/*.yaml' + - '**/*.yml' + - '**/*.xhtml' + + +file-perm: + description: Check for incorrect permissions on source files + platform: lint/opt + treeherder: + symbol: pedantic(file-perm) + # We don't want patches to be backed out because of this, a follow up bug will be + # good enough + tier: 2 + run: + mach: lint -v -l file-perm -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.build' + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.flac' + - '**/*.h' + - '**/*.html' + - '**/*.idl' + - '**/*.js' + - '**/*.jsm' + - '**/*.json' + - '**/*.jsx' + - '**/*.m' + - '**/*.m4s' + - '**/*.md' + - '**/*.mjs' + - '**/*.mm' + - '**/*.mn' + - '**/*.mozbuild' + - '**/*.mp4' + - '**/*.png' + - '**/*.rs' + - '**/*.rst' + - '**/*.svg' + - '**/*.toml' + - '**/*.ttf' + - '**/*.wasm' + - '**/*.webidl' + - '**/*.xhtml' + - '**/*.xml' + - '**/*.yaml' + - '**/*.yml' + + +perfdocs-verify: + description: Verifies Performance Testing documentation + treeherder: + symbol: text(perfdocs) + run: + mach: lint -v -l perfdocs -f treeherder -f json:/builds/worker/mozlint.json . + worker: + env: + WORKSPACE: /builds/worker/checkouts/gecko/ + DECISION_TASK_ID: {task-reference: } + artifacts: + - type: file + name: public/perfdocs.diff + path: /builds/worker/diff.txt + when: + files-changed: + - 'devtools/perfdocs/**' + - 'taskcluster/**' + - '**/perftest.toml' + - 'python/mozperftest/**' + - 'testing/awsy/**' + - 'testing/raptor/**' + - 'testing/talos/**' + - 'testing/performance/**' + - 'testing/perfdocs/**' + - 'tools/lint/perfdocs.yml' + +condprof-addons-verify: + description: | + Verifies all xpi files needed by condprof customization sideloading addons are + included in the firefox-addons fetch task. + treeherder: + symbol: misc(condprof-addons) + run: + mach: | + lint -v -l condprof-addons -f treeherder -f json:/builds/worker/mozlint.json . + fetches: + fetch: + - firefox-addons + when: + files-changed: + - 'testing/condprofile/condprof/customization/*.json' + - 'taskcluster/ci/fetch/browsertime.yml' + +fluent-lint: + description: Check for problems with Fluent files. + platform: lint/opt + treeherder: + symbol: l10n(fluent) + run: + mach: lint --linter fluent-lint -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.ftl' + - 'tools/lint/fluent-lint.yml' + - 'tools/lint/fluent-lint/exclusions.yml' + +trojan-source: + description: Trojan Source attack - CVE-2021-42572 + platform: lint/opt + treeherder: + symbol: misc(trojan-source) + run: + mach: lint -v -l trojan-source --include-third-party -f treeherder -f json:/builds/worker/mozlint.json . + when: + files-changed: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.h' + - '**/*.py' + - '**/*.rs' + - 'tools/lint/trojan-source.yml' diff --git a/taskcluster/ci/source-test/node.yml b/taskcluster/ci/source-test/node.yml new file mode 100644 index 0000000000..f1df46b73d --- /dev/null +++ b/taskcluster/ci/source-test/node.yml @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +newtab-unit-tests: + description: newtab unit tests + platform: linux1804-64/opt + treeherder: + symbol: node(newtab) + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 1800 + require-build: + by-project: + autoland: + linux1804-64/opt: build-linux64/opt + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64-shippable/opt + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}' + command: > + cd /builds/worker/checkouts/gecko && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/aboutwelcome && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/asrouter && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/newtab && + rm -rf node_modules && + npm ci && + node bin/try-runner.js + when: + files-changed: + - "browser/components/newtab/**" + - "browser/components/aboutwelcome/**" + - "browser/components/asrouter/**" + +newtab-unit-tests-ccov: + description: newtab unit tests code coverage + platform: linux1804-64/opt + treeherder: + symbol: node(newtab-ccov) + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 1800 + artifacts: + - name: public/code-coverage-grcov.zip + path: /builds/worker/checkouts/gecko/browser/components/newtab/logs/coverage/code-coverage-grcov.zip + type: file + require-build: + by-project: + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64-shippable/opt + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}' + command: > + cd /builds/worker/checkouts/gecko && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/aboutwelcome && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/asrouter && + rm -rf node_modules && + npm ci && + cd /builds/worker/checkouts/gecko/browser/components/newtab && + rm -rf node_modules && + npm ci && + node bin/try-runner.js + # This should only run on mozilla-central, as the bot will only parse mozilla-central + # coverage anyways. + run-on-projects: ["mozilla-central"] + # Note the lack of a "when" rule here. This is to avoid seeing fluctuating + # coverage on coverage.moz.tools / searchfox depending on whether the latest + # push contained patches touching these files or not. + +devtools-tests: + description: devtools node-based tests (for instance jest) + platform: linux1804-64/opt + attributes: + code-review: true + treeherder: + symbol: node(devtools) + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + artifacts: + - type: file + name: public/code-review/issues.json + path: /builds/worker/issues.json + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}' + # Re-use rather than re-installing to avoid hitting/requiring + # nodejs servers for downloads + command: > + cp -r /build/node_modules_eslint node_modules && + cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + cd devtools/client/bin/ && + node devtools-node-test-runner.js --suite=all --artifact=/builds/worker/issues.json + when: + files-changed: + - 'devtools/client/**' + - 'devtools/shared/**' + +devtools-verify-bundle: + description: devtools bundle checker + platform: linux1804-64/opt + attributes: + code-review: true + treeherder: + symbol: node(devtools-bundle) + kind: test + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1200 + artifacts: + - type: file + name: public/code-review/issues.json + path: /builds/worker/issues.json + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}' + # Re-use rather than re-installing to avoid hitting/requiring + # nodejs servers for downloads + command: > + cp -r /build/node_modules_eslint node_modules && + cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + cd devtools/client/debugger && + yarn && + node bin/bundle.js && + cd ../../.. && + ./mach python taskcluster/scripts/misc/verify-devtools-bundle.py --output /builds/worker/issues.json + when: + files-changed: + - 'devtools/**' + +eslint-plugin-mozilla: + description: eslint-plugin-mozilla integration tests + platform: linux1804-64/opt + treeherder: + symbol: node(epm) + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}/tools/lint/eslint/eslint-plugin-mozilla' + command: > + cp -r /build/node_modules_eslint-plugin-mozilla node_modules && + npm run test + when: + files-changed: + - 'tools/lint/eslint/eslint-plugin-mozilla/**' diff --git a/taskcluster/ci/source-test/puppeteer.yml b/taskcluster/ci/source-test/puppeteer.yml new file mode 100644 index 0000000000..3bfd1a9645 --- /dev/null +++ b/taskcluster/ci/source-test/puppeteer.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux1804-64/opt + require-build: + by-project: + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64-shippable/opt + fetches: + build: + - target.tar.bz2 + toolchain: + - linux64-node + treeherder: + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: ubuntu1804-test} + max-run-time: 3600 + artifacts: + - type: file + name: public/remote/pup_errorsummary.json + path: /builds/worker/pup_errorsummary.json + optimization: + skip-unless-expanded: null + +puppeteer: + description: Puppeteer tests against Firefox CDP-based remote protocol + treeherder: + symbol: Pup(cdp) + run: # Bug 1651542: Use screenshot feature to warm-up the font cache before the actual test + using: run-task + command: > + cd $GECKO_PATH/ && + $MOZ_FETCHES_DIR/firefox/firefox --screenshot http://example.org && + ./mach puppeteer-test --ci -vv --binary $MOZ_FETCHES_DIR/firefox/firefox --cdp --headless --log-tbpl - --log-errorsummary /builds/worker/pup_errorsummary.json + +puppeteer-with-bidi: + description: Puppeteer tests against Firefox Webdriver BiDi remote protocol + treeherder: + symbol: Pup(wd) + run: # Bug 1651542: Use screenshot feature to warm-up the font cache before the actual test + using: run-task + command: > + cd $GECKO_PATH/ && + $MOZ_FETCHES_DIR/firefox/firefox --screenshot http://example.org && + ./mach puppeteer-test --ci -vv --binary $MOZ_FETCHES_DIR/firefox/firefox --headless --log-tbpl - --log-errorsummary /builds/worker/pup_errorsummary.json diff --git a/taskcluster/ci/source-test/python.yml b/taskcluster/ci/source-test/python.yml new file mode 100644 index 0000000000..3fd8b25b32 --- /dev/null +++ b/taskcluster/ci/source-test/python.yml @@ -0,0 +1,636 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux1804-64/opt + always-target: + by-platform: + macosx.*: false + default: true + worker-type: + by-platform: + linux1804-64.*: t-linux-xlarge-source + macosx1015-64.*: t-osx-1015-r8 + windows11-64.*: win11-64-2009-source + worker: + by-platform: + linux1804-64.*: + docker-image: {in-tree: "lint"} + max-run-time: 3600 + macosx.*: + max-run-time: 3600 + env: + PATH: "/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin" + default: + max-run-time: 3600 + treeherder: + kind: test + tier: 1 + run: + using: mach + when: + files-changed: + - 'config/mozunit/**' + - 'python/mach_commands.py' + +firefox-ci: + description: taskcluster/gecko_taskgraph unit tests + always-target: false + python-version: [3] + treeherder: + symbol: ci + run: + using: python-test + subsuite: ci + when: + files-changed: + - '.taskcluster.yml' + - 'taskcluster/ci/**' + - 'taskcluster/**/*.py' + - 'tools/tryselect/selectors/auto.py' + +fog: + description: Python unit tests for Firefox on Glean + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: fp + run: + using: python-test + subsuite: fog + when: + files-changed: + - 'toolkit/components/glean/**' + +mach: + description: python/mach unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: mach + run: + using: python-test + subsuite: mach + when: + files-changed: + - 'python/mach/**' + - 'python/sites/**' + +marionette-harness: + description: testing/marionette/harness unit tests + platform: + - linux1804-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: mnh + run: + using: python-test + subsuite: marionette-harness + when: + files-changed: + - 'testing/marionette/harness/**' + - 'testing/mozbase/mozlog/mozlog/**' + - 'testing/mozbase/mozlog/setup.py' + +mochitest-harness: + description: testing/mochitest unittests + platform: + - linux1804-64/opt + - linux1804-64/debug + - linux1804-64-asan/opt + always-target: false + require-build: + by-project: + try: + linux1804-64-asan/opt: build-linux64-asan/opt + linux1804-64/debug: build-linux64/debug + linux1804-64/opt: build-linux64/opt + default: + linux1804-64-asan/opt: build-linux64-asan/opt + linux1804-64/debug: build-linux64/debug + linux1804-64/opt: build-linux64-shippable/opt + treeherder: + symbol: py3(mch) + worker: + by-platform: + linux1804-64.*: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 3600 + run: + using: run-task + cwd: '{checkout}' + command: > + source /builds/worker/scripts/xvfb.sh && + start_xvfb '1600x1200x24' 0 && + export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox && + export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests && + ./mach python-test --subsuite mochitest + fetches: + build: + - target.tar.bz2 + - artifact: target.common.tests.tar.gz + dest: tests + - artifact: target.mochitest.tests.tar.gz + dest: tests + toolchain: + - linux64-fix-stacks + when: + files-changed: + - 'testing/mochitest/**' + - 'testing/mozbase/mozrunner/mozrunner/**' + - 'testing/mozbase/moztest/moztest/selftest/**' + - 'testing/mozharness/mozharness/base/log.py' + - 'testing/mozharness/mozharness/mozilla/structuredlog.py' + - 'testing/mozharness/mozharness/mozilla/testing/errors.py' + - 'testing/profiles/**' + + +mozbase: + description: testing/mozbase unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: mb + run: + using: python-test + subsuite: mozbase + when: + files-changed: + - 'testing/mozbase/**' + +mozharness: + description: mozharness integration tests + treeherder: + symbol: py3(mh) + run: + using: run-task + cache-dotcache: true + cwd: '{checkout}/testing/mozharness' + command: > + /usr/local/bin/tox -e py311-hg6.4 + when: + files-changed: + - 'testing/mozharness/**' + +mozlint: + description: python/mozlint unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: mozlint + tier: 2 + run: + using: python-test + subsuite: mozlint + dependencies: + clang-tidy: + by-platform: + linux1804-64/opt: toolchain-linux64-clang-tidy + macosx1015-64/opt: toolchain-macosx64-clang-tidy + windows11-64/opt: toolchain-win64-clang-tidy + fetches: + toolchain: + by-platform: + linux1804-64/opt: + - linux64-node + - linux64-rust + macosx1015-64/opt: + - macosx64-node + - macosx64-rust + windows11-64/opt: + - win64-node + - win64-rust + clang-tidy: + - artifact: clang-tidy.tar.zst + dest: clang-tools + when: + files-changed: + - 'python/mozlint/**' + - 'tools/lint/**' + +mozrelease: + description: python/mozrelease unit tests + python-version: [3] + treeherder: + symbol: release + run: + using: python-test + subsuite: mozrelease + when: + files-changed: + - 'python/mozrelease/**' + +mozterm: + description: python/mozterm unit tests + platform: + - linux1804-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: term + run: + using: python-test + subsuite: mozterm + when: + files-changed: + - 'python/mozterm/**' + +mozversioncontrol: + description: python/mozversioncontrol unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: vcs + run: + using: python-test + subsuite: mozversioncontrol + when: + files-changed: + - 'python/mozversioncontrol/**' + +raptor: + description: testing/raptor unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: rap + run: + using: python-test + subsuite: raptor + when: + files-changed: + - 'testing/raptor/**' + +reftest-harness: + description: layout/tools/reftest unittests + platform: + - linux1804-64/opt + - linux1804-64/debug + - linux1804-64-asan/opt + require-build: + by-project: + try: + linux1804-64-asan/opt: build-linux64-asan/opt + linux1804-64/debug: build-linux64/debug + linux1804-64/opt: build-linux64/opt + default: + linux1804-64-asan/opt: build-linux64-asan/opt + linux1804-64/debug: build-linux64/debug + linux1804-64/opt: build-linux64-shippable/opt + always-target: false + treeherder: + symbol: py3(ref) + worker: + by-platform: + linux1804-64.*: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 3600 + run: + using: run-task + cwd: '{checkout}' + command: > + source /builds/worker/scripts/xvfb.sh && + start_xvfb '1600x1200x24' 0 && + export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox && + export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests && + ./mach python-test --subsuite reftest + fetches: + build: + - target.tar.bz2 + - artifact: target.common.tests.tar.gz + dest: tests + - artifact: target.reftest.tests.tar.gz + dest: tests + toolchain: + - linux64-fix-stacks + when: + files-changed: + - 'layout/tools/reftest/**' + - 'testing/mozbase/mozrunner/mozrunner/**' + - 'testing/mozbase/moztest/moztest/selftest/**' + - 'testing/mozharness/mozharness/base/log.py' + - 'testing/mozharness/mozharness/mozilla/structuredlog.py' + - 'testing/mozharness/mozharness/mozilla/testing/errors.py' + +taskgraph-tests: + description: taskcluster/gecko_taskgraph unit tests + python-version: [3] + treeherder: + symbol: tg + run: + using: python-test + subsuite: taskgraph + when: + files-changed: + - 'taskcluster/**/*.py' + - 'python/mach/**/*.py' + +tryselect: + description: tools/tryselect unit tests + platform: + - linux1804-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: try + run: + using: python-test + subsuite: try + when: + files-changed: + - 'taskcluster/ci/config.yml' + - 'taskcluster/ci/test/**' + - 'taskcluster/gecko_taskgraph/transforms/**' + - 'tools/tryselect/**' + +mozbuild: + description: mozbuild unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: mbu + run: + using: python-test + subsuite: mozbuild + fetches: + toolchain: + by-platform: + linux1804-64/opt: + - linux64-node + macosx1015-64/opt: + - macosx64-node + windows11-64/opt: + - win64-node + - win64-mozmake + when: + files-changed: + - '**/moz.configure' + - 'build/moz.configure/**' + - 'config/tests/**' + - 'dom/bindings/mozwebidlcodegen/**' + - 'modules/libpref/init/**' + - 'modules/libpref/test/**' + - 'python/mach/**' + - 'python/mozboot/**' + - 'python/mozbuild/**' + - 'python/mozterm/**' + - 'python/mozversioncontrol/**' + - 'testing/mozbase/**' + - 'testing/xpcshell/xpcshellcommandline.py' + +mozperftest: + description: mozperftest unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + treeherder: + symbol: mpu + run: + mach: perftest-test + fetches: + toolchain: + by-platform: + linux1804-64/opt: + - linux64-node + macosx1015-64/opt: + - macosx64-node + windows11-64/opt: + - win64-node + when: + files-changed: + - 'testing/performance/**' + - 'python/mozperftest/**' + +fxms-schemas: + description: > + Ensure messaging-system schemas are up-to-date and run Experimenter + integration tests. + platform: + - linux1804-64/opt + treeherder: + symbol: py3(fxms) + require-build: + by-project: + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64/opt + fetches: + build: + - artifact: target.tar.bz2 + - artifact: target.common.tests.tar.gz + worker: + by-platform: + linux1804-64.*: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 3600 + run: + using: run-task + cwd: '{checkout}' + command: > + cd browser/components/asrouter/content-src/schemas && + env LD_LIBRARY_PATH=${MOZ_FETCHES_DIR}/firefox \ + ${MOZ_FETCHES_DIR}/bin/xpcshell \ + -g ${MOZ_FETCHES_DIR}/firefox \ + -a ${MOZ_FETCHES_DIR}/firefox/browser \ + extract-test-corpus.js + ../../../../../mach python -- make-schemas.py --check + when: + files-changed: + - 'browser/components/asrouter/content-src/schemas/extract-test-corpus.js' + - 'browser/components/asrouter/content-src/schemas/make-schemas.py' + - 'browser/components/asrouter/content-src/**/*.schema.json' + - 'browser/components/asrouter/modules/CFRMessageProvider.sys.mjs' + - 'browser/components/asrouter/modules/OnboardingMessageProvider.sys.mjs' + - 'browser/components/asrouter/modules/PanelTestProvider.sys.mjs' + +condprof: + description: testing/condprofile unit tests + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: condprof + run: + using: python-test + subsuite: condprof + when: + files-changed: + - 'testing/condprofile/condprof**' + - 'testing/condprofile/setup.py' + +featuregates: + description: featuregates Python unit tests + platform: + - linux1804-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: fg + run: + using: python-test + subsuite: featuregates + fetches: + toolchain: + by-platform: + linux1804-64/opt: + - linux64-node + windows11-64/opt: + - win64-node + when: + files-changed: + - 'toolkit/components/featuregates/**' + +skip-fails: + description: testing/skip-fails unit tests + always-target: false + python-version: [3] + treeherder: + symbol: sf + run: + using: python-test + subsuite: skip-fails + when: + files-changed: + - 'testing/skipfails.py' + - 'testing/test/**' + +talos: + description: testing/talos unit tests + platform: windows11-64/opt + python-version: [3] + treeherder: + symbol: tal + run: + using: python-test + subsuite: talos + when: + files-changed: + - 'testing/talos/**' + +telemetry-integration-tests: + description: pytest-based integration tests for Telemetry + always-target: false + platform: + - linux1804-64/opt + worker: + by-platform: + linux1804-64.*: + docker-image: {in-tree: "ubuntu1804-test"} + max-run-time: 3600 + require-build: + by-project: + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64-shippable/opt + treeherder: + symbol: tt(i) + tier: 3 + run-on-projects: [] + run: + using: run-task + cwd: '{checkout}' + command: > + source /builds/worker/scripts/xvfb.sh && + start_xvfb '1600x1200x24' 0 && + export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox && + ./mach python-test --subsuite telemetry-integration-tests + fetches: + build: + - target.tar.bz2 + - artifact: target.common.tests.tar.gz + dest: tests + - artifact: target.reftest.tests.tar.gz + dest: tests + when: + files-changed: + - 'toolkit/components/telemetry/**' + +telemetry-python: + description: Python unit tests for Telemetry + platform: + - linux1804-64/opt + - macosx1015-64/opt + - windows11-64/opt + python-version: [3] + treeherder: + symbol: tp + run: + using: python-test + subsuite: telemetry-python + fetches: + toolchain: + by-platform: + linux1804-64/opt: + - linux64-node + macosx1015-64/opt: + - macosx64-node + windows11-64/opt: + - win64-node + when: + files-changed: + - 'toolkit/components/telemetry/**' + +webext: + description: WebExtensions python utilities unit tests + platform: + - linux1804-64/opt + python-version: [3] + treeherder: + symbol: webext-py + tier: 2 + run: + using: python-test + subsuite: webext-python + when: + files-changed: + - 'toolkit/components/extensions/webidl-api/**' + - 'toolkit/components/extensions/schemas/*.json' + - 'browser/components/extensions/schemas/*.json' + - 'mobile/android/components/extensions/schemas/*.json' + +xpcom: + description: xpcom unit tests + platform: + - linux1804-64/opt + python-version: [3] + treeherder: + symbol: xpcom + run: + using: python-test + subsuite: xpcom + when: + files-changed: + - 'third_party/python/ply/**' + - 'xpcom/components/*.py' + - 'xpcom/components/test/**' + - 'xpcom/ds/tools/**' + - 'xpcom/ds/test/**' + - 'xpcom/idl-parser/**' diff --git a/taskcluster/ci/source-test/shadow-scheduler.yml b/taskcluster/ci/source-test/shadow-scheduler.yml new file mode 100644 index 0000000000..97d1b1f30b --- /dev/null +++ b/taskcluster/ci/source-test/shadow-scheduler.yml @@ -0,0 +1,178 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: gecko-decision/opt + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 3600 + artifacts: + - type: file + name: public/shadow-scheduler/optimized-tasks.json + path: /builds/worker/optimized-tasks.json + env: + DECISION_TASK_ID: {task-reference: } + treeherder: + kind: other + tier: 3 + run-on-projects: ['autoland-only'] + run: + using: mach + mach: taskgraph optimized -v --json -p task-id=$DECISION_TASK_ID --output-file /builds/worker/optimized-tasks.json + sparse-profile: taskgraph + +relevant_tests: + description: Runs the relevant_tests optimization strategy instead of the default. + treeherder: + symbol: SS(relevant_tests) + index: + product: source + job-name: shadow-scheduler-relevant_tests + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.relevant_tests + +bugbug_tasks_medium: + description: Runs the bugbug_tasks_medium optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_tasks_medium) + index: + product: source + job-name: shadow-scheduler-bugbug_tasks_medium + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_tasks_medium + +bugbug_tasks_high: + description: Runs the bugbug_tasks_high optimization strategy (with a high confidence threshold) instead of the default. + treeherder: + symbol: SS(bugbug_tasks_high) + index: + product: source + job-name: shadow-scheduler-bugbug_tasks_high + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_tasks_high + +bugbug_debug_disperse: + description: Runs the bugbug_debug_disperse optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_debug_disperse) + index: + product: source + job-name: shadow-scheduler-bugbug_debug_disperse + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_debug_disperse + +bugbug_disperse_low: + description: Runs the bugbug_disperse_low optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_disperse_low) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_low + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_low + +bugbug_disperse_medium: + description: Runs the bugbug_disperse_medium optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_disperse_med) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_medium + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_medium + +bugbug_disperse_reduced_medium: + description: Runs the bugbug_disperse_reduced_medium optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_di_r_med_noun) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_reduced_medium + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_reduced_medium + +bugbug_reduced_manifests_config_selection_low: + description: Runs the bugbug_reduced_manifests_config_selection_low optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_r_c_s_low) + index: + product: source + job-name: shadow-scheduler-bugbug_reduced_manifests_config_selection_low + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_reduced_manifests_config_selection_low + +bugbug_reduced_manifests_config_selection_medium: + description: Runs the bugbug_reduced_manifests_config_selection_medium optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_r_c_s_med) + index: + product: source + job-name: shadow-scheduler-bugbug_reduced_manifests_config_selection_medium + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_reduced_manifests_config_selection_medium + +bugbug_disperse_medium_no_unseen: + description: Runs the bugbug_disperse_medium_no_unseen optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_disp_med_noun) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_medium_no_unseen + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_medium_no_unseen + +bugbug_disperse_medium_only_one: + description: Runs the bugbug_disperse_medium_only_one optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_disp_med_one) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_medium_only_one + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_medium_only_one + +bugbug_disperse_high: + description: Runs the bugbug_disperse_high optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_disperse_high) + index: + product: source + job-name: shadow-scheduler-bugbug_disperse_high + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_disperse_high + +bugbug_reduced: + description: Runs the bugbug_reduced optimization strategy instead of the default. + treeherder: + symbol: SS(bugbug_reduced) + index: + product: source + job-name: shadow-scheduler-bugbug_reduced + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_reduced + +bugbug_reduced_high: + description: Runs the bugbug_reduced optimization strategy (with a low confidence threshold) instead of the default. + treeherder: + symbol: SS(bugbug_reduced_high) + index: + product: source + job-name: shadow-scheduler-bugbug_reduced_high + worker: + env: + TASKGRAPH_OPTIMIZE_STRATEGIES: gecko_taskgraph.optimize:experimental.bugbug_reduced_high diff --git a/taskcluster/ci/source-test/taskgraph.yml b/taskcluster/ci/source-test/taskgraph.yml new file mode 100644 index 0000000000..132d5e488c --- /dev/null +++ b/taskcluster/ci/source-test/taskgraph.yml @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +diff: + description: Generates diffs of CI changes + platform: linux1804-64/opt + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + # Uncomment to upload logs of the generations as artifacts. + # env: + # XDG_CACHE_HOME: "/builds/worker/artifacts/cache" + artifacts: + - type: directory + name: public/taskgraph + path: /builds/worker/artifacts + attributes: + code-review: true + always-target: true + treeherder: + kind: test + platform: gecko-decision/opt + symbol: tgdiff + tier: 2 + run: + using: run-task + cwd: '{checkout}' + command: >- + mkdir -p /builds/worker/artifacts/diffs && + ./mach taskgraph target --fast -p "taskcluster/test/params" -o "/builds/worker/artifacts/diffs/diff.txt" --diff {base_rev} && + python taskcluster/scripts/misc/summarize-tgdiff.py /builds/worker/artifacts/diffs/ 20 + when: + files-changed: + - 'taskcluster/ci/**' + - 'taskcluster/**/*.py' diff --git a/taskcluster/ci/source-test/vendor.yml b/taskcluster/ci/source-test/vendor.yml new file mode 100644 index 0000000000..c6594c6f68 --- /dev/null +++ b/taskcluster/ci/source-test/vendor.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: t-linux-xlarge-source + attributes: + code-review: true + worker: + docker-image: {in-tree: static-analysis-build} + max-run-time: 1200 + artifacts: + - type: file + name: public/code-review/issues.json + path: /builds/worker/issues.json + treeherder: + kind: test + tier: 1 + platform: lint/opt + run: + using: run-task + +rust: + description: Rust vendoring validation + treeherder: + symbol: rust(vendor) + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + run: + cwd: '{checkout}' + command: >- + ./mach vendor rust --issues-json /builds/worker/issues.json + fetches: + toolchain: + - linux64-cargo-vet + - linux64-rust + when: + files-changed: + - 'Cargo.lock' + - '**/Cargo.toml' + - 'python/mozbuild/mozbuild/vendor/vendor_rust.py' + - 'supply-chain/**' + - 'third_party/rust/**' diff --git a/taskcluster/ci/source-test/webcompat.yml b/taskcluster/ci/source-test/webcompat.yml new file mode 100644 index 0000000000..66ff7910a6 --- /dev/null +++ b/taskcluster/ci/source-test/webcompat.yml @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux1804-64/opt + require-build: + by-project: + try: + linux1804-64/opt: build-linux64/opt + default: + linux1804-64/opt: build-linux64-shippable/opt + fetches: + build: + - target.tar.bz2 + toolchain: + - linux64-geckodriver + run-on-projects: [] + treeherder: + kind: test + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: ubuntu1804-test} + max-run-time: 1800 + optimization: + skip-unless-expanded: null + +interventions: + description: webcompat intervention tests + treeherder: + symbol: wc(I) + tier: 3 + python-version: [3] + run: + using: mach + # Need to start Xvfb if we remove --headless + mach: test-interventions --headless --binary $MOZ_FETCHES_DIR/firefox/firefox --webdriver-binary $MOZ_FETCHES_DIR/geckodriver --log-tbpl - diff --git a/taskcluster/ci/source-test/webidl.yml b/taskcluster/ci/source-test/webidl.yml new file mode 100644 index 0000000000..eec7a75e38 --- /dev/null +++ b/taskcluster/ci/source-test/webidl.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +test: + description: WebIDL parser tests + platform: lint/opt + treeherder: + symbol: Wp + kind: test + tier: 1 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + mach: webidl-parser-test --verbose + when: + files-changed: + - 'dom/bindings/parser/runtests.py' + - 'dom/bindings/parser/WebIDL.py' + - 'dom/bindings/parser/tests/**' + - 'other-licenses/ply/**' diff --git a/taskcluster/ci/source-test/wpt-manifest.yml b/taskcluster/ci/source-test/wpt-manifest.yml new file mode 100644 index 0000000000..06b29b60f5 --- /dev/null +++ b/taskcluster/ci/source-test/wpt-manifest.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux64/opt + treeherder: + kind: test + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + +upload: + description: Generate and store the web-platform-tests manifest + treeherder: + symbol: Wm + index: + product: source + job-name: manifest-upload + rank: build_date + run: + using: run-task + cwd: '{checkout}' + command: > + ./mach wpt-manifest-update --config testing/web-platform/wptrunner.ini --no-download + && tar -cvzf manifests.tar.gz -C testing/web-platform/ meta/MANIFEST.json mozilla/meta/MANIFEST.json + worker: + artifacts: + - type: file + path: /builds/worker/checkouts/gecko/manifests.tar.gz + name: public/manifests.tar.gz + + max-run-time: 3600 + when: + files-changed: + - 'testing/web-platform/tests/**' + - 'testing/web-platform/mozilla/tests/**' diff --git a/taskcluster/ci/source-test/wpt-metadata.yml b/taskcluster/ci/source-test/wpt-metadata.yml new file mode 100644 index 0000000000..b2c54d62b9 --- /dev/null +++ b/taskcluster/ci/source-test/wpt-metadata.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: lint/opt + treeherder: + kind: test + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + +summary: + description: Summarize wpt metadata + treeherder: + symbol: wpt-meta + index: + product: source + job-name: source-wpt-metadata-summary + run: + using: mach + mach: wpt-metadata-summary --out-dir=/builds/worker/artifacts + worker: + artifacts: + - type: directory + path: /builds/worker/artifacts + name: public + max-run-time: 2700 + when: + files-changed: + - 'testing/web-platform/meta/**' + - 'testing/web-platform/mozilla/meta/**' + - 'testing/web-platform/metasummary.py' + + +fission-regression: + description: Summarize fission regressions in wpt + treeherder: + symbol: wpt-fis + index: + product: source + job-name: source-wpt-fission-regressions + run: + using: mach + mach: wpt-fission-regressions --all-json /builds/worker/artifacts/regressions.json --untriaged /builds/worker/artifacts/untriaged.txt + worker: + artifacts: + - type: directory + path: /builds/worker/artifacts + name: public + max-run-time: 2700 + when: + files-changed: + - 'testing/web-platform/meta/**' + - 'testing/web-platform/mozilla/meta/**' + - 'testing/web-platform/fissionregressions.py' diff --git a/taskcluster/ci/spidermonkey/kind.yml b/taskcluster/ci/spidermonkey/kind.yml new file mode 100644 index 0000000000..358ffeeb3f --- /dev/null +++ b/taskcluster/ci/spidermonkey/kind.yml @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.spidermonkey:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + treeherder: + kind: build + tier: 1 + index: + product: firefox + worker-type: b-linux-gcp + run: + using: spidermonkey + when: + files-changed: + # any when.files-changed specified below in a job will be + # appended to this list + - build/** + - config/** + - configure.py + - dom/bindings/** + - intl/icu/** + - js/moz.configure + - js/public/** + - js/src/** + - layout/tools/reftest/reftest/** + - Makefile.in + - media/webrtc/trunk/tools/gyp/** + - memory/** + - mfbt/** + - modules/fdlibm/** + - modules/zlib/src/** + - mozglue/** + - moz.build + - moz.configure + - nsprpub/** + - python/** + - taskcluster/moz.build + - taskcluster/ci/spidermonkey/kind.yml + - testing/mozbase/** + - testing/web-platform/** + - test.mozbuild + - toolkit/mozapps/installer/package-name.mk + - toolkit/mozapps/installer/upload-files.mk + fetches: + toolchain: + by-worker-type: + .*-b-win2022: + - win64-clang + - win64-rust + default: + - linux64-clang + - linux64-gcc + - linux64-rust + - linux64-dump_syms + - sysroot-x86_64-linux-gnu + +jobs-from: + - linux.yml + - windows.yml diff --git a/taskcluster/ci/spidermonkey/linux.yml b/taskcluster/ci/spidermonkey/linux.yml new file mode 100644 index 0000000000..cf7604bf5a --- /dev/null +++ b/taskcluster/ci/spidermonkey/linux.yml @@ -0,0 +1,606 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker: + max-run-time: 10800 + docker-image: {in-tree: debian12-amd64-build} + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "0" + run: + tooltool-downloads: public + +sm-package-linux64/opt: + description: "Spidermonkey source package and test" + index: + job-name: sm-package-linux64-opt + treeherder: + symbol: SM(pkg) + platform: linux64/opt + run: + using: spidermonkey-package + spidermonkey-variant: plain + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-gcc + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-plain-linux64/debug: + description: "Spidermonkey Plain" + index: + job-name: sm-plain-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(p) + run: + spidermonkey-variant: plaindebug + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-plain-linux64/opt: + description: "Spidermonkey Plain" + index: + job-name: sm-plain-linux64-opt + treeherder: + symbol: SM(p) + platform: linux64/opt + run: + spidermonkey-variant: plain + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-gcc + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-smoosh-linux64/debug: + description: "Spidermonkey SmooshMonkey" + index: + job-name: sm-smoosh-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(smoosh) + tier: 3 + run: + spidermonkey-variant: smooshdebug + run-on-projects: [] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + when: + files-changed: + - 'third_party/rust/jsparagus/**' + - 'js/src/frontend/Frontend2.cpp' + - 'js/src/frontend/Frontend2.h' + - 'js/src/frontend/SourceNotes.h' + - 'js/src/frontend/smoosh/**' + - 'js/public/Symbol.h' + - 'js/src/vm/AsyncFunctionResolveKind.h' + - 'js/src/vm/BytecodeFormatFlags.h' + - 'js/src/vm/CheckIsObjectKind.h' + - 'js/src/vm/FunctionFlags.h' + - 'js/src/vm/FunctionPrefixKind.h' + - 'js/src/vm/GeneratorAndAsyncKind.h' + - 'js/src/vm/GeneratorResumeKind.h' + - 'js/src/vm/Opcodes.h' + - 'js/src/vm/ThrowMsgKind.h' + - 'js/src/vm/StencilEnums.h' + +sm-smoosh-linux64/opt: + description: "Spidermonkey SmooshMonkey" + index: + job-name: sm-smoosh-linux64-opt + treeherder: + symbol: SM(smoosh) + platform: linux64/opt + tier: 3 + run: + spidermonkey-variant: smoosh + run-on-projects: [] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + when: + files-changed: + - 'third_party/rust/jsparagus/**' + - 'js/src/frontend/Frontend2.cpp' + - 'js/src/frontend/Frontend2.h' + - 'js/src/frontend/SourceNotes.h' + - 'js/src/frontend/smoosh/**' + - 'js/public/Symbol.h' + - 'js/src/vm/AsyncFunctionResolveKind.h' + - 'js/src/vm/BytecodeFormatFlags.h' + - 'js/src/vm/CheckIsObjectKind.h' + - 'js/src/vm/FunctionFlags.h' + - 'js/src/vm/FunctionPrefixKind.h' + - 'js/src/vm/GeneratorAndAsyncKind.h' + - 'js/src/vm/GeneratorResumeKind.h' + - 'js/src/vm/Opcodes.h' + - 'js/src/vm/ThrowMsgKind.h' + - 'js/src/vm/StencilEnums.h' + +sm-nojit-linux64/opt: + description: "Spidermonkey no JIT" + index: + job-name: sm-nojit-linux64-opt + treeherder: + symbol: SM(nojit) + platform: linux64/opt + run: + spidermonkey-variant: nojit + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-pbl-linux64/opt: + description: "Spidermonkey PBL" + index: + job-name: sm-pbl-linux64-opt + treeherder: + symbol: SM(pbl) + platform: linux64/opt + tier: 2 + run: + spidermonkey-variant: pbl + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-plain-linux32/debug: + description: "Spidermonkey Plain" + index: + job-name: sm-plain-linux32-debug + treeherder: + platform: linux32/debug + symbol: SM(p) + run: + spidermonkey-variant: plaindebug + spidermonkey-platform: linux + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + +sm-arm-sim-linux32/debug: + description: "Spidermonkey ARM sim" + index: + job-name: sm-arm-sim-linux32-debug + treeherder: + platform: linux32/debug + symbol: SM(arm) + run: + spidermonkey-variant: arm-sim + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + +sm-arm64-sim-linux64/debug: + description: "Spidermonkey ARM64 sim" + index: + job-name: sm-arm64-sim-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(arm64) + run: + spidermonkey-variant: arm64-sim + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-asan-linux64/opt: + description: "Spidermonkey Address Sanitizer" + index: + job-name: sm-linux64-asan-opt + treeherder: + symbol: SM(asan) + platform: linux64/opt + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + run: + spidermonkey-variant: asan + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-compacting-linux64/debug: + description: "Spidermonkey Compacting" + index: + job-name: sm-compacting-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(cgc) + run: + spidermonkey-variant: compacting + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-linux64-wasi/opt: + description: "Spidermonkey WASI opt build" + index: + job-name: sm-linux64-wasi-opt + treeherder: + platform: linux64/opt + symbol: SM(wasi) + tier: 2 + run: + spidermonkey-variant: wasi + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +# Like sm-linux64-wasi/opt but with Intl/ICU enabled. +sm-linux64-wasi-intl/opt: + description: "Spidermonkey WASI Intl opt build" + index: + job-name: sm-linux64-wasi-intl-opt + treeherder: + platform: linux64/opt + symbol: SM(wasi-intl) + tier: 2 + run: + spidermonkey-variant: wasi-intl + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +sm-wasm-no-experimental-linux64/debug: + description: "Spidermonkey WebAssembly No experimental features" + index: + job-name: sm-wasm-no-experimental-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(wasm-no-experimental) + tier: 2 + run: + spidermonkey-variant: wasm-noexperimental + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-gcc + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-linux64-wasi-pbl/opt: + description: "Spidermonkey WASI opt build with PBL" + index: + job-name: sm-pbl-linux64-wasi-opt + treeherder: + platform: linux64/opt + symbol: SM(wasi-pbl) + tier: 2 + run: + spidermonkey-variant: wasi-pbl + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + +sm-msan-linux64/opt: + description: "Spidermonkey Memory Sanitizer" + index: + job-name: sm-linux64-msan-opt + treeherder: + symbol: SM(msan) + platform: linux64/opt + tier: 3 + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + run: + spidermonkey-variant: msan + # Disable by default by allow try pushes to explicitly request. + run-on-projects: [] + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-tsan-linux64/opt: + description: "Spidermonkey Thread Sanitizer" + index: + job-name: sm-linux64-tsan-opt + treeherder: + symbol: SM(tsan) + platform: linux64/opt + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + run: + spidermonkey-variant: tsan + fetches: + toolchain: + - linux64-clang + - linux64-gcc + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust-dev + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-rootanalysis-linux64/debug: + description: "Spidermonkey Root Analysis" + index: + job-name: sm-rootanalysis-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(r) + run: + spidermonkey-variant: rootanalysis + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-nonunified-linux64/debug: + description: "Spidermonkey Non-Unified Debug" + index: + job-name: sm-nonunified-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(nu) + run: + spidermonkey-variant: nonunified + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-fuzzing-linux64/opt: + description: "Spidermonkey Fuzzing" + index: + job-name: sm-linux64-fuzzing + treeherder: + platform: linux64/opt + symbol: SM(f) + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + run: + spidermonkey-variant: fuzzing + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-fuzzilli-linux64/debug: + description: "Spidermonkey Fuzzilli debug" + index: + job-name: sm-linux64-fuzzilli-debug + treeherder: + platform: linux64/debug + symbol: SM(fuzzilli) + worker: + env: + MOZ_JS_UPLOAD_BINARIES_DEFAULT: "1" + run: + spidermonkey-variant: fuzzilli + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-gdb-linux64/debug: + description: "Spidermonkey GDB Pretty-printers" + index: + job-name: sm-gdb-linux64 + worker: + docker-image: {in-tree: gdb-test} + treeherder: + platform: linux64/debug + tier: 2 + symbol: SM(gdb) + run: + spidermonkey-variant: gdb + fetches: + toolchain: + - linux64-clang + - linux64-gcc + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-rt-linux64/debug: + description: "SpiderMonkey Records&Tuples" + index: + job-name: sm-rt-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(rt) + run: + spidermonkey-variant: rtdebug + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + +sm-temporal-linux64/debug: + description: "SpiderMonkey Temporal" + index: + job-name: sm-temporal-linux64-debug + treeherder: + platform: linux64/debug + symbol: SM(temporal) + run: + spidermonkey-variant: temporaldebug + fetches: + toolchain: + - linux64-clang + - linux64-cbindgen + - linux64-dump_syms + - linux64-breakpad-injector + - linux64-minidump-stackwalk + - linux64-llvm-symbolizer + - linux64-rust + - linux64-pkgconf + - sysroot-x86_64-linux-gnu diff --git a/taskcluster/ci/spidermonkey/windows.yml b/taskcluster/ci/spidermonkey/windows.yml new file mode 100644 index 0000000000..2fa90deb8e --- /dev/null +++ b/taskcluster/ci/spidermonkey/windows.yml @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-win2022 + worker: + max-run-time: 10800 + run: + tooltool-downloads: public + +sm-plain-win64/debug: + description: "Spidermonkey Plain win64 debug" + index: + job-name: sm-plain-win64-debug + treeherder: + platform: windows2012-64/debug + symbol: SM(p) + run: + spidermonkey-variant: plaindebug + spidermonkey-platform: win64 + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs + +sm-plain-win32/debug: + description: "Spidermonkey Plain win32 debug" + index: + job-name: sm-plain-win32-debug + treeherder: + platform: windows2012-32/debug + symbol: SM(p) + tier: 2 + run: + spidermonkey-variant: plaindebug + spidermonkey-platform: win32 + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs + +sm-plain-win64/opt: + description: "Spidermonkey Plain win64 opt" + index: + job-name: sm-plain-win64-opt + treeherder: + platform: windows2012-64/opt + symbol: SM(p) + run: + spidermonkey-variant: plain + spidermonkey-platform: win64 + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs + +sm-plain-win32/opt: + description: "Spidermonkey Plain win32 opt" + index: + job-name: sm-plain-win32-opt + treeherder: + platform: windows2012-32/opt + symbol: SM(p) + tier: 2 + run: + spidermonkey-variant: plain + spidermonkey-platform: win32 + run-on-projects: ['mozilla-central'] + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs + +sm-compacting-win64/debug: + description: "Spidermonkey Compacting win64 debug" + index: + job-name: sm-compacting-win64-debug + treeherder: + platform: windows2012-64/debug + symbol: SM(cgc) + run: + spidermonkey-variant: compacting + spidermonkey-platform: win64 + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs + +sm-compacting-win32/debug: + description: "Spidermonkey Compacting win32 debug" + index: + job-name: sm-compacting-win32-debug + treeherder: + platform: windows2012-32/debug + symbol: SM(cgc) + tier: 2 + run: + spidermonkey-variant: compacting + spidermonkey-platform: win32 + run-on-projects: [] + fetches: + toolchain: + - win64-cbindgen + - win64-clang + - win64-rust + - win64-mozmake + - vs diff --git a/taskcluster/ci/startup-test/kind.yml b/taskcluster/ci/startup-test/kind.yml new file mode 100644 index 0000000000..bc62e7e764 --- /dev/null +++ b/taskcluster/ci/startup-test/kind.yml @@ -0,0 +1,129 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + # We want to test the signed version of a build, to make sure + # any startup problems or crashes caused by signing are caught + # This means we depend on a different kind depending on the platform + # non-system python + - toolchain + # linux + - build-signing + # mac + - repackage + # windows + - repackage-signing + +transforms: + - gecko_taskgraph.transforms.startup_test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: startup-test + description: Check whether or not a product crashes on startup + run-on-projects: ['mozilla-central'] + worker: + max-run-time: 3600 + run: + sparse-profile: startup-test + attributes: + build_type: opt + treeherder: + symbol: SUT + kind: test + tier: 1 + use-system-python: false + +jobs: + linux32: + dependencies: + build-signing: build-signing-linux-shippable/opt + run: + using: run-task + cwd: "{checkout}" + extra-config: + upstream_kind: build-signing + upstream_artifact: target.tar.bz2 + binary: firefox + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: ubuntu1804-test} + shipping-product: firefox + attributes: + build_platform: linux-shippable + treeherder: + platform: linux-shippable/opt + + linux64: + dependencies: + build-signing: build-signing-linux64-shippable/opt + run: + using: run-task + cwd: "{checkout}" + extra-config: + upstream_kind: build-signing + upstream_artifact: target.tar.bz2 + binary: firefox + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: ubuntu1804-test} + shipping-product: firefox + attributes: + build_platform: linux64-shippable + treeherder: + platform: linux64-shippable/opt + + macosx64: + dependencies: + repackage: repackage-macosx64-shippable/opt + run: + using: mach + python-version: 3 + extra-config: + upstream_kind: repackage + upstream_artifact: target.dmg + binary: Contents/MacOS/firefox + worker-type: t-osx-1015-r8 + shipping-product: firefox + attributes: + build_platform: macosx64-shippable + treeherder: + platform: macosx64-shippable/opt + + win32: + dependencies: + repackage-signing: repackage-signing-win32-shippable/opt + run: + using: mach + python-version: 3 + extra-config: + upstream_kind: repackage-signing + upstream_artifact: target.installer.exe + binary: core/firefox.exe + worker-type: win11-64-2009-source + shipping-product: firefox + attributes: + build_platform: win32-shippable + treeherder: + platform: windows2012-32-shippable/opt + + win64: + dependencies: + repackage-signing: repackage-signing-win64-shippable/opt + run: + using: mach + python-version: 3 + extra-config: + upstream_kind: repackage-signing + upstream_artifact: target.installer.exe + binary: core/firefox.exe + worker-type: win11-64-2009-source + shipping-product: firefox + attributes: + build_platform: win64-shippable + treeherder: + platform: windows2012-64-shippable/opt diff --git a/taskcluster/ci/static-analysis-autotest/kind.yml b/taskcluster/ci/static-analysis-autotest/kind.yml new file mode 100644 index 0000000000..6a01325598 --- /dev/null +++ b/taskcluster/ci/static-analysis-autotest/kind.yml @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + index: + product: firefox + worker: + skip-artifacts: true + max-run-time: 3600 + env: + PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest + use-system-python: false + run-on-projects: ['mozilla-central'] + run: + using: mozharness + actions: [static-analysis-autotest] + script: mozharness/scripts/fx_desktop_build.py + treeherder: + symbol: Sa + kind: build + tier: 1 + +jobs: + linux64-st-autotest/debug: + description: "Linux64 Debug Static Analysis Autotest" + index: + job-name: linux64-st-autotest-debug + treeherder: + platform: linux64/debug + worker-type: t-linux-large + worker: + docker-image: {in-tree: static-analysis-build} + env: + # clang-tidy needs a recent libstdc++, which can be found in the clang + # toolchain. + LD_LIBRARY_PATH: /builds/worker/checkouts/gecko/clang/lib + PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest + run: + config: + - builds/releng_base_firefox.py + - builds/releng_sub_linux_configs/64_stat_and_debug.py + tooltool-downloads: public + keep-artifacts: false + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy + - linux64-rust + - linux64-sccache + - linux64-cbindgen + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + + win64-st-autotest/debug: + description: "Win64 Debug Static Analysis Autotest" + index: + job-name: win64-st-autotest-debug + treeherder: + platform: windows2012-64/debug + tier: 2 + worker-type: b-win2022 + run: + config: + - builds/releng_base_firefox.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + - builds/taskcluster_sub_win64/debug.py + extra-config: + mozconfig_variant: debug + use-sccache: false + fetches: + toolchain: + - win64-clang + - win64-rust + - win64-cbindgen + - win64-clang-tidy + - win64-nasm + - win64-node + - win64-dump_syms + - win64-winchecksec + - vs + - windows-rs + - sysroot-wasm32-wasi diff --git a/taskcluster/ci/system-symbols-reprocess/kind.yml b/taskcluster/ci/system-symbols-reprocess/kind.yml new file mode 100644 index 0000000000..5b81817188 --- /dev/null +++ b/taskcluster/ci/system-symbols-reprocess/kind.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +# we need that to apply grouping only on jobs that have "build_platform", so we +# dont fail for the "fetch" kind that does is a dependency but does not have it +only-for-attributes: + - build_platform + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.reprocess_symbols + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - system-symbols + - system-symbols-upload + - fetch + +jobs: + system-symbols-reprocess: + from-deps: + kinds: ["system-symbols", "system-symbols-upload"] + group-by: platform + description: Reprocess Symbols on Crash-Stats + attributes: + cron: true + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 7200 + env: + # {level} gets replaced in the reprocess_symbols transform + CRASHSTATS_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-reprocess" + DUMP_SYMS: "/bin/false" + run: + using: run-task + command: {artifact-reference: "${MOZ_FETCHES_DIR}/symbol-scrapers/reprocess.sh "} + fetches: + fetch: + - symbol-scrapers + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-reprocess diff --git a/taskcluster/ci/system-symbols-upload/kind.yml b/taskcluster/ci/system-symbols-upload/kind.yml new file mode 100644 index 0000000000..dd9462a84b --- /dev/null +++ b/taskcluster/ci/system-symbols-upload/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.upload_symbols + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - system-symbols + +jobs: + system-symbols-upload: + description: Upload Symbols + from-deps: {} + attributes: + cron: true + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1200 + env: + # {level} gets replaced in the upload_symbols transform + SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload" + run: + using: mach + mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py --ignore-missing"} + sparse-profile: upload-symbols + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload diff --git a/taskcluster/ci/system-symbols/kind.yml b/taskcluster/ci/system-symbols/kind.yml new file mode 100644 index 0000000000..0090454a68 --- /dev/null +++ b/taskcluster/ci/system-symbols/kind.yml @@ -0,0 +1,320 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + cron: true + # This enables uploading symbols. + enable-full-crashsymbols: true + fetches: + toolchain: + - linux64-dump_syms + treeherder: + platform: symbols/opt + kind: other + tier: 2 + worker-type: b-linux-gcp + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + +jobs: + mac: + description: Extract macOS system symbols found in crashreports + attributes: + build_platform: macos + index: + product: system-symbols + job-name: mac + type: generic + worker: + env: + PROCESSED_PACKAGES_INDEX: gecko.v2.mozilla-central.latest.system-symbols.mac + PROCESSED_PACKAGES_PATH: public/build/processed-packages.gz + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-mac} + max-run-time: 28800 + run: + using: run-task + command: >- + ./run.sh + treeherder: + symbol: system-symbols-mac + fetches: + toolchain: + - linux64-libdmg + win: + description: Extract windows system symbols found in crashreports + attributes: + build_platform: windows + index: + product: system-symbols + job-name: win + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-win} + max-run-time: 3600 + run: + using: run-task + command: >- + ./run.sh + treeherder: + symbol: system-symbols-win + archlinux: + description: Extract ArchLinux system symbols found in crashreports + attributes: + build_platform: archlinux + index: + product: system-symbols + job-name: archlinux + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh arch + treeherder: + symbol: sys-symbols-archlinux + fetches: + fetch: + - symbol-scrapers + debian: + description: Extract Debian system symbols found in crashreports + attributes: + build_platform: debian + index: + product: system-symbols + job-name: debian + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh debian + treeherder: + symbol: sys-symbols-debian + fetches: + fetch: + - symbol-scrapers + firefox-snap: + description: Extract Firefox Snap symbols found in crashreports + attributes: + build_platform: firefox-snap + index: + product: system-symbols + job-name: firefox-snap + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh firefox-snap + treeherder: + symbol: firefox-symbols-snap + fetches: + fetch: + - symbol-scrapers + flatpak: + description: Extract Firefox Flatpak symbols found in crashreports + attributes: + build_platform: firefox-flatpak + index: + product: system-symbols + job-name: firefox-flatpak + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh firefox-flatpak + treeherder: + symbol: firefox-symbols-fltpk + fetches: + fetch: + - symbol-scrapers + gnome-sdk-snap: + description: Extract GNOME SDK Snap symbols found in crashreports + attributes: + build_platform: gnome-sdk-snap + index: + product: system-symbols + job-name: gnome-sdk-snap + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh gnome-sdk-snap + treeherder: + symbol: gnome-sdk-symbols-snap + fetches: + fetch: + - symbol-scrapers + fedora: + description: Extract Fedora system symbols found in crashreports + attributes: + build_platform: fedora + index: + product: system-symbols + job-name: fedora + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh fedora + treeherder: + symbol: sys-symbols-fedora + fetches: + fetch: + - symbol-scrapers + opensuse: + description: Extract OpenSUSE system symbols found in crashreports + attributes: + build_platform: opensuse + index: + product: system-symbols + job-name: opensuse + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh opensuse + treeherder: + symbol: sys-symbols-opensuse + fetches: + fetch: + - symbol-scrapers + ubuntu: + description: Extract Ubuntu system symbols found in crashreports + attributes: + build_platform: ubuntu + index: + product: system-symbols + job-name: ubuntu + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh ubuntu + treeherder: + symbol: sys-symbols-ubuntu + fetches: + fetch: + - symbol-scrapers + win-gfx: + description: Extract symbols from closed-source Windows graphics drivers + attributes: + build_platform: windows-graphics + index: + product: system-symbols + job-name: win-gfx + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-win-gfx} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh + treeherder: + symbol: sys-symbols-win-gfx + fetches: + fetch: + - symbol-scrapers + alpine: + description: Extract Alpine Linux system symbols found in crashreports + attributes: + build_platform: alpine + index: + product: system-symbols + job-name: alpine + type: generic + worker: + artifacts: + - name: public/build + type: directory + path: /builds/worker/artifacts/ + docker-image: {in-tree: system-symbols-linux-scraper} + max-run-time: 14400 + run: + using: run-task + command: >- + ./run.sh alpine + treeherder: + symbol: sys-symbols-alpine + fetches: + fetch: + - symbol-scrapers diff --git a/taskcluster/ci/test/awsy.yml b/taskcluster/ci/test/awsy.yml new file mode 100644 index 0000000000..c3cafe7e9a --- /dev/null +++ b/taskcluster/ci/test/awsy.yml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: awsy + max-run-time: 7200 + instance-size: xlarge + virtualization: + by-test-platform: + windows.*(?:-shippable)?-qr/.*: virtual-with-gpu + default: virtual + allow-software-gl-layers: false + mozharness: + script: awsy_script.py + config: + by-test-platform: + windows.*: + - awsy/taskcluster_windows_config.py + macosx.*/opt: + - awsy/macosx_config.py + default: + - awsy/linux_config.py + tier: default + test-manifest-loader: null # don't load tests in the taskgraph + + +awsy: + description: Are we slim yet - tp5 pageset + treeherder-symbol: SY(sy) + run-on-projects: [] + +awsy-tp6: + description: Are we slim yet - tp6 pageset + treeherder-symbol: SY(sy-tp6) + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition + windows.*-32.*/opt: [] + (linux|windows|mac)(?!.*shippable).*/opt: [] + default: [trunk] + mozharness: + extra-options: + - --tp6 + +awsy-dmd: + description: Are we slim yet - dmd enabled + treeherder-symbol: SY(sy-d) + run-on-projects: [] + mozharness: + extra-options: + - --dmd + +awsy-base: + description: Are we slim yet - about:blank base case + treeherder-symbol: SY(ab) + run-on-projects: + by-test-platform: + .*-devedition/.*: [] # don't run on devedition. + windows.*-32.*/opt: [] + (linux|windows|mac)(?!.*shippable).*/opt: [] + default: [trunk] + tier: default + mozharness: + extra-options: + - --base + +awsy-base-dmd: + description: Are we slim yet - about:blank base case - dmd enabled + treeherder-symbol: SY(ab-d) + run-on-projects: [] + mozharness: + extra-options: + - --base + - --dmd diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml new file mode 100644 index 0000000000..b1422e1b2f --- /dev/null +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -0,0 +1,918 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: + by-test-platform: + .*-qr/.*: 2400 + default: 1800 + suite: raptor + run-on-projects: + by-test-platform: + linux.*shippable[^-qr].*: [mozilla-central] + default: [] + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + virtualization: hardware + mozharness: + script: raptor_script.py + config: + by-test-platform: + macosx.*: + - raptor/mac_config.py + windows.*: + - raptor/windows_config.py + default: + - raptor/linux_config.py + extra-options: + - --conditioned-profile=settled + fetches: + toolchain: + by-app: + custom-car: + by-test-platform: + # this satisfied taskgraph for now, but once mac/win is implemented + # then linux will be the default + linux.*: + - linux64-custom-car + - linux64-minidump-stackwalk + - linux64-fix-stacks + windows10-64.*: + - win64-custom-car + - win64-minidump-stackwalk + - win64-fix-stacks + macosx1015.*: + - macosx64-custom-car + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + macosx1400.*: + - macosx-arm64-custom-car + - macosx64-aarch64-minidump-stackwalk + - macosx64-aarch64-fix-stacks + default: [] + default: + by-test-platform: + windows.*-32.*.: + - win32-minidump-stackwalk + - win32-fix-stacks + windows10-64.*: + - win64-minidump-stackwalk + - win64-fix-stacks + macos.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + +browsertime-tp6: + <<: &tp6-defaults + description: Raptor (browsertime) tp6 page-load tests + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + safari: + - .*macos.*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + max-run-time: + by-subtest: + imgur: 4800 + default: 4000 + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-subtest: + netflix: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + (macosx1015|linux|windows10-64).*shippable-qr.*: [trunk] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + treeherder-symbol: Btime(tp6) + raptor: + apps: [firefox, chrome] + test: tp6 + run-visual-metrics: true + subtests: + - buzzfeed + - ebay + - espn + - expedia + - [facebook, fb] + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - imdb + - imgur + - linkedin + - microsoft + - netflix + - nytimes + - office + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - wikia + - youtube + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-responsiveness: + <<: *tp6-defaults + raptor: + apps: [firefox, chrome] + test: tp6 + run-visual-metrics: true + subtests: + - cnn-nav + - [facebook-nav, fbnav] + - [reddit-billgates-ama, rdt-ama] + - [reddit-billgates-post-1, rdt-post-1] + - [reddit-billgates-post-2, rdt-post-2] + mozharness: + extra-options: + by-test-platform: + windows10-.*: + - --cold + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --extra-profiler-run + default: + - --cold + - --conditioned-profile=settled + - --extra-profiler-run + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + +browsertime-tp6-essential: + <<: *tp6-defaults + <<: &tp6-essential-defaults + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --collect-perfstats + - --extra-profiler-run + default: + - --chimera + - --conditioned-profile=settled + - --extra-profiler-run + raptor: + apps: [firefox, chrome, custom-car] + test: tp6 + run-visual-metrics: + by-test-platform: + macosx1400.*shippable.*: false + # Bug 1845014 Resolve osx 1300 vismet permafails + default: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + tier: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: + by-app: + firefox: 1 + default: 3 + +browsertime-tp6-bytecode: + <<: *tp6-defaults + <<: *tp6-essential-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + - [bing-search, bing] + - cnn + - fandom + - [google-slides, gslides] + - instagram + - twitter + - wikipedia + - yahoo-mail + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + default: [] + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + default: + - --chimera + - --browsertime + - --conditioned-profile=settled + - --test-bytecode-cache + - --extra-profiler-run + tier: + by-app: + firefox: 2 + default: 3 + treeherder-symbol: Btime-cache(tp6) + +browsertime-benchmark: + description: Raptor (browsertime) Benchmark tests + raptor: + apps: [firefox, chrome, safari, custom-car] + subtests: + by-test-platform: + macosx1400.*shippable.*: + - [speedometer, sp] + - [speedometer3, sp3] + default: + by-app: + firefox: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [twitch-animation, tw-a] + - [unity-webgl, ugl] + - [webaudio, wa] + safari: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + # - [matrix-react-bench, mrb] Perma fails on osx 10.15.x machines (Bug 1807754) + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + # - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535) + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + custom-car: + - [speedometer, sp] + - [speedometer3, sp3] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite-ramp, mm-hr] + chrome: + by-test-platform: + macosx1015.*shippable.*: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite, mm-h] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + default: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + - [matrix-react-bench, mrb] + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite, mm-h] + - [motionmark-htmlsuite-ramp, mm-hr] + - [speedometer, sp] + - [speedometer3, sp3] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + - [webaudio, wa] + treeherder-symbol: Btime() + tier: + by-app: + firefox: + by-subtest: + motionmark-animometer: + by-test-platform: + linux1804-64-clang-trunk-qr/opt: 2 + default: 1 + default: + by-test-platform: + windows10-64-ccov.*/.*: 3 + linux64-ccov.*/.*: 3 + default: 2 + default: + by-subtest: + unity-webgl: 3 + default: 2 + run-on-projects: + by-subtest: + speedometer3: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + .*nightlyasrelease.*: [autoland, mozilla-central] + default: [] + custom-car: + by-test-platform: + (linux|windows10-64).*shippable-qr.*: [mozilla-central] + default: [] + default: [] + unity-webgl: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + # Disabled on autoland due to bug 1720063, and bug 1795826 + .*64.*shippable-qr/.*: [mozilla-central] + default: [] + default: [] + speedometer: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + .*nightlyasrelease.*: [autoland] + default: [] + custom-car: + by-test-platform: + (linux|windows10-64).*shippable-qr.*: [mozilla-central] + default: [] + default: [] + default: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [trunk] + default: [] + default: [] + max-run-time: + by-subtest: + ares6: 1500 + assorted-dom: + by-test-platform: + .*-qr/.*: 2100 + default: 1500 + jetstream2: 8000 + matrix-react-bench: 1500 + motionmark-htmlsuite: 1500 + motionmark-htmlsuite-ramp: 3000 + motionmark-animometer-ramp: 2100 + unity-webgl: 2400 + speedometer3: + by-test-platform: + macosx1400.*shippable.*: 4200 + default: 2100 + speedometer: 2100 + default: 900 + mozharness: + extra-options: + by-test-platform: + macosx1400.*shippable.*: + - --extra-profiler-run + - --page-timeout=900000 + default: + - --extra-profiler-run + +browsertime-benchmark-wasm: + description: Raptor (browsertime) WASM benchmarks + raptor: + apps: [firefox, chrome, safari] + subtests: + by-app: + firefox: + - [wasm-godot, godot] + - [wasm-godot-baseline, godot-b] + - [wasm-godot-optimizing, godot-i] + - [wasm-misc, wm] + - [wasm-misc-baseline, wm-b] + - [wasm-misc-optimizing, wm-i] + safari: + # - [wasm-misc, wm] Perma fails (Bug 1809533) + - [wasm-godot, godot] + default: + - [wasm-misc, wm] + - [wasm-godot, godot] + treeherder-symbol: Btime() + tier: 2 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + default: [] + max-run-time: + by-subtest: + wasm-godot: 1500 + wasm-godot-baseline: 1500 + wasm-misc-baseline: 1500 + default: 900 + mozharness: + extra-options: + - --extra-profiler-run + +browsertime-youtube-playback: + description: Browsertime YouTube Playback on Firefox + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + - [youtube-playback-av1-sfr, ytp-av1-sfr] + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + - [youtube-playback-vp9-sfr, ytp-vp9-sfr] + - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + tier: 2 + limit-platforms: + by-subtest: + youtube-playback-av1-sfr: + - macosx1015-64-shippable-qr/opt + default: [] + max-run-time: + by-subtest: + youtube-playback-hfr: 3600 + youtube-playback-widevine.*: 3600 + default: 2700 + +browsertime-youtube-playback-power: + description: Browsertime YouTube Playback on Firefox with power usage tracking + raptor: + apps: [firefox] + subtests: + by-app: + firefox: + + - [youtube-playback-h264-1080p30, ytp-h264-1080p30] + - [youtube-playback-h264-1080p60, ytp-h264-1080p60] + - [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30] + - [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60] + - [youtube-playback-v9-1080p30, ytp-v9-1080p30] + - [youtube-playback-v9-1080p60, ytp-v9-1080p60] + - [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30] + - [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60] + treeherder-symbol: Btime() + run-on-projects: + by-test-platform: + macosx.*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + max-run-time: 2700 + +browsertime-tp6-live: + <<: *tp6-defaults + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=15 + - --chimera + - --live-sites + - --extra-profiler-run + raptor: + apps: [firefox, chrome, safari] + test: tp6 + run-visual-metrics: true + subtests: + by-app: + safari: + - amazon + - [bing-search, bing] + - ebay + # - [facebook, fb] Perma fails on osx 10.15.7 + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + # - instagram # (see bug 1736999) + - linkedin + - microsoft + # - netflix # Perma fails on osx 10.15.7 (Bug 1802922) + # - office (site loads blank page if not signed in) + # - outlook Perma fails on osx 10.15.7 + - paypal + - pinterest + - reddit + - tumblr + - twitch + # - twitter Perma fails on osx 10.15.7 + - wikipedia + - yahoo-mail + - youtube + chrome: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (see bugs 1825337 & 1817420) + - instagram + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + default: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + - instagram + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-live-sheriffed: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - cnn + description: Raptor (browsertime) tp6 on live-sites + mozharness: + extra-options: + - --browser-cycles=5 + - --chimera + - --live-sites + - --extra-profiler-run + - --browsertime-arg=maxLoadTime=240000 + run-on-projects: + by-test-platform: + (linux|windows10-64|macos)(?!.*shippable).*: [] + (linux|windows|macosx1015)(?!.*-qr).*: [] + windows.*-32.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime-live(tp6) + +browsertime-tp6-profiling: + <<: *tp6-defaults + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: [amazon] + description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling + mozharness: + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central] + default: [] + tier: 2 + treeherder-symbol: Btime-Prof(tp6) + +browsertime-custom: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [[process-switch, ps]] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central] + default: [] + tier: 1 + treeherder-symbol: Btime(ps) + +browsertime-first-install: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [welcome] + description: Raptor (browsertime) custom pageload tests + max-run-time: 2700 + mozharness: + extra-options: + - --chimera + - --extra-profiler-run + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk] + default: [] + tier: 1 + treeherder-symbol: Btime(welcome) + +browsertime-regression-tests: + raptor: + apps: [firefox] + subtests: [[constant-regression, regress-c]] + description: Raptor (browsertime) tests for our regression detection system + max-run-time: 700 + run-on-projects: [autoland] + limit-platforms: + - .*linux(?!.*shippable)(?!.*clang).* + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: [] + default: [] + tier: 2 + treeherder-symbol: Btime(regressor) + +browsertime-upload: + raptor: + apps: [firefox, chrome, safari] + subtests: + - [upload, upload-h2] + - upload-h3 + description: Raptor (browsertime) http2, http3 file upload performance test + max-run-time: 2700 + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime(upload) + fetches: + fetch: + - upload-speed-test-file + +browsertime-pageload-benchmark: + raptor: + apps: [firefox] + subtests: [tp6-bench] + description: Raptor (browsertime) TP6 pageload benchmark test + max-run-time: 4500 + mozharness: + extra-options: + - --browser-cycles=5 + run-on-projects: [] + tier: 2 + treeherder-symbol: Btime(tp6-bench) + +browsertime-speculative: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [connect] + description: Raptor (browsertime) speculative connect pageload tests + max-run-time: 420 + run-on-projects: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland] + default: [] + tier: 2 + treeherder-symbol: Btime(speculative) + +browsertime-throttled: + raptor: + apps: [firefox] + run-visual-metrics: true + subtests: [throttled] + description: Raptor (browsertime) throttled network pageload tests + max-run-time: 420 + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime(throttled) + +browsertime-tp6-webextensions: + <<: *tp6-defaults + <<: *tp6-essential-defaults + description: Raptor (browsertime) tp6 page-load tests with webextensions installed on Firefox Desktop + raptor: + apps: [firefox] + test: tp6 + run-visual-metrics: true + subtests: + - amazon + fetches: + fetch: + - firefox-addons + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + macosx1100.*shippable.*: [mozilla-central] + .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk-only] + default: [] + default: [] + mozharness: + extra-options: + by-test-platform: + windows10-64.*shippable.*-qr.*: + - --chimera + - --browsertime + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled-webext + default: + - --chimera + - --browsertime + - --conditioned-profile=settled-webext + # TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3. + tier: 3 + treeherder-symbol: Btime-webext(tp6) + +browsertime-indexeddb: + description: Raptor (browsertime) tests for IndexedDB + max-run-time: 2700 + treeherder-symbol: Btime(idb) + limit-platforms: + by-app: + chrome: + - .*shippable.* + chromium: + - .*shippable.* + custom-car: + - .*64.*shippable.* + default: [] + raptor: + apps: [firefox, chrome, chromium] + run-visual-metrics: false + subtests: + - addMab1 + - addMabN + - addMar1 + - addMarN + - addMbl1 + - addMblN + - addkAB1 + - addkABN + - addkAR1 + - addkARN + - addkBL1 + - addkBLN + - getkeyrng + run-on-projects: + by-app: + firefox: + by-test-platform: + windows.*-32.*: [] + .*64.*shippable-qr/.*: [autoland, mozilla-central] + default: [] + default: [] + tier: + by-app: + firefox: 2 + default: 3 + mozharness: + extra-options: + - --extra-profiler-run diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml new file mode 100644 index 0000000000..89bbc5fb6b --- /dev/null +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -0,0 +1,456 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + max-run-time: 3600 + suite: raptor + variants: + - geckoview-no-fission + run-on-projects: [] + tier: 3 + test-manifest-loader: null # don't load tests in the taskgraph + target: + by-app: + fenix: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v3.firefox-android.apks.fenix-nightly.latest.arm64-v8a + name: build/target.arm64-v8a.apk + default: + index: mobile.v3.firefox-android.apks.fenix-nightly.latest.armeabi-v7a + name: build/target.armeabi-v7a.apk + refbrow: + by-test-platform: + android-hw.*-aarch64.*/.*: + index: mobile.v2.reference-browser.raptor.latest.arm64-v8a + name: target.arm64-v8a.apk + default: + index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a + name: target.armeabi-v7a.apk + default: geckoview_example.apk + virtualization: hardware + raptor: + activity: + by-app: + fenix: org.mozilla.fenix.IntentReceiverActivity + geckoview: org.mozilla.geckoview_example.GeckoViewActivity + refbrow: org.mozilla.reference.browser.BrowserTestActivity + default: None + binary-path: + by-app: + fenix: org.mozilla.fenix + geckoview: org.mozilla.geckoview_example + refbrow: org.mozilla.reference.browser.raptor + default: None + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + # Bug 1635749, disable window recorder temporarily + - --browsertime-no-ffwindowrecorder + optimization: + skip-unless-backstop: null + limit-platforms: + by-app: + chrome-m: + - .*shippable-qr.* + cstm-car-m: + - .*shippable-qr.* + default: [] + fetches: + toolchain: + by-app: + cstm-car-m: + - android-custom-car + - linux64-minidump-stackwalk + - linux64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + +browsertime-tp6m: + <<: &tp6m-defaults + description: Raptor (browsertime) tp6 page-load tests on android + max-run-time: + by-subtest: + reddit: + by-app: + geckoview: 7200 + default: 3600 + imdb: + by-app: + geckoview: 7200 + default: 3600 + default: 3600 + mozharness: + extra-options: + - --chimera + tier: + by-app: + geckoview: 1 + refbrow: 2 + default: 3 + treeherder-symbol: Btime(tp6m) + raptor: + apps: [geckoview, fenix, chrome-m, refbrow] + run-visual-metrics: true + test: tp6m + subtests: + - amazon + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - cnn + - [cnn-ampstories, cnn-amp] + - dailymail + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - [facebook-cristiano, fb-cris] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - reddit + - sina + - [stackoverflow, stacko] + - web-de + - wikipedia + - youtube + run-on-projects: + by-app: + geckoview: + by-variant: + geckoview-no-fission: + by-test-platform: + android-hw-a51.*shippable-qr.*: [trunk-only] + default: [] + default: + by-subtest: + wikipedia: + by-test-platform: + android-hw-a51.*shippable-qr.*: [] + default: [] + google-search-restaurants: + by-test-platform: + android-hw-a51.*shippable-qr.*: [] + default: [] + default: + by-test-platform: + android-hw-a51.*shippable-qr.*: [] + default: [] + default: [] + +browsertime-tp6m-essential: + <<: *tp6m-defaults + run-without-variant: false + raptor: + apps: [geckoview, fenix, chrome-m, refbrow] + run-visual-metrics: true + test: tp6m + subtests: + - allrecipes + - [amazon-search, amazon-s] + - espn + # Bug 1682027 - Disabled due to high failure rate + # - facebook + - google + - [microsoft-support, micros-sup] + - [youtube-watch, youtube-w] + run-on-projects: + by-app: + geckoview: + by-subtest: + allrecipes: + by-test-platform: + android-hw-a51.*shippable.*: [trunk] + default: [] + default: + by-test-platform: + android-hw-a51.*shippable.*: [trunk-only] + default: [] + default: [] + tier: + by-app: + geckoview: 1 + refbrow: 2 + default: 3 + +browsertime-tp6m-webextensions: + <<: *tp6m-defaults + description: Raptor (browsertime) tp6 page-load tests with webextensions installed on android + treeherder-symbol: Btime-webext(tp6m) + run-without-variant: false + mozharness: + extra-options: + - --chimera + - --browsertime-no-ffwindowrecorder + - --conditioned-profile=settled-webext + - --browser-cycles=5 + raptor: + apps: [fenix] + run-visual-metrics: true + test: tp6m + subtests: + - [amazon-search, amazon-s] + fetches: + fetch: + - firefox-addons + run-on-projects: + by-app: + fenix: + by-test-platform: + android-hw-a51.*shippable.*: [mozilla-central] + default: [] + default: [] + # TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3. + tier: 3 + +browsertime-youtube-playback-mobile: + description: Browsertime YouTube Playback on Android + treeherder-symbol: Btime(ytp) + run-without-variant: false + raptor: + apps: [fenix, geckoview, refbrow] + subtests: + by-test-platform: + android-hw-a51.*: + - [youtube-playback-h264-sfr, ytp-h264-sfr] + - [youtube-playback-hfr, ytp-hfr] + # Bug 1699469 - Disabled until a crash is fixed. + # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] + # - [youtube-playback-widevine-hfr, ytp-widevine-hfr] + test-url-param: + by-subtest: + youtube-playback-h264-sfr: # remove high resolution tests + exclude=1,2, + 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38 + youtube-playback-hfr: + by-test-platform: + android-hw-a51.*: # remove VP9(1-34) and AV1(51-74) tests + exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34, + 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74 + default: "" + youtube-playback-widevine-hfr: + by-test-platform: + android-hw-a51.*: # remove VP9(1-34) tests + exclude=1,2, + 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34 + default: "" + default: "" + run-on-projects: + by-app: + geckoview: + by-test-platform: + android-hw-a51.*shippable.*: [mozilla-central] + default: [] + default: [] + max-run-time: 3600 + +browsertime-tp6m-live: + <<: *tp6m-defaults + run-without-variant: false + raptor: + apps: [chrome-m, fenix, geckoview] + run-visual-metrics: true + test: tp6m + subtests: + - allrecipes + - amazon + - [amazon-search, amazon-s] + - bing + - [bing-search-restaurants, bing-s-r] + - booking + - booking-sf + - cnn + - [cnn-ampstories, cnn-amp] + - dailymail + - discord + - [ebay-kleinanzeigen, ebay-k] + - [ebay-kleinanzeigen-search, ebay-k-s] + - espn + - expedia + - facebook + - [facebook-cristiano, fb-cris] + - [fashionbeans, fashbea] + - google + - [google-accounts, gacc] + - [google-maps, gmaps] + - [google-search-restaurants, gsearch-r] + - instagram + - imdb + - [imdb-firefox, imdb-f] + - [medium-article, medium] + - [microsoft-support, micros-sup] + - nytimes + - [people-article, people] + - reddit + - [reddit-thread, reddit-thr] + - rumble-fox + - [stackoverflow, stacko] + - [stackoverflow-question, stack-q] + - [urbandictionary-define, urbict-def] + - web-de + - [wikia-marvel, wikia-m] + - wikipedia + - youtube + - [youtube-watch, youtube-w] + mozharness: + extra-options: + - --live-sites + - --chimera + run-on-projects: + by-subtest: + cnn-ampstories: + by-app: + geckoview: + by-test-platform: + android-hw-a51.*shippable-qr.*: [autoland] + default: [] + default: [] + default: [] + tier: 3 + treeherder-symbol: Btime-live(tp6m) + +browsertime-tp6m-profiling: + description: Raptor (browsertime) tp6 page-load tests on android with Gecko profiling + run-without-variant: false + raptor: + apps: [geckoview] + test: tp6m + subtests: [amazon, youtube] + treeherder-symbol: Btime-Prof(tp6m) + mozharness: + script: raptor_script.py + config: + - raptor/android_hw_config.py + extra-options: + - --chimera + - --gecko-profile + run-on-projects: + by-test-platform: + android-hw-(a51).*aarch64-shippable-qr/opt: [mozilla-central] + default: [] + +browsertime-benchmark-speedometer-mobile: + description: Browsertime Speedometer on Android + run-without-variant: false + raptor: + apps: [geckoview, fenix, chrome-m, cstm-car-m] + tier: + by-app: + fenix: 1 + default: 3 + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + android-hw-a51.*shippable-qr.*: [trunk] + default: [] + cstm-car-m: + by-test-platform: + android-hw-p6.*shippable-qr.*: [] + android-hw-s21.*shippable-qr.*: [] + default: [mozilla-central] + default: [] + treeherder-symbol: Btime(sp) + mozharness: + extra-options: + - --test=speedometer + +browsertime-benchmark-speedometer3-mobile: + description: Browsertime Speedometer 3 on Android + run-without-variant: false + raptor: + apps: [geckoview, fenix, chrome-m, cstm-car-m] + tier: + by-app: + fenix: 2 + geckoview: 2 + default: 3 + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + android-hw-a51.*shippable-qr.*: [trunk-only] + default: [] + cstm-car-m: + by-test-platform: + android-hw-p6.*shippable-qr.*: [] + android-hw-s21.*shippable-qr.*: [] + default: [mozilla-central] + default: [] + treeherder-symbol: Btime(sp3) + mozharness: + extra-options: + - --test=speedometer3 + +browsertime-benchmark-motionmark: + description: Raptor (browsertime) Motionmark benchmark tests + run-without-variant: false + raptor: + apps: [fenix, chrome-m, geckoview] + subtests: + - [motionmark-animometer, mm-a] + - [motionmark-animometer-ramp, mm-ar] + - [motionmark-htmlsuite, mm-h] + - [motionmark-htmlsuite-ramp, mm-hr] + run-on-projects: + by-app: + chrome-m: [] + default: + by-test-platform: + android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central] + default: [] + treeherder-symbol: Btime() + tier: 2 + +browsertime-benchmark-unity-webgl-mobile: + description: Browsertime Unity WebGL on Android + run-without-variant: false + raptor: + apps: [geckoview, refbrow, fenix, chrome-m] + tier: + by-app: + geckoview: 2 + default: 3 + treeherder-symbol: Btime(ugl) + run-on-projects: + by-app: + refbrow: [] + geckoview: + by-test-platform: + # Disabled on autoland due to bug 1720063, and bug 1795826 + android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central] + default: [] + default: + by-test-platform: + android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central] + default: [] + max-run-time: + by-test-platform: + anrdroid-hw-a51.*: 1500 + default: 900 + mozharness: + extra-options: + - --test=unity-webgl + +browsertime-power: + description: Browsertime Power Usage Tests on Android + run-without-variant: false + raptor: + apps: [geckoview, fenix, refbrow] + subtests: + - idle + - idle-bg + - [speedometer, sp] + tier: 2 + run-on-projects: [] + treeherder-symbol: Btime-P(power) + mozharness: + extra-options: + - --power-test diff --git a/taskcluster/ci/test/compiled.yml b/taskcluster/ci/test/compiled.yml new file mode 100644 index 0000000000..08434d5ac7 --- /dev/null +++ b/taskcluster/ci/test/compiled.yml @@ -0,0 +1,135 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + run-without-variant: false + variants: + - 1proc + virtualization: virtual + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + supports-artifact-builds: false + +cppunittest: + description: "CPP Unit Tests" + suite: cppunittest + treeherder-symbol: cppunit + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + tier: default + run-on-projects: + by-variant: + 1proc: built-projects + default: [] + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: GTest + instance-size: xlarge + run-on-projects: + by-variant: + 1proc: + by-test-platform: + .*android.*: built-projects + .*(devedition|tsan|wayland).*: [] + linux1804-32-qr/opt: [] # try only, shippable on m-c + linux1804-32-qr/debug: ['mozilla-central'] # try and m-c + linux1804-32-shippable-qr/opt: ['mozilla-central'] # try and m-c + default: built-projects + default: [] + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + tier: default + +jittest: + description: "JIT Test run" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-variant: + 1proc: + by-test-platform: + android-hw-.*-aarch64-qr/debug: ['mozilla-central', 'release'] + android-hw-.*-arm7-qr/debug: ['mozilla-central', 'release'] + default: built-projects + default: [] + chunks: + by-test-platform: + windows.*-ccov-qr/opt: 8 + windows.*: 1 + macosx.*/opt: 1 + macosx.*/debug: 3 + android.*: 10 + default: 6 + max-run-time: + by-test-platform: + windows.*-ccov.*/.*: 7200 + macosx.*-ccov.*/.*: 7200 + android-hw.*: 5400 + default: 3600 + mozharness: + chunked: + by-test-platform: + windows.*: false + macosx.*: false + default: true + target: + by-test-platform: + android-.*: geckoview-test_runner.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default + +jittest-all: + description: "JIT Test run with all flags for android arm7 opt" + suite: jittest + treeherder-symbol: Jit + run-on-projects: + by-test-platform: + android-hw-.*-arm7(?:-shippable)?-qr/.*: ['mozilla-central', 'release'] + default: [] + chunks: 10 + max-run-time: 3600 + mozharness: + chunked: true + extra-options: + by-test-platform: + android.*: + - --jittest-flags=all + default: None + target: + by-test-platform: + android-.*: geckoview-test_runner.apk + default: null + tier: + by-test-platform: + android-hw.*: 2 + default: default diff --git a/taskcluster/ci/test/firefox-ui.yml b/taskcluster/ci/test/firefox-ui.yml new file mode 100644 index 0000000000..c45bb3d138 --- /dev/null +++ b/taskcluster/ci/test/firefox-ui.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: firefox-ui + max-run-time: 5400 + test-manifest-loader: null # don't load tests in the taskgraph + run-on-projects: built-projects + mozharness: + script: firefox_ui_tests.py + config: + by-test-platform: + windows.*: + - firefox_ui_tests/taskcluster_windows.py + macosx.*: + - firefox_ui_tests/taskcluster.py + - firefox_ui_tests/taskcluster_mac.py + linux.*: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + +firefox-ui-functional: + description: "Firefox-ui-tests functional run" + treeherder-symbol: Fxfn(en-US) + tier: 2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml new file mode 100644 index 0000000000..514fb648c4 --- /dev/null +++ b/taskcluster/ci/test/kind.yml @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.test:loader + +kind-dependencies: + - build + - build-signing + - repackage-signing-shippable-l10n-msix + - repackage-signing-msix + - repackage-signing + - repackage + - fetch + - toolchain + +transforms: + - gecko_taskgraph.transforms.test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +# Each stanza in a file pointed to by 'jobs-from' describes a particular test +# suite or sub-suite. These are processed through the transformations described +# above to produce a bunch of tasks. See the schema in +# `taskcluster/gecko_taskgraph/transforms/test/__init__.py` for a description of the fields +# used in these files. + +# Adding a new test type or running tests on a new platform? Be sure to review +# https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy + +jobs-from: + - awsy.yml + - browsertime-desktop.yml + - browsertime-mobile.yml + - compiled.yml + - firefox-ui.yml + - marionette.yml + - misc.yml + - mochitest.yml + - reftest.yml + - talos.yml + - web-platform.yml + - xpcshell.yml + +job-defaults: + attributes: + retrigger: true + require-signed-extensions: + by-release-type: + release|esr.*: true + beta: + by-test-platform: + .*-devedition/.*: false + default: true + default: false + mozharness: + requires-signed-builds: + by-test-platform: + macosx1100.*: true + windows1.*: + by-variant: + msix: true + default: false + default: false + fetches: + toolchain: + by-test-platform: + win.*: + - win32-minidump-stackwalk + - win32-fix-stacks + macosx1015.*: + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + macosx1100.*: + - macosx64-aarch64-minidump-stackwalk + - macosx64-aarch64-fix-stacks + macosx1400.*: + - macosx64-aarch64-minidump-stackwalk + - macosx64-aarch64-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks diff --git a/taskcluster/ci/test/marionette.yml b/taskcluster/ci/test/marionette.yml new file mode 100644 index 0000000000..d586a6cf24 --- /dev/null +++ b/taskcluster/ci/test/marionette.yml @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: marionette + mozharness: + script: marionette.py + config: + by-test-platform: + macosx.*: + - marionette/prod_config.py + - marionette/mac_taskcluster_config.py + windows.*: + - marionette/windows_taskcluster_config.py + default: + - marionette/prod_config.py + - remove_executables.py + test-manifest-loader: null # don't load tests in the taskgraph + tier: + by-variant: + headless: 2 + default: default + variants: + - webrender-sw + - headless + run-on-projects: + by-variant: + headless: + by-test-platform: + (linux.*64)(-shippable)?-qr/opt: built-projects + default: [] + webrender-sw: + by-test-platform: + .*wayland.*: [] + .*/debug: built-projects + default: [] + default: + by-test-platform: + .*(tsan|wayland).*: [] + default: built-projects + virtualization: virtual + +marionette: + description: "Marionette unittest run" + treeherder-symbol: Mn + max-run-time: + by-test-platform: + .*asan.*: 5400 + .*ccov.*: 3600 + .*debug.*: 3600 + default: 2700 + instance-size: default + chunks: 1 diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml new file mode 100644 index 0000000000..b31860a165 --- /dev/null +++ b/taskcluster/ci/test/misc.yml @@ -0,0 +1,269 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + test-manifest-loader: null # don't load tests in the taskgraph + +geckoview-junit: + description: "Geckoview junit run" + suite: geckoview-junit + variants: ["geckoview-fission", "no-fission"] + treeherder-symbol: gv-junit + loopback-video: true + target: geckoview-androidTest.apk + max-run-time: 3600 + run-on-projects: + by-variant: + geckoview-fission: ['trunk'] + no-fission: built-projects + default: [] + tier: default + mozharness: + script: android_emulator_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + extra-options: + - --test-suite=geckoview-junit + +telemetry-tests-client: + description: "Telemetry tests client run" + suite: telemetry-tests-client + treeherder-symbol: tt(c) + run-on-projects: built-projects + max-run-time: 1200 + tier: default + mozharness: + script: telemetry/telemetry_client.py + config: + by-test-platform: + linux.*: + - remove_executables.py + windows.*: [] + macosx.*: [] + +test-verify: + description: "Extra verification of tests modified on this push" + suite: test-verify + treeherder-symbol: TV + loopback-video: true + max-run-time: 10800 + allow-software-gl-layers: false + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: + by-test-platform: + # do not run on ccov or asan or shippable or wayland + .*(-ccov|-asan|-shippable|-wayland).*: [] + # we prefer -qr platforms with fis, no need to run on both + (?!.*(-ccov|-asan|-shippable))(?!.*-qr).*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + android.*: ['integration'] + default: [] + default: + by-test-platform: + # do not run on ccov or asan or shippable or wayland + .*(-ccov|-asan|-shippable|-wayland).*: [] + # we prefer -qr platforms with fis, no need to run on both + (?!.*(-ccov|-asan|-shippable))(?!.*-qr).*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + android.*: [] + default: ['integration'] + built-projects-only: true + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + default: null + optimization: + test-verify: ["test-verify"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-emulator-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-verify-gpu: + description: "Extra verification of tests modified on this push on gpu instances" + suite: + category: test-verify + name: test-verify-gpu + treeherder-symbol: TVg + loopback-video: true + virtualization: virtual-with-gpu + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on ccov or asan or shippable or wayland + .*(-ccov|-asan|-shippable|-wayland).*: [] + # do not run on mozilla-central, beta or release: usually just confirms earlier results + linux.*: ['integration'] + default: [] + built-projects-only: true + optimization: + test-verify: ["test-verify-gpu"] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --verify + - --gpu-required + +test-coverage: + description: "Per-test coverage" + suite: test-coverage + treeherder-symbol: TC + loopback-video: true + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + tier: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-emulator-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + +test-coverage-gpu: + description: "Per-test coverage for tests that require gpu instances" + suite: test-coverage + treeherder-symbol: TCg + loopback-video: true + virtualization: virtual-with-gpu + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # only run on mozilla-central and try. + .*-ccov.*/.*: ['mozilla-central'] + default: [] + tier: + by-test-platform: + windows.*asan.*: 3 + default: 2 + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + extra-options: + - --per-test-coverage + - --gpu-required diff --git a/taskcluster/ci/test/mochitest.yml b/taskcluster/ci/test/mochitest.yml new file mode 100644 index 0000000000..f2399d7116 --- /dev/null +++ b/taskcluster/ci/test/mochitest.yml @@ -0,0 +1,1171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: mochitest + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + android-hw.*: geckoview-test_runner.apk + windows11-64-2009.*: + by-variant: + msix: target.installer.msix + default: null + default: null + tier: + by-variant: + xorigin: + by-test-platform: + linux.*64.*-qr/debug: 1 + default: 2 + default: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + android-hw.*: android_hardware_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + android-hw.*: + - android/android_common.py + - android/android_hw.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + +mochitest-plain: + description: "Mochitest plain run" + treeherder-symbol: M() + schedules-component: mochitest-plain + loopback-video: true + virtualization: virtual + variants: + - aab+no-fission + - conditioned_profile + - xorigin + - no-fission + - socketprocess_networking + - webrender-sw+no-fission + - webrender-sw + - headless + - headless+socketprocess_networking + - http3 + - http2 + - geckoview-fission-isolate-high-value + run-on-projects: + by-variant: + conditioned_profile: + by-test-platform: + .*(asan|ccov|wayland).*: [] + linux1804-64(-shippable)?-qr/opt: ['trunk'] + windows11-64.*(-shippable)?-qr/opt: ['trunk'] + default: [] + xorigin: + by-test-platform: + .*(asan|wayland).*: [] + windows.*ccov.*: [] + (linux.*64|macosx.*64|windows11.*)(-shippable)?-qr/(opt|debug): ['trunk'] + windows.*64-ccov-qr/opt: ['mozilla-central'] + linux.*64-ccov-qr/opt: ['mozilla-central'] + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android-em.*: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + .*(asan|ccov|wayland).*: [] + (linux.*64|macosx.*64|windows11-64.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*asan.*: ['trunk'] + linux.*64(-shippable)?-qr/(opt|debug): ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + aab+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + headless: + by-test-platform: + .*wayland.*: [] + linux1804-64.*: ['mozilla-central'] + default: [] + headless+socketprocess_networking: + by-test-platform: + .*wayland.*: [] + linux1804-64.*: ['mozilla-central'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + http3: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + http2: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + geckoview-fission-isolate-high-value: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android-em.*: built-projects + default: [] + default: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + default: built-projects + chunks: + by-test-platform: + android.*/debug: 10 + linux.*/debug: 16 + linux.*64-tsan-qr/opt: 20 + .*(asan|ccov).*: 10 + linux.*/opt: 8 + windows.*/debug: 8 + default: 5 + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB + default: default + max-run-time: 3600 + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android-em.*: + - --test-suite=mochitest-plain + default: [] + chunked: + by-test-platform: + android-em.*: false + default: true + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-node + - linux64-minidump-stackwalk + +mochitest-a11y: + description: "Mochitest a11y run" + treeherder-symbol: M(a11y) + schedules-component: mochitest-a11y + test-manifest-loader: default # ensure we don't run with manifest-scheduling + loopback-video: true + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - webrender-sw+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: + by-test-platform: + linux.*64-qr/debug: ['autoland', 'mozilla-central'] + default: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + 1proc: built-projects + mozharness: + mochitest-flavor: a11y + +mochitest-browser-chrome: + description: "Mochitest browser-chrome run" + suite: + name: mochitest-browser-chrome + treeherder-symbol: M(bc) + schedules-component: mochitest-browser-chrome + loopback-video: true + variants: + - webrender-sw + - webrender-sw+a11y-checks + - socketprocess_networking + - msix + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + linux.*64(-asan)?-qr/(opt|debug): ['all'] + linux.*64-tsan-qr/opt: built-projects + linux.*64-shippable-qr/opt: ['release'] + linux.*64(-shippable)?-qr/opt: built-projects + linux.*64-devedition-qr/opt: built-projects + windows1.*-64.*-qr/debug: built-projects + .*wayland.*: [] # will change to built-projects when greened up + default: [] + webrender-sw+a11y-checks: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + .*asan.*: [] + .*ccov.*: [] + (linux.*64|macosx.*64|windows1.*-64.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + msix: + by-test-platform: + windows11-64-2009-qr/debug: built-projects + windows11-64-2009(-shippable)?-qr/opt: built-projects + default: [] + default: + by-test-platform: + android.*: [] + linux.*: [] # linux runs M-swr(bc), not M(bc) + default: built-projects + tier: + by-variant: + webrender-sw: + by-test-platform: + linux.*64(-asan|-shippable)?-qr/(opt|debug): 1 + default: default + default: default + chunks: + by-variant: + msix: + by-test-platform: + .*/debug: 16 + default: 7 + default: + by-test-platform: + linux.*wayland.*/opt: 8 + linux.*wayland.*/debug: 12 + linux.*-qr/debug: 16 + linux.*64-asan-qr/opt: 16 + linux.*64-tsan-qr/opt: 32 + macosx.*64-qr/debug: 12 + macosx.*64.*/opt: 10 + windows.*-ccov.*/.*: 14 + windows.*-asan-qr/opt: 14 + windows.*32-qr/debug: 12 + default: 8 + max-run-time: + by-variant: + msix: 5400 + socketprocess_networking: + by-test-platform: + linux.*-qr/debug: 7200 + default: 5400 + default: + by-test-platform: + linux.*64-ccov.*/.*: 9000 + linux.*64-tsan-qr/opt: 10800 + linux.*-qr/opt: 5400 + macosx.*64-ccov.*/.*: 10800 + windows.*-ccov.*/.*: 10800 + windows.*-asan-qr/opt: 5400 + .*/debug: 5400 + default: 3600 + mozharness: + mochitest-flavor: browser + chunked: true + # Bug 1281241: migrating to m3.large instances + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB + linux.*64-asan-qr/opt: xlarge # runs out of memory using 8GB + default: default + allow-software-gl-layers: false + +mochitest-browser-chrome-failures: + description: "Mochitest browser-chrome failures run" + suite: + name: mochitest-browser-chrome + treeherder-symbol: M-f(bc) + schedules-component: mochitest-browser-chrome + loopback-video: true + variants: + - webrender-sw + run-on-projects: + by-test-platform: + android.*: [] + .*wayland.*: [] # will change to built-projects when greened up + default: [] + tier: 3 + chunks: 2 + max-run-time: + by-test-platform: + linux.*wayland.*/opt: 1800 + linux.*wayland.*/debug: 3600 + default: 3600 + mozharness: + mochitest-flavor: browser + chunked: true + extra-options: + by-test-platform: + linux.*wayland.*: + - --run-failures="os == 'linux' && os_version == '22.04'" + - --timeout-as-pass + - --crash-as-pass + instance-size: default + allow-software-gl-layers: false + +mochitest-browser-a11y: + description: "Mochitest browser-a11y run" + suite: + name: mochitest-browser-a11y + treeherder-symbol: M(ba) + schedules-component: mochitest-browser-a11y + loopback-video: true + run-on-projects: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + .*tsan.*: 2700 + linux.*/debug: 2700 + default: 1800 + mozharness: + mochitest-flavor: browser + chunked: + by-test-platform: + .*tsan.*: true + default: false + chunks: + by-test-platform: + .*tsan.*: 2 + .*asan.*: 2 + default: 1 + instance-size: default + allow-software-gl-layers: false + virtualization: virtual + +mochitest-browser-media: + description: "Mochitest browser media" + suite: + name: mochitest-browser-media + treeherder-symbol: M(mda-bc) + schedules-component: mochitest-browser-media + loopback-video: true + test-manifest-loader: null + variants: ["wmf-media-engine"] + run-on-projects: + by-variant: + wmf-media-engine: + by-test-platform: + .*asan.*: [] + windows11-64.*/debug: [] # Bug 1805736 + windows1.*-64.*: ['autoland', 'mozilla-central'] + default: [] + default: + by-test-platform: + android.*: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + .*tsan.*: 1800 + default: 1200 + mozharness: + mochitest-flavor: browser + instance-size: default + allow-software-gl-layers: false + virtualization: virtual + +mochitest-browser-screenshots: + description: "Browser Screenshots" + suite: + name: mochitest-browser-screenshots + treeherder-symbol: M(ss) + loopback-video: true + test-manifest-loader: null # don't load tests in the taskgraph + variants: ["no-fission"] + run-on-projects: + by-test-platform: + linux1804-64.*/opt: ['trunk'] + default: [] + max-run-time: 3600 + mozharness: + mochitest-flavor: browser + allow-software-gl-layers: false + +mochitest-chrome: + description: "Mochitest chrome run" + treeherder-symbol: M(c) + schedules-component: mochitest-chrome + loopback-video: true + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - webrender-sw+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: + by-test-platform: + .*asan.*: [] + .*ccov.*: [] + (linux.*64|macosx.*64|windows11-64.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + 1proc: + by-test-platform: + .*wayland.*: [] + default: built-projects + chunks: + by-test-platform: + .*tsan.*/opt: 6 + default: 3 + max-run-time: 3600 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory otherwise using 8GB + default: default + +mochitest-devtools-chrome: + description: "Mochitest devtools-chrome run" + suite: + name: mochitest-devtools-chrome + schedules-component: mochitest-browser-chrome + treeherder-symbol: M(dt) + loopback-video: true + variants: + - a11y-checks + - socketprocess_networking + - webrender-sw + - devtools-no-eft+no-fission + - http3 + run-on-projects: + by-variant: + a11y-checks: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + devtools-no-eft+no-fission: + by-test-platform: + linux.*64(-shippable)?-qr/opt: ['trunk'] + default: [] + http3: + by-test-platform: + linux1804-64-qr/debug: built-projects + default: [] + socketprocess_networking: + by-test-platform: + .*asan.*: [] + .*ccov.*: [] + (linux.*64|macosx.*64|windows.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + .*wayland.*: [] # will change to built-projects when greened up + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + linux.*64-ccov-qr/opt: [] + default: built-projects + tier: default + max-run-time: + by-test-platform: + windows.*-ccov.*/.*: 10800 + macosx.*64-ccov.*/.*: 9000 + linux.*64-ccov.*/.*: 7200 + linux.*64-tsan-qr/opt: 7200 + default: 5400 + chunks: + by-test-platform: + .*-ccov.*/.*: 12 + linux.*wayland.*/debug: 14 + linux.*64-qr/debug: 14 + macosx.*64-qr/debug: 8 + .*-asan(-qr)?/opt: 8 + .*-tsan-qr/opt: 16 + default: 5 + mozharness: + mochitest-flavor: chrome + chunked: true + instance-size: + by-test-platform: + linux.*64-[at]san(-qr)?/opt: xlarge # runs out of memory otherwise using 8GB + default: default + # Bug 1296086: high number of intermittents observed with software GL and large instances + allow-software-gl-layers: false + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-node + - linux64-minidump-stackwalk + + +mochitest-devtools-chrome-failures: + description: "Mochitest devtools-chrome run" + suite: + name: mochitest-devtools-chrome + schedules-component: mochitest-browser-chrome + treeherder-symbol: M-f(dt) + loopback-video: true + variants: + - webrender-sw + run-on-projects: + by-test-platform: + android.*: [] + .*wayland.*: [] # will change to built-projects when greened up + default: [] + tier: 3 + chunks: 2 + max-run-time: + by-test-platform: + linux.*wayland.*/opt: 1800 + linux.*wayland.*/debug: 3600 + default: 3600 + mozharness: + mochitest-flavor: chrome + chunked: true + extra-options: + by-test-platform: + linux.*wayland.*: + - --run-failures="os == 'linux' && os_version == '22.04'" + - --timeout-as-pass + - --crash-as-pass + instance-size: default + allow-software-gl-layers: false + + +mochitest-plain-gpu: + description: "Mochitest plain GPU run" + suite: + name: mochitest-plain-gpu + treeherder-symbol: M(gpu) + schedules-component: mochitest-plain + loopback-video: true + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*asan.*: ['trunk'] + linux.*64(-shippable)?-qr/(opt|debug): ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux.*64-ccov-qr/opt: [] + linux1804-32.*: [] + .*-tsan-qr/opt: [] + .*wayland.*: [] + linux.*: ['trunk'] # linux opt/debug/asan + android.*: built-projects + default: [] + default: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + default: built-projects + virtualization: virtual-with-gpu + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + mochitest-flavor: plain + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-plain-gpu + default: + - --mochitest-suite=mochitest-plain-gpu + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory otherwise using 8GB + default: default + +mochitest-chrome-gpu: + description: "Mochitest chrome GPU run" + suite: + name: mochitest-chrome-gpu + treeherder-symbol: M(gpu-c) + loopback-video: true + virtualization: virtual-with-gpu + run-without-variant: false + variants: + - 1proc + - socketprocess_networking+1proc + - webrender-sw+1proc + run-on-projects: + by-variant: + socketprocess_networking+1proc: [] + webrender-sw+1proc: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + 1proc: + by-test-platform: + android.*/.*: [] + default: built-projects + tier: default + mozharness: + mochitest-flavor: chrome + extra-options: + by-test-platform: + android.*: + # note that Android runs fewer suites than other platforms + - --test-suite=mochitest-chrome-gpu + default: + - --mochitest-suite=mochitest-chrome-gpu + instance-size: + by-test-platform: + linux.*64-tsan-qr/opt: xlarge # runs out of memory using 8GB of memory + default: default + +mochitest-media: + description: "Mochitest media run" + treeherder-symbol: M(mda) + schedules-component: mochitest-plain + max-run-time: + by-test-platform: + windows.*-ccov.*/.*: 7200 + macosx.*64-ccov.*/.*: 7200 + linux.*64-tsan-qr/opt: 7200 + default: 5400 + variants: + - media-gpu + - no-fission + - socketprocess+no-fission + - socketprocess + - webgl-ipc + - webrender-sw+no-fission + - webrender-sw + - wmf-media-engine + - msix + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + android.*: [] + windows.*ccov.*: [] + .*hw-ref.*: [] + linux.*64(-shippable|-asan|-tsan)?-qr/opt: ['autoland', 'mozilla-central'] + macosx.*-64(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + windows1.*(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central'] + windows.*-asan-qr/opt: ['autoland', 'mozilla-central'] + default: [] + socketprocess+no-fission: + by-test-platform: + .*hw-ref.*: [] + android-em-7.0-x86_64-qr/debug: ['autoland', 'mozilla-central'] + android-hw-.*-aarch64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + socketprocess: + by-test-platform: + android.*: [] + windows.*ccov.*: [] + .*hw-ref.*: [] + linux.*64(-asan|-devedition|-shippable)?-qr/(opt|debug): built-projects + linux.*64-tsan-qr/opt: ['autoland', 'mozilla-central'] + macosx.*-64(-devedition|-shippable)?-qr/(opt|debug): built-projects + windows1.*(-devedition|-shippable)?-qr/(opt|debug): built-projects + windows.*-asan-qr/opt: ['autoland', 'mozilla-central'] + default: [] + webrender-sw+no-fission: + by-test-platform: + .*hw-ref.*: [] + android-em-7.0-x86_64-qr/debug: ['autoland', 'mozilla-central'] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + android.*: [] + .*hw-ref.*: [] + linux1804-64-qr/debug: ['autoland', 'mozilla-central'] + default: [] + no-fission: + by-test-platform: + android-hw-.*/opt: [] + .*hw-ref.*: [] + android-hw-.*-arm7.*/(?:debug)?: ['trunk', 'mozilla-beta', 'mozilla-release'] + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + wmf-media-engine: + by-test-platform: + .*hw-ref.*: [] + windows1.*ccov.*: [] + windows1.*-64.*: ['autoland', 'mozilla-central'] + default: [] + msix: + by-test-platform: + .*hw-ref.*: [] + windows11-64-2009-shippable-qr/opt: built-projects + windows11-64-2009-qr/opt: ['autoland'] + windows11-64-2009-qr/debug: ['autoland'] + default: [] + media-gpu: + by-test-platform: + .*hw-ref.*: [] + (macosx.*64|windows11.*)(-shippable)?-qr/(opt|debug): ['autoland', 'mozilla-central', 'mozilla-beta'] + default: [] + # TODO : add Android, it hasn't supported 'tags' + # TODO : add Linux, it would crash on gfxFeature.cpp:180 + default: + by-test-platform: + .*hw-ref.*: [] + android-hw.*/opt: [] + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux.*(asan|ccov)/opt: [] + .*(ccov/|asan/|-32).*: built-projects + (android|linux|windows10-64-2009|windows11-64-2009|macos)(?!.*-qr).*: [] + default: built-projects + loopback-video: true + chunks: + by-variant: + wmf-media-engine: 1 + media-gpu: 1 + default: + by-test-platform: + (android|macos).*/opt: 1 + ^(?!.*asan).*windows1.*/opt: 1 + linux.*/debug: 3 + linux.*64-tsan-qr/opt: 5 + default: 2 + mozharness: + mochitest-flavor: plain + chunked: + by-test-platform: + android.*: false + macosx.*64.*: false + windows1.*: false + default: true + extra-options: + by-test-platform: + android-em-7.*: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + - --setpref=gfx.canvas.accelerated=false + default: + - --setpref=webgl.out-of-process=false + - --setpref=security.sandbox.content.headless=false + tier: + by-test-platform: + .*ccov.*: 2 + android-em.*: 1 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + android-hw.*: 1 + default: default + +mochitest-valgrind: + description: "Mochitest plain Valgrind run" + suite: + name: mochitest-valgrind-plain + treeherder-symbol: M-V() + run-on-projects: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 3 + loopback-video: true + chunks: 40 + max-run-time: 14400 + run-without-variant: false + variants: ["1proc"] + allow-software-gl-layers: false + mozharness: + mochitest-flavor: plain + chunked: true + +mochitest-webgl1-core: + description: "Mochitest webgl1-core run" + treeherder-symbol: M(gl1c) + schedules-component: mochitest-plain + virtualization: virtual-with-gpu + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: + by-test-platform: + macosx.*64-ccov.*/.*: 7200 + default: 1800 + variants: + - no-fission + - webgl-ipc + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webgl-ipc: + by-test-platform: + .*-(ccov|tsan|wayland).*: [] + linux.*-64.*: ["trunk"] + mac.*: ["trunk"] + win.*: ["trunk"] + default: [] + webrender-sw+no-fission: + by-test-platform: + linux.*64-qr/debug: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-hw-.*(?/, where is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + + +## +# Linux platforms (matching /linux.*/) +# Linux1804-64 uses Ubuntu 18.04 and is run on docker + +linux1804-32-qr/debug: + build-platform: linux/debug + test-sets: + - linux32-gtest + +linux1804-32-qr/opt: + build-platform: linux/opt + test-sets: + - linux32-tests + - linux32-gtest + +linux1804-32-shippable-qr/opt: + build-platform: linux-shippable/opt + test-sets: + - linux32-tests + - linux32-gtest + +linux1804-64-qr/opt: + build-platform: linux64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - talos + - browsertime + +linux1804-64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + - web-platform-tests-backlog + - talos + - talos-profiling + - browsertime + - browsertime-profiling + +linux1804-64-nightlyasrelease-qr/opt: + build-platform: linux64-nightlyasrelease/opt + test-sets: + - browsertime + +linux1804-64-qr/debug: + build-platform: linux64/debug + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-asan-qr/opt: + build-platform: linux64-asan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-tsan-qr/opt: + build-platform: linux64-tsan/opt + test-sets: + - linux1804-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-ccov-qr/opt: + build-platform: linux64-ccov/opt + test-sets: + - ccov-code-coverage-tests + - linux1804-tests + - linux1804-ccov-tests + - mochitest-webgpu + - web-platform-tests + +linux1804-64-devedition-qr/opt: + build-platform: linux64-devedition/opt + test-sets: + - desktop-screenshot-capture + - linux1804-tests + - web-platform-tests + +linux1804-64-clang-trunk-qr/opt: + build-platform: linux64-plain-clang-trunk/opt + test-sets: + - awsy + - browsertime-clang + +linux2204-64-wayland/debug: + build-platform: linux64/debug + test-sets: + - linux2204-wayland-tests + - web-platform-tests + +linux2204-64-wayland/opt: + build-platform: linux64/opt + test-sets: + - linux2204-wayland-tests + - web-platform-tests + +linux2204-64-wayland-shippable/opt: + build-platform: linux64-shippable/opt + test-sets: + - linux2204-wayland-tests + - web-platform-tests + +## +# Windows platforms (matching /windows.*/) + +# win32 +windows10-32-shippable-qr/opt: + build-platform: win32-shippable/opt + build-signing-label: build-signing-win32-shippable/opt + test-sets: + - browsertime + - talos-profiling + - windows-talos + +windows10-32-qr/opt: + build-platform: win32/opt + test-sets: + - browsertime + - windows-talos + + +# win11-32 +windows11-32-2009-shippable-qr/opt: + build-platform: win32-shippable/opt + build-signing-label: build-signing-win32-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - windows32-2009-tests + +windows11-32-2009-qr/opt: + build-platform: win32/opt + build-signing-label: build-signing-win32/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - windows32-2009-tests + +windows11-32-2009-qr/debug: + build-platform: win32/debug + build-signing-label: build-signing-win32/debug + test-sets: + - web-platform-tests + - windows32-2009-tests + +windows11-32-2009-mingwclang-qr/debug: + build-platform: win32-mingwclang/debug + test-sets: + - mingwclang-win32-tests-debug + +windows11-32-2009-mingwclang-qr/opt: + build-platform: win32-mingwclang/opt + test-sets: + - mingwclang-tests-opt + +# win64 +windows10-64-qr/opt: + build-platform: win64/opt + test-sets: + - windows-talos + - browsertime + +windows10-64-ref-hw-2017-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - talos-ref-hw + +windows11-64-2009-hw-ref-shippable/opt: + build-platform: win64-shippable/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - windows-ref-hw-tests + +windows11-64-2009-hw-ref/opt: + build-platform: win64/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - windows-ref-hw-tests + +windows10-64-shippable-qr/opt: + build-platform: win64-shippable/opt + test-sets: + - talos-profiling + - windows-talos + - browsertime + +windows10-64-nightlyasrelease-qr/opt: + build-platform: win64-nightlyasrelease/opt + test-sets: + - browsertime + +# win10-64 2004 +windows10-64-2009-qr/debug: + build-platform: win64/debug + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/debug + default: build-signing-win64/debug + test-sets: + - windows10-2009-tests + +windows10-64-2009-qr/opt: + build-platform: win64/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - windows10-2009-tests + +windows10-64-2009-shippable-qr/opt: + build-platform: win64-shippable/opt + build-signing-label: + by-variant: + msix: repackage-signing-shippable-l10n-msix-win64-shippable/opt + default: build-signing-win64-shippable/opt + test-sets: + - windows10-2009-tests + +# win11-64 2009 +windows11-64-2009-ccov-qr/opt: + build-platform: win64-ccov/opt + build-signing-label: build-signing-win64-ccov/opt + test-sets: + - desktop-screenshot-capture + - ccov-code-coverage-tests + - windows11-64-2009-ccov-tests + - windows11-2009-tests + +windows11-64-2009-qr/debug: + build-platform: win64/debug + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/debug + default: build-signing-win64/debug + test-sets: + - web-platform-tests + - windows11-2009-tests + +windows11-64-2009-qr/opt: + build-platform: win64/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - web-platform-tests-backlog + - windows11-2009-tests + - talos-xperf + +windows11-64-2009-shippable-qr/opt: + build-platform: win64-shippable/opt + build-signing-label: + by-variant: + msix: repackage-signing-shippable-l10n-msix-win64-shippable/opt + default: build-signing-win64-shippable/opt + test-sets: + - awsy + - desktop-screenshot-capture + - web-platform-tests + - web-platform-tests-backlog + - windows11-2009-tests + - talos-xperf + +windows11-64-2009-devedition-qr/opt: + build-platform: win64-devedition/opt + build-signing-label: build-signing-win64-devedition/opt + test-sets: + - desktop-screenshot-capture + - web-platform-tests + - windows11-2009-tests + +windows11-64-2009-asan-qr/opt: + build-platform: win64-asan/opt + test-sets: + - windows11-2009-tests + +windows11-64-2009-mingwclang-qr/debug: + build-platform: win64-mingwclang/debug + test-sets: + - mingwclang-tests-debug + +windows11-64-2009-mingwclang-qr/opt: + build-platform: win64-mingwclang/opt + test-sets: + - mingwclang-tests-opt + + +## +# MacOS X platforms (matching /macosx.*/) + +macosx1100-64-qr/debug: + build-platform: macosx64/debug + build-signing-label: repackage-macosx64/debug + test-sets: + - macosx1100-64-jittests + +macosx1100-64-aarch64-qr/debug: + build-platform: macosx64-aarch64/debug + build-signing-label: repackage-macosx64-aarch64/debug + test-sets: + - macosx1100-aarch64-qr-tests + +macosx1100-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + build-signing-label: repackage-macosx64-shippable/opt + test-sets: + - macosx1100-64-jittests + - macosx1100-64-qr-tests + # - macosx64-talos + # - browsertime-high-priority + +macosx1015-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - awsy + - browsertime + - browsertime-profiling + - desktop-screenshot-capture + - macosx1015-64-qr-tests + - macosx64-talos + - talos-profiling + - web-platform-tests + - web-platform-tests-backlog + +macosx1015-64-nightlyasrelease-qr/opt: + build-platform: macosx64-nightlyasrelease/opt + test-sets: + - browsertime + +macosx1015-64-devedition-qr/opt: + build-platform: macosx64-devedition/opt + test-sets: + - macosx1015-64-qr-tests + - web-platform-tests + +macosx1015-64-qr/opt: + build-platform: macosx64/opt + test-sets: + - desktop-screenshot-capture + - macosx1015-64-qr-tests + - web-platform-tests + - web-platform-tests-backlog + - browsertime + +macosx1015-64-qr/debug: + build-platform: macosx64/debug + test-sets: + - macosx1015-64-qr-tests + - web-platform-tests + +macosx1015-64-ccov-qr/opt: + build-platform: macosx64-ccov/opt + test-sets: + - macosx1015-64-qr-tests + +macosx1400-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - macosx1400-64-qr-tests + +## +# Android platforms (matching /android-em.*/) +# +# android-em test platforms execute on android emulators. + +android-em-7.0-x86_64-qr/opt: + build-platform: android-x86_64/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-lite-qr/opt: + build-platform: android-x86_64-lite/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable-qr/opt: + build-platform: android-x86_64-shippable/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-shippable-lite-qr/opt: + build-platform: android-x86_64-shippable-lite/opt + test-sets: + - android-x86_64-tests + - web-platform-tests-backlog + +android-em-7.0-x86_64-qr/debug: + build-platform: android-x86_64/debug + test-sets: + - android-x86_64-tests + +android-em-7.0-x86_64-qr/debug-isolated-process: + build-platform: android-x86_64/debug-isolated-process + test-sets: + - android-x86_64-tests + +android-em-7.0-x86-qr/opt: + build-platform: android-x86/opt + test-sets: + - android-x86-tests + +# android-hw test platforms execute on real devices attached to Autophone hosts. + +# android-hw-a51-11-0 Samsung Galaxy A51 Android 11.0 + +android-hw-a51-11-0-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-browsertime +# - android-hw-browsertime-power + +android-hw-a51-11-0-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-browsertime +# - android-hw-browsertime-power + +# android-hw-p5-13-0 Google Pixel 5 Android 13.0 + +android-hw-p5-13-0-arm7-qr/opt: + build-platform: android-arm/opt + test-sets: + - android-hw-arm7-pgo-unittests + +android-hw-p5-13-0-arm7-shippable-qr/opt: + build-platform: android-arm-shippable/opt + test-sets: + - android-hw-arm7-pgo-unittests + +android-hw-p5-13-0-arm7-qr/debug: + build-platform: android-arm/debug + test-sets: + - android-hw-arm7-debug-unittests + +android-hw-p5-13-0-android-aarch64-qr/debug: + build-platform: android-aarch64/debug + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-debug-unittests + +android-hw-p5-13-0-android-aarch64-qr/opt: + build-platform: android-aarch64/opt + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-opt-unittests + +android-hw-p5-13-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-aarch64 + - android-hw-aarch64-pgo-unittests + +android-hw-p6-13-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-speedometer-tests + +android-hw-s21-13-0-android-aarch64-shippable-qr/opt: + build-platform: android-aarch64-shippable/opt + test-sets: + - android-hw-speedometer-tests diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml new file mode 100644 index 0000000000..90ef2145fe --- /dev/null +++ b/taskcluster/ci/test/test-sets.yml @@ -0,0 +1,499 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# +# Test names given here reference tests.yml. + +### Feature-specific test groups + +web-platform-tests: + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-privatebrowsing + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + - web-platform-tests-webgpu + - web-platform-tests-canvas + +web-platform-tests-backlog: + - web-platform-tests-backlog + - web-platform-tests-reftest-backlog + +mochitest-devtools-chrome: + - mochitest-devtools-chrome + +mochitest-webgpu: + - mochitest-webgpu + +talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g3 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions + +talos-profiling: + - talos-bcv-profiling + - talos-chrome-profiling + # - talos-damp-profiling - Disabled b/c of Bug 1597734 + - talos-dromaeojs-profiling + - talos-g1-profiling + - talos-g3-profiling + - talos-g4-profiling + - talos-g5-profiling + - talos-motionmark-profiling + - talos-other-profiling + - talos-sessionrestore-many-windows-profiling + - talos-perf-reftest-profiling + - talos-perf-reftest-singletons-profiling + - talos-svgr-profiling + - talos-tp5o-profiling + - talos-tabswitch-profiling + - talos-realworld-webextensions-profiling + - talos-webgl-profiling + +talos-ref-hw: + - talos-webgl + +browsertime: + - browsertime-tp6 + - browsertime-tp6-essential + - browsertime-tp6-bytecode + - browsertime-tp6-live + - browsertime-tp6-live-sheriffed + - browsertime-tp6-webextensions + - browsertime-benchmark + - browsertime-benchmark-wasm + - browsertime-youtube-playback + - browsertime-youtube-playback-power + - browsertime-custom + - browsertime-first-install + - browsertime-responsiveness + - browsertime-regression-tests + - browsertime-upload + - browsertime-speculative + - browsertime-throttled + - browsertime-pageload-benchmark + - browsertime-indexeddb + +browsertime-clang: + - browsertime-tp6 + - browsertime-tp6-essential + - browsertime-tp6-bytecode + - browsertime-tp6-live + - browsertime-tp6-live-sheriffed + - browsertime-tp6-webextensions + - browsertime-benchmark + - browsertime-benchmark-wasm + - browsertime-youtube-playback + - browsertime-responsiveness + +browsertime-profiling: + - browsertime-tp6-profiling + +browsertime-high-priority: + - browsertime-benchmark + - browsertime-tp6 + +awsy: + - awsy + - awsy-base + - awsy-dmd + - awsy-base-dmd + - awsy-tp6 + +ccov-code-coverage-tests: + - test-coverage + - test-coverage-wpt + +mochitest-valgrind: + - mochitest-valgrind + +desktop-screenshot-capture: + - mochitest-browser-screenshots + +### Specific platform test sets + +# Linux +linux32-gtest: + - gtest + +linux32-tests: + - web-platform-tests + +linux1804-tests: + - cppunittest + - crashtest + - firefox-ui-functional + - gtest + # - jittest # See comment in linux1804-ccov-tests set. + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - reftest-snapshot + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +linux1804-ccov-tests: + # jittest run on regular linux builds is redundant with SM(p), so we omit it + # from the linux1804-tests and linux1804-qr-tests sets. But, we still want + # to run it on ccov builds, so we have this set that has it and which runs + # on ccov builds. + - jittest + +linux2204-wayland-tests: + - mochitest-plain + - cppunittest + - crashtest + - firefox-ui-functional + - gtest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-chrome-failures + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - reftest-snapshot + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +# Windows +windows32-2009-tests: + - cppunittest + - crashtest + - gtest + - firefox-ui-functional + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgpu + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +windows10-2009-tests: + - cppunittest + - gtest + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-plain + - mochitest-plain-gpu + - xpcshell + +windows11-64-2009-ccov-tests: + - jittest + - test-coverage + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-print-reftest + - web-platform-tests-reftest + - web-platform-tests-wdspec + +windows11-2009-tests: + - cppunittest + - crashtest + - firefox-ui-functional + - gtest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +mingwclang-tests-debug: + - cppunittest + - firefox-ui-functional + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + +# The following is the same as the above block, minus reftests. +# Reftests were disabled due to frequent hangs; see bug 1642719 +# for re-enabling. +mingwclang-win32-tests-debug: + - cppunittest + - firefox-ui-functional + - mochitest-a11y + - mochitest-chrome-gpu + - mochitest-plain-gpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + # - reftest - disabled, see bug 1642719 + - telemetry-tests-client + +mingwclang-tests-opt: + - cppunittest + - mochitest-chrome-gpu + - mochitest-plain-gpu + +windows-talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-svgr + - talos-tp5o + - talos-tabswitch + - talos-realworld-webextensions + - talos-webgl + # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions + +windows-ref-hw-tests: + - mochitest-media + +talos-xperf: + - talos-xperf + +# macOS +macosx1100-64-jittests: + - jittest + +macosx1100-64-qr-tests: + - crashtest + - jsreftest + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome-gpu + - mochitest-media + - mochitest-webgpu + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - reftest + - xpcshell + +macosx1100-aarch64-qr-tests: + - mochitest-media + +macosx1015-64-qr-tests: + - cppunittest + - crashtest + - firefox-ui-functional + - gtest + - jittest + - jsreftest + - marionette + - mochitest-a11y + - mochitest-browser-a11y + - mochitest-browser-chrome + - mochitest-browser-media + - mochitest-chrome + - mochitest-chrome-gpu + - mochitest-devtools-chrome + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + - mochitest-remote + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + - mochitest-webgl2-ext + - mochitest-webgpu + - reftest + - telemetry-tests-client + - test-verify + - test-verify-gpu + - test-verify-wpt + - xpcshell + +macosx64-talos: + - talos-bcv + - talos-chrome + - talos-damp-inspector + - talos-damp-webconsole + - talos-damp-other + - talos-dromaeojs + - talos-g1 + - talos-g4 + - talos-g5 + - talos-other + - talos-sessionrestore-many-windows + - talos-svgr + - talos-tp5o + - talos-perf-reftest + - talos-perf-reftest-singletons + - talos-realworld-webextensions + - talos-webgl + - talos-tabswitch + # - talos-h1 # too long to unpack profile- Bug 1442893 + +macosx1400-64-qr-tests: + - browsertime-benchmark + +# Android + +android-x86_64-tests: + - cppunittest + - crashtest + - geckoview-junit + - gtest + - mochitest-media + - mochitest-plain + - mochitest-plain-gpu + # - mochitest-webgl1-core # Harness issues? + # - mochitest-webgl1-ext + # - mochitest-webgl2-core # Emulator not creating ES3? + # - mochitest-webgl2-ext + - reftest + - test-verify + - web-platform-tests + - web-platform-tests-crashtest + - web-platform-tests-reftest + - web-platform-tests-wdspec + - xpcshell + +android-x86-tests: + - cppunittest + +android-hw-aarch64-opt-unittests: + - jittest + +android-hw-arm7-pgo-unittests: + - jittest-all + +android-hw-aarch64-pgo-unittests: + - jittest + +android-hw-arm7-debug-unittests: + - jittest + +android-hw-aarch64-debug-unittests: + - jittest + +android-hw-aarch64: + - crashtest-qr + - reftest-qr + - mochitest-media + - mochitest-webgl1-core + - mochitest-webgl1-ext + - mochitest-webgl2-core + +android-hw-browsertime: + - browsertime-tp6m + - browsertime-tp6m-essential + - browsertime-tp6m-webextensions + - browsertime-tp6m-live + - browsertime-benchmark-speedometer-mobile + - browsertime-benchmark-speedometer3-mobile + - browsertime-youtube-playback-mobile + - browsertime-benchmark-unity-webgl-mobile + - browsertime-benchmark-motionmark + +android-hw-browsertime-profiling: + - browsertime-tp6m-profiling + +android-hw-speedometer-tests: + - browsertime-benchmark-speedometer-mobile + - browsertime-benchmark-speedometer3-mobile diff --git a/taskcluster/ci/test/variants.yml b/taskcluster/ci/test/variants.yml new file mode 100644 index 0000000000..1bfe00d001 --- /dev/null +++ b/taskcluster/ci/test/variants.yml @@ -0,0 +1,324 @@ +--- +1proc: + description: "{description} with e10s disabled" + component: "Testing::General" + expiration: "never" + suffix: "1proc" + mozinfo: "e10s" + merge: + mozharness: + extra-options: + - "--disable-e10s" + - "--disable-fission" + +headless: + description: "{description} without a window" + component: "Firefox::Headless" + expiration: "never" + suffix: "headless" + mozinfo: "headless" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--headless" + +a11y-checks: + description: "{description} with accessibility checks enabled" + component: "Core::Disability Access APIs" + expiration: "2024-03-01" + suffix: "a11y-checks" + mozinfo: "a11y_checks" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--enable-a11y-checks" + +aab: + description: "{description} with aab test_runner" + component: "GeckoView::General" + expiration: "2024-03-01" + when: &gv_e10s_filter + $eval: '"android" in task["test-platform"]' + suffix: "aab" + # Need to add jdk but there isn't a good way to do that + # so we just replace the toolchain list + merge: + fetches: + toolchain: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-jdk + replace: + target: "geckoview-test_runner.aab" + +geckoview-no-fission: + description: "{description} without fission enabled" + component: "GeckoView::Sandboxing" + expiration: "2024-03-01" + when: + <<: *gv_e10s_filter + suffix: "nofis" + mozinfo: "fission" + merge: + mozharness: + extra-options: + - "--disable-fission" + +geckoview-fission: + description: "{description} with fission enabled and default isolation strategy" + component: "GeckoView::Sandboxing" + expiration: "2024-03-01" + when: + <<: *gv_e10s_filter + suffix: "fis" + mozinfo: "fission" + +geckoview-fission-isolate-high-value: + description: "{description} with fission enabled and isolation strategy isolateHighValue" + component: "GeckoView::Sandboxing" + expiration: "2024-03-01" + when: + <<: *gv_e10s_filter + suffix: "fis-hv" + mozinfo: "fission-high-value" + merge: + mozharness: + extra-options: + - "--setpref=fission.webContentIsolationStrategy=2" + +no-fission: + description: "{description} without fission enabled" + component: "Core::DOM: Content Processes" + expiration: "2024-03-01" + suffix: "nofis" + mozinfo: "fission" + merge: + mozharness: + extra-options: + - "--disable-fission" + +xorigin: + description: "{description} with cross-origin" + component: "Core::DOM: Content Processes" + expiration: "never" + suffix: "xorig" + mozinfo: "xorigin" + replace: + e10s: true + merge: + mozharness: + extra-options: + - "--enable-xorigin-tests" + - "--setpref=privacy.partition.always_partition_third_party_non_cookie_storage=false" + +conditioned_profile: + description: "{description} with conditioned profile" + component: "Testing::General" + expiration: "2024-03-01" + suffix: "condprof" + mozinfo: "condprof" + merge: + mozharness: + extra-options: + - "--conditioned-profile" + replace: + chunks: 8 + +socketprocess: + description: "{description} with socket process enabled" + component: "Core::Networking" + expiration: "2024-09-01" + suffix: "spi" + mozinfo: "socketprocess_e10s" + merge: + mozharness: + extra-options: + - "--setpref=media.peerconnection.mtransport_process=true" + - "--setpref=network.process.enabled=true" + +socketprocess_networking: + description: "{description} with networking on socket process enabled" + component: "Core::Networking" + expiration: "2024-09-01" + suffix: "spi-nw" + mozinfo: "socketprocess_networking" + merge: + mozharness: + extra-options: + - "--setpref=network.process.enabled=true" + - "--setpref=network.http.network_access_on_socket_process.enabled=true" + +# - +# Webrender DComp variants for Win10 + +webrender-dcomp-disabled: + description: "{description} with dcomp disabled" + component: "Core::Graphics: WebRender" + expiration: "2024-03-01" + suffix: "wr-dc0" + when: + $eval: '"windows1" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=false" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-present: # Default variant ends up like this. + description: "{description} with dcomp present but not overlays" + component: "Core::Graphics: WebRender" + expiration: "2024-03-01" + suffix: "wr-dc1-p" + when: + $eval: '"windows1" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=false" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=false" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-overlays: + description: "{description} with dcomp overlays but not compositing" + component: "Core::Graphics: WebRender" + expiration: "2024-03-01" + suffix: "wr-dc2-o" + when: + $eval: '"windows1" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=true" + - "--setpref=gfx.webrender.compositor=false" + - "--setpref=gfx.webrender.compositor.force-enabled=false" + +webrender-dcomp-compositing: + description: "{description} with dcomp compositing" + component: "Core::Graphics: WebRender" + expiration: "2024-03-01" + suffix: "wr-dc3-c" + when: + $eval: '"windows1" in task["test-platform"]' + merge: + mozharness: + extra-options: + - "--setpref=gfx.webrender.dcomp-win.enabled=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win=true" + - "--setpref=gfx.webrender.dcomp-video-overlay-win.force-enabled=true" + - "--setpref=gfx.webrender.compositor=true" + - "--setpref=gfx.webrender.compositor.force-enabled=true" + +# - + +webrender-sw: + description: "{description} with software webrender enabled" + component: "Core::Graphics: WebRender" + expiration: "2024-03-01" + suffix: "swr" + mozinfo: "swgl" + merge: + webrender: true + mozharness: + extra-options: + - "--setpref=gfx.webrender.software=true" + +webgl-ipc: + description: "{description} with WebGL IPC process enabled" + component: "Core::Graphics: CanvasWebGL" + expiration: "2024-03-01" + suffix: "gli" + merge: + mozharness: + extra-options: + - "--setpref=webgl.out-of-process=true" + - "--setpref=security.sandbox.content.headless=true" + +devtools-no-eft: + description: "{description} with DevTools EFT disabled" + component: "DevTools::Framework" + expiration: "2024-03-01" + suffix: "dt-no-eft" + merge: + mozharness: + extra-options: + - "--setpref=devtools.every-frame-target.enabled=false" + +wmf-media-engine: + description: "{description} with Windows Media Foundation media engine enabled" + component: "Core::Audio/Video: Playback" + expiration: "2024-04-10" + when: + $eval: '"windows1" in task["test-platform"]' + suffix: "wmfme" + mozinfo: "wmfme" + replace: + tier: 2 + merge: + mozharness: + extra-options: + - "--tag=media-engine-compatible" + - "--setpref=media.wmf.media-engine.enabled=1" + - "--setpref=media.wmf.media-engine.channel-decoder.enabled=true" + - "--setpref=media.eme.wmf.clearkey.enabled=true" + +media-gpu: + description: "{description} for running media tests on the GPU worker" + component: "Core::Audio/Video: Playback" + expiration: "2024-04-21" + suffix: "mda-gpu" + mozinfo: "mda_gpu" + merge: + virtualization: virtual-with-gpu + mozharness: + extra-options: + - "--tag=media-gpu" + - "--setpref=media.hardware-video-decoding.force-enabled=true" + +msix: + description: "{description} from msix packages" + component: "Firefox:Installer" + expiration: "never" + suffix: "msix" + mozinfo: "msix" + when: + $eval: '"windows1" in task["test-platform"] && "-64" in task["test-platform"] && ("opt" in task["test-platform"] || "debug" in task["test-platform"]) && !("asan" in task["test-platform"]) && !("ccov" in task["test-platform"])' + merge: + mozharness: + extra-options: + - "--installer-path installer.msix" + +http3: + description: "{description} with http3 server" + component: "Core::Networking" + expiration: "2024-09-01" + suffix: "http3" + mozinfo: "http3" + merge: + mozharness: + extra-options: + - "--use-http3-server" + +http2: + description: "{description} with http2 server" + component: "Core::Networking" + expiration: "2024-09-01" + suffix: "http2" + mozinfo: "http2" + merge: + mozharness: + extra-options: + - "--use-http2-server" diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml new file mode 100644 index 0000000000..5f850d16c1 --- /dev/null +++ b/taskcluster/ci/test/web-platform.yml @@ -0,0 +1,598 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: web-platform-tests + instance-size: xlarge + max-run-time: + by-test-platform: + android-em-7.0-x86_64/debug: 7200 + default: 5400 + mozharness: + script: web_platform_tests.py + config: + by-test-platform: + windows.*: + - web_platform_tests/prod_config_windows_taskcluster.py + macosx.*: + - web_platform_tests/prod_config_mac.py + android-em.*: + - android/android-x86_64.py + - web_platform_tests/prod_config_android.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + target: + by-test-platform: + android-em-7.0-x86_64-shippable(-lite)?-qr/opt: geckoview-test_runner.apk + android-em-7.0-x86_64(-lite)?-qr/opt: geckoview-test_runner.apk + android-em-7.0-x86_64(-lite)?-qr/debug(-isolated-process)?: geckoview-test_runner.apk + default: null + +web-platform-tests: + description: "Web platform test run" + suite: web-platform-tests + treeherder-symbol: W(wpt) + virtualization: virtual + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + chunks: + by-test-platform: + android.*/debug: 32 + android.*: 24 + linux.*64-tsan-qr/opt: 32 + linux.*64-asan(-qr)?/opt: 22 + linux.*64-ccov.*/opt: 20 + linux.*/debug: 16 + macosx.*/debug: 18 + windows.*32.*/opt: 12 + windows.*/debug: 16 + windows.*-ccov.*/opt: 14 + default: 10 + max-run-time: + by-test-platform: + .*-ccov.*/.*: 10800 + linux.*64(-tsan|-asan)-qr/opt: 14400 + linux.*64(-qr)?/debug: 10800 + macosx.*/debug: 10800 + android-em.*/.*: 10800 + default: 7200 + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + android-em-7.0-x86_64-qr/debug: [] + default: [] + webrender-sw+no-fission: + by-test-platform: + .*-32.*/.*: [] + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + - --skip-timeout + - --skip-crash + # Because we have a separate job for them, don't run the (core) + # webgpu/* tests in the main WPT runs. (but still run tests in dirs + # other than webgpu/* that are tagged webgpu, e.g. canvas tests) + - --exclude-tag=webgpu + - --exclude-tag=canvas + +web-platform-tests-backlog: + description: "Web platform test (backlog) run" + treeherder-symbol: W-b(wpt) + variants: + - no-fission + chunks: + by-test-platform: + android.*: 3 + default: 2 + max-run-time: + by-test-platform: + android.*/opt: 7200 + default: 3600 + run-on-projects: + by-variant: + no-fission: + by-test-platform: + android.*: ['mozilla-central'] + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + - --exclude-tag=webgpu + +web-platform-tests-privatebrowsing: + description: "Web platform privatebrowsing test run" + suite: web-platform-tests + treeherder-symbol: W(pb) + virtualization: virtual + tier: default + chunks: 1 + max-run-time: 1800 + variants: [] # We don't need sw or nofis testing. + run-on-projects: + by-test-platform: + android.*: [] + macos.*: [] + .*-(wayland).*/.*: [] + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --default-exclude + - --setpref=browser.privatebrowsing.autostart=true + - --setpref=dom.cache.privateBrowsing.enabled=true + - --tag=privatebrowsing + +web-platform-tests-reftest: + description: "Web platform reftest run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W(Wr) + virtualization: + by-test-platform: + windows.*-64.*-qr/.*: virtual-with-gpu + default: virtual + chunks: + by-test-platform: + .*-ccov.*/.*: 8 + linux.*64-tsan-qr/opt: 16 + linux1804-64(-asan)?-qr/.*: 6 + macosx10.*-64/debug: 6 + macosx10.*-64/opt: 3 + windows.*-(32|64)(-qr)?/debug: 5 + android.*: 6 + default: 4 + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + - --skip-timeout + - --skip-crash + +web-platform-tests-webgpu: + description: "Web platform WebGPU test run" + suite: web-platform-tests + treeherder-symbol: W(webgpu) + virtualization: virtual-with-gpu + tier: + by-test-platform: + .*-32.*/.*: 3 + android.*: 3 # https://bugzilla.mozilla.org/show_bug.cgi?id=1849914 + macos.*: 3 # https://bugzilla.mozilla.org/show_bug.cgi?id=1849913 + linux2204.*-64.*: 3 # https://bugzilla.mozilla.org/show_bug.cgi?id=1849912 + windows.*-64.*: 3 # https://bugzilla.mozilla.org/show_bug.cgi?id=1849911 + default: 3 + # Choice of `chunks`: + # Time-to-TEST-START on Windows is ~7m (~9m for debug). + # (though we're working on reducing this!) + # Devs would love ~10-15m runs, but we don't want to be more than + # 50% overhead, so let's aim for 15m (20m debug) + chunks: + by-test-platform: + linux2204.*-64.*: 15 + macos.*: 20 + windows.*-64.*/opt.*: 15 + windows.*-64.*/debug.*: 20 + default: 10 + max-run-time: + by-test-platform: + .*/debug: 5400 # 90m + default: 3200 # 60m + variants: [] # We don't need sw or nofis testing. + run-on-projects: + by-test-platform: + android.*: [] # Out of scope + linux1804.*: [] # We don't support anything less than Mesa Lavapipe 22.1.2, so this is _out_. + .*-wayland/debug.*: [] # https://bugzilla.mozilla.org/show_bug.cgi?id=1864700 + default: ['mozilla-central'] + mozharness: + chunked: true + extra-options: + by-test-platform: + macos.*: + - --test-type=testharness + - --default-exclude + - --tag=webgpu + - --exclude-tag=canvas + - --setpref=gfx.webgpu.ignore-blocklist=true + default: + - --test-type=testharness + - --default-exclude + - --tag=webgpu + - --exclude-tag=canvas + +web-platform-tests-canvas: + description: "Web platform Canvas test run" + suite: web-platform-tests + treeherder-symbol: W(canvas) + virtualization: virtual-with-gpu + tier: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + chunks: + by-test-platform: + windows.*: 2 + .*tsan.*: 2 + linux.*/debug: 2 + linux.*asan.*: 2 + default: 1 + max-run-time: + by-test-platform: + .*tsan.*: 3600 + default: 1800 + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + android-em-7.0-x86_64-qr/debug: [] + default: [] + webrender-sw+no-fission: + by-test-platform: + .*-32.*/.*: [] + android-em-7.0-x86_64-qr/debug: built-projects + linux.*64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + mozharness: + chunked: true + extra-options: + - --test-type=testharness + - --skip-implementation-status=backlog + - --skip-implementation-status=not-implementing + - --skip-timeout + - --skip-crash + # Because we have a separate job for them, don't run the (core) + # webgpu/* tests in the main WPT runs. (but still run tests in dirs + # other than webgpu/* that are tagged webgpu, e.g. canvas tests) + - --default-exclude + - --tag=canvas + - --exclude-tag=webgpu + +web-platform-tests-reftest-backlog: + description: "Web platform reftest (backlog) run" + schedules-component: web-platform-tests-reftest + treeherder-symbol: W-b(Wr) + virtualization: virtual + chunks: 1 + variants: + - no-fission + run-on-projects: + by-variant: + no-fission: + by-test-platform: + android.*: ['mozilla-central'] + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: ['mozilla-central'] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --test-type=reftest + - --skip-implementation-status=implementing + - --skip-implementation-status=not-implementing + - --backlog + - --exclude-tag=webgpu + +web-platform-tests-wdspec: + description: "Web platform webdriver-spec run" + schedules-component: web-platform-tests-wdspec + treeherder-symbol: W(Wd) + chunks: + by-test-platform: + .*-ccov.*/.*: 4 + .*-tsan.*/.*: 8 + default: 3 + mozharness: + extra-options: + by-test-platform: + android-em-7.*: + - --test-type=wdspec + - --setpref=gfx.canvas.accelerated=false + default: + - --test-type=wdspec + variants: + - no-fission + - webrender-sw+no-fission + - webrender-sw + - headless + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + webrender-sw: + by-test-platform: + android-em-7.0-x86_64-qr/debug: [] + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + default: [] + headless: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + tier: default + fetches: + toolchain: + by-test-platform: + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*64.*: + - linux64-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + linux.*32.*: + - linux32-geckodriver + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-geckodriver + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-32.*: + - win32-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + win.*-64.*: + - win64-geckodriver + - win32-minidump-stackwalk + - win32-fix-stacks + default: + - linux64-minidump-stackwalk + - linux64-fix-stacks + - linux64-geckodriver + +web-platform-tests-crashtest: + description: "Web platform crashtests run" + schedules-component: web-platform-tests-crashtest + treeherder-symbol: W(Wc) + chunks: 1 + mozharness: + extra-options: + - --test-type=crashtest + variants: + - no-fission + - webrender-sw + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + macosx101.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + android-em-7.0-x86_64-qr/debug: built-projects + default: built-projects + tier: default + +web-platform-tests-print-reftest: + description: "Web platform print-reftest run" + schedules-component: web-platform-tests-print-reftest + treeherder-symbol: W(Wp) + chunks: 1 + test-manifest-loader: null # don't load tests in the taskgraph + tier: default + variants: + - no-fission + - webrender-sw + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw: + by-test-platform: + android.*: [] + .*-32.*/.*: [] + linux.*64-qr/debug: built-projects + windows.*-qr/debug: built-projects + default: [] + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + default: [] + no-fission: + by-test-platform: + linux1804-64-qr/debug: ['trunk'] + linux1804-64(-shippable|-asan)?-qr/opt: ['trunk'] + android.*: built-projects + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + default: built-projects + mozharness: + extra-options: + - --test-type=print-reftest + +test-verify-wpt: + description: "Extra verification of web-platform tests modified on this push" + suite: + category: test-verify + name: test-verify-wpt + treeherder-symbol: TVw + max-run-time: 10800 + run-on-projects: + by-test-platform: + # do not run on asan, ccov, shippable or wayland + .*-(asan|ccov|shippable|wayland).*/.*: [] + # do not run on beta or release: usually just confirms earlier results + default: ['integration'] + built-projects-only: true + test-manifest-loader: null # don't load tests in the taskgraph + optimization: + test-verify: ["test-verify-wpt"] + tier: 2 + mozharness: + extra-options: + - --verify + +test-coverage-wpt: + description: "Per web-platform test coverage" + suite: + category: test-coverage + name: test-coverage-wpt + schedules-component: test-coverage-wpt + treeherder-symbol: TCw + max-run-time: 10800 + variants: + - no-fission + - webrender-sw+no-fission + run-on-projects: + by-variant: + webrender-sw+no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug: built-projects + default: [] + default: + by-test-platform: + .*-ccov.*/.*: built-projects + default: [] + test-manifest-loader: null # don't load tests in the taskgraph + tier: 2 + mozharness: + extra-options: + - --per-test-coverage diff --git a/taskcluster/ci/test/xpcshell.yml b/taskcluster/ci/test/xpcshell.yml new file mode 100644 index 0000000000..46ecd690e5 --- /dev/null +++ b/taskcluster/ci/test/xpcshell.yml @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + suite: + category: xpcshell + name: + by-variant: + msix: xpcshell-msix + default: xpcshell + target: + by-test-platform: + android-em-7.*: geckoview-test_runner.apk + windows11-64-2009.*: + by-variant: + msix: target.installer.msix + default: null + default: null + mozharness: + script: + by-test-platform: + android-em.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android-em-7.*: + - android/android_common.py + - android/android-x86_64.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_unittest.py + requires-signed-builds: + by-test-platform: + macosx1100.*: true + windows.*-asan-qr/opt: false # No XPCShell on ASAN yet + windows.*-mingwclang-qr/.*: false + windows.*: true + default: false + +xpcshell: + description: "xpcshell test run" + treeherder-symbol: X(X) + variants: + - conditioned_profile + - socketprocess_networking+no-fission + - socketprocess_networking + - no-fission + - msix + run-on-projects: + by-variant: + conditioned_profile: + by-test-platform: + .*asan.*: [] + .*ccov.*: [] + (linux1804-64|windows11-64.*)(-shippable)?-qr/opt: ['trunk'] + default: [] + socketprocess_networking+no-fission: + by-test-platform: + linux1804-64.*: ['trunk'] + default: [] + socketprocess_networking: + by-test-platform: + (linux1804-64|macosx).*: ['trunk'] # All Linux & Mac variants + ^(?!.*asan).*windows11.*: ['trunk'] # All Windows11 except ASAN + default: [] + no-fission: + by-test-platform: + android-em-7.0-x86_64-qr/debug-isolated-process: [] + android.*: built-projects + linux1804-64.*: ['trunk'] + default: [] + msix: + by-test-platform: + windows11-64-2009(-shippable)?-qr/opt: built-projects + windows11-64-2009-qr/debug: built-projects + default: [] + default: + by-test-platform: + android.*: [] + .*wayland.*: [] + windows.*-asan-qr/opt: [] # No XPCShell on ASAN yet + default: built-projects + chunks: + by-test-platform: + .*-asan-qr/opt: 4 + .*-ccov.*/.*: 8 + .*-tsan-qr/opt: 8 + (android-em-7.*|macosx.*|linux1804-64.*/opt): 2 + default: 4 + max-run-time: + by-variant: + msix: 7200 + conditioned_profile: 7200 + default: + by-test-platform: + .*-ccov(-qr)?/.*: 7200 + .*-tsan-qr/opt: 7200 + default: 5400 + allow-software-gl-layers: false + tier: + by-test-platform: + windows.*-asan.*: 3 + android-em-7.0-x86_64-qr/debug-isolated-process: 3 + default: default + mozharness: + extra-options: + by-test-platform: + windows1.*: + - --threads=2 + default: [] + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-node + - linux64-minidump-stackwalk + - linux64-fix-stacks + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*-64.*: + - win64-node + - win32-minidump-stackwalk + - win32-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks + android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - android-emulator-linux + - linux64-node + - linux64-minidump-stackwalk + +xpcshell-failures: + description: "xpcshell specific failures test run" + treeherder-symbol: X-f(X) + run-on-projects: ["mozilla-central"] + chunks: 2 + max-run-time: 2700 + allow-software-gl-layers: false + tier: 3 + mozharness: + script: + by-test-platform: + macos.*: mac_unittest.py + windows1.*: desktop_unittest.py + config: + by-test-platform: + macos.*: + - unittests/mac_unittest.py + windows1.*: + - unittests/win_unittest.py + extra-options: + by-test-platform: + macosx1100.*: + - --run-failures=apple_silicon + - --timeout-as-pass + - --crash-as-pass + fetches: + toolchain: + by-test-platform: + macosx.*: + - macosx64-node + - macosx64-minidump-stackwalk + - macosx64-fix-stacks + win.*32.*: + - win32-node + - win32-minidump-stackwalk + - win32-fix-stacks diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml new file mode 100644 index 0000000000..ed8cef3dc1 --- /dev/null +++ b/taskcluster/ci/toolchain/android.yml @@ -0,0 +1,244 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: android-build} + max-run-time: 1800 + +# In automation, we generate the AVD using the linux64-android-avd jobs. On +# developers' machines, bootstrap will download the artifact from the avd job +# instead of generating it. +# +# The Android AVD needs the corresponding system image generated in the +# linux64-android-system-image TL job. System images are not redistributable, +# so locally bootstrap will use sdkmanager to download the corresponding system +# images. +linux64-android-avd-x86_64-repack: + attributes: + local-toolchain: true + description: "Android AVD (Linux) repack toolchain build" + treeherder: + symbol: TL(avd-x86_64-linux) + run: + script: repack-android-avd-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: public/build/android-avd-linux.tar.zst + toolchain-alias: android-avd-x86_64-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android31-avd-x86_64-repack: + description: "Android AVD (Linux) repack toolchain build" + treeherder: + symbol: TL(avd31-x86_64-linux) + worker-type: b-linux-kvm-gcp + worker: + kvm: true + run: + script: repack-android-avd-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/android31-x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: public/build/android-avd-linux.tar.zst + toolchain-alias: android31-avd-x86_64-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + + +linux64-android-avd-arm-repack: + attributes: + local-toolchain: true + description: "Android AVD (Linux) repack toolchain build" + treeherder: + symbol: TL(avd-arm-linux) + run: + script: repack-android-avd-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/arm.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: public/build/android-avd-linux.tar.zst + toolchain-alias: android-avd-arm-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android-avd-arm64-repack: + attributes: + local-toolchain: true + description: "Android AVD (Linux) repack toolchain build" + treeherder: + symbol: TL(avd-arm64-linux) + run: + script: repack-android-avd-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/arm64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: public/build/android-avd-linux.tar.zst + toolchain-alias: android-avd-arm64-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android-system-image-x86_64-repack: + description: "Android System Images (Linux) repack toolchain build" + treeherder: + symbol: TL(x86_64-avd-img-linux) + run: + script: repack-android-system-images-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-system-images/android-system-images-linux.tar.zst + toolchain-alias: android-system-image-x86_64-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android31-system-image-x86_64-repack: + description: "Android System Images (Linux) repack toolchain build" + treeherder: + symbol: TL(x86_64-avd31-img-linux) + run: + script: repack-android-system-images-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/android31-x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-system-images/android-system-images-linux.tar.zst + toolchain-alias: android31-system-image-x86_64-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + + +linux64-jdk-repack: + description: "Java JDK (Linux) repack toolchain build" + treeherder: + symbol: TL(jdk) + run: + script: repack-jdk-linux.sh + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/jdk/jdk-linux.tar.zst + toolchain-alias: linux64-jdk + toolchain-env: + JAVA_HOME: "/builds/worker/fetches/jdk/jdk-17.0.9+9" + +linux64-android-sdk-linux-repack: + description: "Android SDK (Linux) repack toolchain build" + treeherder: + symbol: TL(android-sdk-linux) + run: + script: repack-android-sdk-linux.sh + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.zst + toolchain-alias: android-sdk-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android-emulator-linux-repack: + description: "Android Emulator (Linux) repack toolchain build" + treeherder: + symbol: TL(android-emulator-linux) + run: + script: repack-android-emulator-linux.sh + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-emulator/android-emulator-linux.tar.zst + toolchain-alias: android-emulator-linux + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-jdk-repack + +linux64-android-ndk-linux-repack: + description: "Android NDK (Linux) repack toolchain build" + treeherder: + symbol: TL(android-ndk-linux) + run: + script: repack-android-ndk-linux.sh + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-ndk/android-ndk.tar.zst + toolchain-alias: android-ndk-linux + +linux64-android-gradle-dependencies: + description: "Android Gradle dependencies toolchain task" + treeherder: + symbol: TL(gradle-dependencies) + worker: + env: + GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online" + run: + script: android-gradle-dependencies.sh + sparse-profile: null + resources: + - 'taskcluster/scripts/misc/android-gradle-dependencies/**' + - '*.gradle' + - 'mobile/android/**/*.gradle' + - 'mobile/android/config/mozconfigs/android-arm-gradle-dependencies/**' + - 'mobile/android/config/mozconfigs/common*' + - 'mobile/android/gradle.configure' + toolchain-artifact: public/build/android-gradle-dependencies.tar.zst + toolchain-alias: android-gradle-dependencies + fetches: + fetch: + - sonatype-nexus + # Nexus needs Java 8 + - jdk-8-linux64 + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-android-sdk-linux-repack + - linux64-jdk-repack + - linux64-node + +linux64-android-gradle-dependencies-lite: + description: "Android Lite Gradle dependencies toolchain task" + treeherder: + symbol: TL(gradle-dependencies-lite) + worker: + env: + GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online" + run: + script: android-gradle-dependencies-lite.sh + sparse-profile: null + resources: + - 'taskcluster/scripts/misc/android-gradle-dependencies/**' + - '*.gradle' + - 'mobile/android/**/*.gradle' + - 'mobile/android/config/mozconfigs/android-arm-gradle-dependencies/**' + - 'mobile/android/config/mozconfigs/common*' + - 'mobile/android/gradle.configure' + toolchain-artifact: public/build/android-gradle-dependencies.tar.zst + toolchain-alias: android-gradle-dependencies-lite + fetches: + fetch: + - sonatype-nexus + # Nexus needs Java 8 + - jdk-8-linux64 + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-android-sdk-linux-repack + - linux64-jdk-repack + - linux64-node diff --git a/taskcluster/ci/toolchain/cargo-vet.yml b/taskcluster/ci/toolchain/cargo-vet.yml new file mode 100644 index 0000000000..3f1cb9756a --- /dev/null +++ b/taskcluster/ci/toolchain/cargo-vet.yml @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "cargo-vet toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/cargo-vet.tar.zst + fetches: + fetch: + - cargo-vet + +linux64-cargo-vet: + treeherder: + symbol: TL(cargo-vet) + run: + arguments: ['x86_64-unknown-linux-gnu', 'reqwest/native-tls-vendored'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-cargo-vet: + treeherder: + symbol: TM(cargo-vet) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-cargo-vet: + treeherder: + symbol: TM(cargo-vet-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +win64-cargo-vet: + treeherder: + symbol: TW64(cargo-vet) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/cbindgen.yml b/taskcluster/ci/toolchain/cbindgen.yml new file mode 100644 index 0000000000..69d34b6f3b --- /dev/null +++ b/taskcluster/ci/toolchain/cbindgen.yml @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "cbindgen toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/cbindgen.tar.zst + fetches: + fetch: + # If you update this, make sure to update the minimum version in + # build/moz.configure/bindgen.configure as well. + - cbindgen-0.26.0 + +linux64-cbindgen: + treeherder: + symbol: TL(cbindgen) + worker: + max-run-time: 1800 + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-cbindgen: + treeherder: + symbol: TM(cbindgen) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-cbindgen: + treeherder: + symbol: TM(cbindgen-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +win64-cbindgen: + treeherder: + symbol: TW64(cbindgen) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/clang-tidy.yml b/taskcluster/ci/toolchain/clang-tidy.yml new file mode 100644 index 0000000000..8d357e8c82 --- /dev/null +++ b/taskcluster/ci/toolchain/clang-tidy.yml @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "Clang-tidy build" + attributes: + local-toolchain: true + index: + product: static-analysis + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + toolchain-artifact: public/build/clang-tidy.tar.zst + resources: + - 'build/clang-plugin/**/*.cpp' + - 'build/clang-plugin/**/*.h' + - 'build/clang-plugin/*.inc' + - 'build/clang-plugin/*.py' + - 'build/clang-plugin/*.txt' + - 'build/clang-plugin/moz.build' + - 'build/clang-plugin/Makefile.in' + - 'build/build-clang/build-clang.py' + fetches: + fetch: + - clang-17 + +linux64-clang-tidy: + index: + job-name: linux64-clang-tidy + treeherder: + symbol: TL(clang-tidy) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/clang-tidy-linux64.json' + resources: + - 'build/build-clang/clang-tidy-linux64.json' + fetches: + toolchain: + - linux64-clang-17 + - linux64-toolchain-sysroot + +macosx64-clang-tidy: + index: + job-name: macosx64-clang-tidy + treeherder: + symbol: TM(clang-tidy) + worker-type: b-linux-large-gcp + run: + script: build-clang.sh + arguments: + - 'build/build-clang/clang-tidy-macosx64.json' + - 'build/build-clang/macosx64.json' + resources: + - 'build/build-clang/clang-tidy-macosx64.json' + - 'build/build-clang/macosx64.json' + fetches: + toolchain: + - linux64-clang-17 + - macosx64-sdk-toolchain + +macosx64-aarch64-clang-tidy: + index: + job-name: macosx64-aarch64-clang-tidy + treeherder: + symbol: TM(clang-tidy-aarch64) + worker-type: b-linux-large-gcp + run: + script: build-clang.sh + arguments: + - 'build/build-clang/clang-tidy-macosx64.json' + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + resources: + - 'build/build-clang/clang-tidy-macosx64.json' + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + fetches: + toolchain: + - linux64-clang-17 + - macosx64-sdk-toolchain + +win64-clang-tidy: + description: "Clang-tidy toolchain build" + index: + job-name: win64-clang-tidy + treeherder: + symbol: TW64(clang-tidy) + tier: 2 + worker: + max-run-time: 7200 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/clang-tidy-win64.json' + - 'build/build-clang/win64.json' + resources: + - 'build/build-clang/clang-tidy-win64.json' + - 'build/build-clang/win64.json' + fetches: + toolchain: + - linux64-clang-17 + - vs-toolchain + +linux64-clang-tidy-external: + attributes: + local-toolchain: false + index: + job-name: linux64-clang-tidy-external + treeherder: + symbol: TL(clang-tidy-e) + run: + toolchain-artifact: project/civet/clang-tidy.tar.zst + script: build-clang-tidy-external.sh + arguments: + - 'build/build-clang/clang-tidy-external-linux64.json' + resources: + - 'build/build-clang/clang-tidy-external-linux64.json' + run-on-projects: [] + fetches: + fetch: + - civet-source + toolchain: + - linux64-clang-17 + - linux64-toolchain-sysroot diff --git a/taskcluster/ci/toolchain/clang.yml b/taskcluster/ci/toolchain/clang.yml new file mode 100644 index 0000000000..1223afd4e5 --- /dev/null +++ b/taskcluster/ci/toolchain/clang.yml @@ -0,0 +1,1278 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 7200 + run: + resources: + - 'build/build-clang/build-clang.py' + +linux64-clang-8.0-raw: + description: "Clang 8.0 toolchain build" + treeherder: + symbol: TL(clang-8.0-raw) + worker-type: b-linux-large-gcp + run: + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-8.0.json' + - 'build/build-clang/2stages.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-8.0.json' + - 'build/build-clang/2stages.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-8.0 + toolchain: + - linux64-toolchain-sysroot + +linux64-clang-8.0: + description: "Clang 8.0 toolchain build" + treeherder: + symbol: TL(clang-8.0) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: linux64-clang-base + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-clang-8.0-raw + - wasm32-wasi-compiler-rt-8.0 + +linux64-clang-14-stage1: + description: "Clang 14 toolchain build" + treeherder: + symbol: TL(clang-14-stage1) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-14.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-14.json' + - 'build/build-clang/1stage.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-14 + toolchain: + - linux64-toolchain-sysroot + +linux64-clang-14: + description: "Clang 14 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(clang-14) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-clang-14-stage1 + +macosx64-clang-14-raw: + description: "Clang 14 toolchain build" + treeherder: + symbol: TM(clang-14-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-14.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-14.json' + - 'build/build-clang/1stage.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-14 + toolchain: + - linux64-clang-14-stage1 + - macosx64-sdk-toolchain + +linux64-clang-17-mingw-x86: + description: "MinGW-Clang 17 x86 toolchain build" + treeherder: + symbol: TMW(clang-17-x86) + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x86' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-alias: + by-project: + toolchains: null + default: linux64-clang-mingw-x86 + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-17 + - mingw-w64 + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-17 + +linux64-clang-17-mingw-x64: + description: "MinGW-Clang 17 x64 toolchain build" + treeherder: + symbol: TMW(clang-17-x64) + tier: 1 + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x64' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-alias: + by-project: + toolchains: null + default: linux64-clang-mingw-x64 + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-17 + - mingw-w64 + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-17 + +linux64-clang-17-stage1: + description: "Clang 17 toolchain build" + treeherder: + symbol: TL(clang-17-stage1) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/1stage.json' + toolchain-alias: + by-project: + toolchains: null + default: linux64-clang-toolchain + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-toolchain-sysroot + +linux64-clang-17-profile: + description: "Clang 17 toolchain build" + treeherder: + symbol: TL(clang-17-profile) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + toolchain-artifact: public/build/merged.profdata + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-x64-compiler-rt-17 + - linux64-toolchain-sysroot + +linux64-clang-17-raw: + description: "Clang 17 toolchain build" + treeherder: + symbol: TL(clang-17-raw) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-clang-17-profile + - linux64-toolchain-sysroot + +linux64-clang-17: + description: "Clang 17 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(clang-17) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: linux64-clang + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang-17-raw + - android-aarch64-compiler-rt-17 + - android-aarch64-libunwind-17 + - android-arm-compiler-rt-17 + - android-arm-libunwind-17 + - android-x64-compiler-rt-17 + - android-x64-libunwind-17 + - android-x86-compiler-rt-17 + - android-x86-libunwind-17 + - linux64-aarch64-compiler-rt-17 + - linux64-x64-compiler-rt-17 + - linux64-x86-compiler-rt-17 + - macosx64-aarch64-compiler-rt-17 + - macosx64-x64-compiler-rt-17 + - wasm32-wasi-compiler-rt-17 + - win32-compiler-rt-17 + - win64-compiler-rt-17 + +macosx64-clang-17-raw: + description: "Clang 17 toolchain build" + treeherder: + symbol: TM(clang-17-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-clang-17-profile + - macosx64-sdk-toolchain + - macosx64-x64-compiler-rt-17 + +macosx64-clang-17: + description: "Clang 17 toolchain repack with MacOS Compiler RT libs" + attributes: + local-toolchain: true + treeherder: + symbol: TM(clang-17) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: + - macosx64-clang + - macosx64-clang-toolchain + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-clang-17-raw + - android-aarch64-compiler-rt-17 + - android-aarch64-libunwind-17 + - android-arm-compiler-rt-17 + - android-arm-libunwind-17 + - android-x64-compiler-rt-17 + - android-x64-libunwind-17 + - android-x86-compiler-rt-17 + - android-x86-libunwind-17 + - linux64-aarch64-compiler-rt-17 + - linux64-x64-compiler-rt-17 + - linux64-x86-compiler-rt-17 + - macosx64-aarch64-compiler-rt-17 + - macosx64-x64-compiler-rt-17 + - wasm32-wasi-compiler-rt-17 + - win32-compiler-rt-17 + - win64-compiler-rt-17 + +macosx64-aarch64-clang-17-raw: + description: "Clang 17 toolchain build" + treeherder: + symbol: TM(clang-17-aarch64-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-clang-17-profile + - macosx64-sdk-toolchain + - macosx64-aarch64-compiler-rt-17 + +macosx64-aarch64-clang-17: + description: "Clang 17 toolchain repack with MacOS Compiler RT libs" + attributes: + local-toolchain: true + treeherder: + symbol: TM(clang-17-aarch64) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: macosx64-aarch64-clang + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-aarch64-clang-17-raw + - android-aarch64-compiler-rt-17 + - android-aarch64-libunwind-17 + - android-arm-compiler-rt-17 + - android-arm-libunwind-17 + - android-x64-compiler-rt-17 + - android-x64-libunwind-17 + - android-x86-compiler-rt-17 + - android-x86-libunwind-17 + - linux64-aarch64-compiler-rt-17 + - linux64-x64-compiler-rt-17 + - linux64-x86-compiler-rt-17 + - macosx64-aarch64-compiler-rt-17 + - macosx64-x64-compiler-rt-17 + - wasm32-wasi-compiler-rt-17 + - win32-compiler-rt-17 + - win64-compiler-rt-17 + +win64-clang-17-stage1: + description: "Clang-cl 17 toolchain build stage 1" + treeherder: + symbol: TW64(clang-17-stage1) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/1stage.json' + toolchain-alias: + by-project: + toolchains: null + default: win64-clang-toolchain + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - vs-toolchain + +win64-clang-17-raw: + description: "Clang-cl 17 toolchain build" + treeherder: + symbol: TW64(clang-17-raw) + worker-type: b-win2022 + worker: + max-run-time: 9000 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-17.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-17 + - cmake + - ninja + toolchain: + - win64-clang-17-stage1 + - win64-compiler-rt-17 + - vs-toolchain + +win64-clang-17: + description: "Clang-cl 17 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(clang-17) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: win64-clang + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - win64-clang-17-raw + - android-aarch64-compiler-rt-17 + - android-aarch64-libunwind-17 + - android-arm-compiler-rt-17 + - android-arm-libunwind-17 + - android-x64-compiler-rt-17 + - android-x64-libunwind-17 + - android-x86-compiler-rt-17 + - android-x86-libunwind-17 + - linux64-aarch64-compiler-rt-17 + - linux64-x64-compiler-rt-17 + - linux64-x86-compiler-rt-17 + - macosx64-aarch64-compiler-rt-17 + - macosx64-x64-compiler-rt-17 + - wasm32-wasi-compiler-rt-17 + - win32-compiler-rt-17 + - win64-compiler-rt-17 + +linux64-clang-18-mingw-x86: + description: "MinGW-Clang 18 x86 toolchain build" + treeherder: + symbol: TMW(clang-18-x86) + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x86' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-18 + - mingw-w64 + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-18 + +linux64-clang-18-mingw-x64: + description: "MinGW-Clang 18 x64 toolchain build" + treeherder: + symbol: TMW(clang-18-x64) + tier: 1 + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x64' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-18 + - mingw-w64 + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-18 + +linux64-clang-18-stage1: + description: "Clang 18 toolchain build" + treeherder: + symbol: TL(clang-18-stage1) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/1stage.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-toolchain-sysroot + +linux64-clang-18-profile: + description: "Clang 18 toolchain build" + treeherder: + symbol: TL(clang-18-profile) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + toolchain-artifact: public/build/merged.profdata + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-x64-compiler-rt-18 + - linux64-toolchain-sysroot + +linux64-clang-18-raw: + description: "Clang 18 toolchain build" + treeherder: + symbol: TL(clang-18-raw) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-clang-18-profile + - linux64-toolchain-sysroot + +linux64-clang-18: + description: "Clang 18 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(clang-18) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang-18-raw + - android-aarch64-compiler-rt-18 + - android-aarch64-libunwind-18 + - android-arm-compiler-rt-18 + - android-arm-libunwind-18 + - android-x64-compiler-rt-18 + - android-x64-libunwind-18 + - android-x86-compiler-rt-18 + - android-x86-libunwind-18 + - linux64-aarch64-compiler-rt-18 + - linux64-x64-compiler-rt-18 + - linux64-x86-compiler-rt-18 + - macosx64-aarch64-compiler-rt-18 + - macosx64-x64-compiler-rt-18 + - wasm32-wasi-compiler-rt-18 + - win32-compiler-rt-18 + - win64-compiler-rt-18 + +macosx64-clang-18-raw: + description: "Clang 18 toolchain build" + treeherder: + symbol: TM(clang-18-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-clang-18-profile + - macosx64-sdk-toolchain + - macosx64-x64-compiler-rt-18 + +macosx64-clang-18: + description: "Clang 18 toolchain repack with MacOS Compiler RT libs" + attributes: + local-toolchain: true + treeherder: + symbol: TM(clang-18) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-clang-18-raw + - android-aarch64-compiler-rt-18 + - android-aarch64-libunwind-18 + - android-arm-compiler-rt-18 + - android-arm-libunwind-18 + - android-x64-compiler-rt-18 + - android-x64-libunwind-18 + - android-x86-compiler-rt-18 + - android-x86-libunwind-18 + - linux64-aarch64-compiler-rt-18 + - linux64-x64-compiler-rt-18 + - linux64-x86-compiler-rt-18 + - macosx64-aarch64-compiler-rt-18 + - macosx64-x64-compiler-rt-18 + - wasm32-wasi-compiler-rt-18 + - win32-compiler-rt-18 + - win64-compiler-rt-18 + +macosx64-aarch64-clang-18-raw: + description: "Clang 18 toolchain build" + treeherder: + symbol: TM(clang-18-aarch64-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-clang-18-profile + - macosx64-sdk-toolchain + - macosx64-aarch64-compiler-rt-18 + +macosx64-aarch64-clang-18: + description: "Clang 18 toolchain repack with MacOS Compiler RT libs" + attributes: + local-toolchain: true + treeherder: + symbol: TM(clang-18-aarch64) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-aarch64-clang-18-raw + - android-aarch64-compiler-rt-18 + - android-aarch64-libunwind-18 + - android-arm-compiler-rt-18 + - android-arm-libunwind-18 + - android-x64-compiler-rt-18 + - android-x64-libunwind-18 + - android-x86-compiler-rt-18 + - android-x86-libunwind-18 + - linux64-aarch64-compiler-rt-18 + - linux64-x64-compiler-rt-18 + - linux64-x86-compiler-rt-18 + - macosx64-aarch64-compiler-rt-18 + - macosx64-x64-compiler-rt-18 + - wasm32-wasi-compiler-rt-18 + - win32-compiler-rt-18 + - win64-compiler-rt-18 + +win64-clang-18-stage1: + description: "Clang-cl 18 toolchain build stage 1" + treeherder: + symbol: TW64(clang-18-stage1) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/1stage.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - vs-toolchain + +win64-clang-18-raw: + description: "Clang-cl 18 toolchain build" + treeherder: + symbol: TW64(clang-18-raw) + worker-type: b-win2022 + worker: + max-run-time: 9000 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-18.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-18 + - cmake + - ninja + toolchain: + - win64-clang-18-stage1 + - win64-compiler-rt-18 + - vs-toolchain + +win64-clang-18: + description: "Clang-cl 18 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(clang-18) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - win64-clang-18-raw + - android-aarch64-compiler-rt-18 + - android-aarch64-libunwind-18 + - android-arm-compiler-rt-18 + - android-arm-libunwind-18 + - android-x64-compiler-rt-18 + - android-x64-libunwind-18 + - android-x86-compiler-rt-18 + - android-x86-libunwind-18 + - linux64-aarch64-compiler-rt-18 + - linux64-x64-compiler-rt-18 + - linux64-x86-compiler-rt-18 + - macosx64-aarch64-compiler-rt-18 + - macosx64-x64-compiler-rt-18 + - wasm32-wasi-compiler-rt-18 + - win32-compiler-rt-18 + - win64-compiler-rt-18 + +linux64-clang-trunk-mingw-x86: + description: "MinGW-Clang trunk x86 toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TMW(clang-trunk-x86) + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x86' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-alias: + by-project: + toolchains: linux64-clang-mingw-x86 + default: null + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-trunk + - mingw-w64-trunk + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-trunk + +linux64-clang-trunk-mingw-x64: + description: "MinGW-Clang trunk x64 toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TMW(clang-trunk-x64) + tier: 1 + worker-type: b-linux-gcp + run: + script: build-clang-mingw.sh + arguments: + - 'x64' + resources: + - 'taskcluster/scripts/misc/mingw-*.patch' + toolchain-alias: + by-project: + toolchains: linux64-clang-mingw-x64 + default: null + toolchain-artifact: public/build/clangmingw.tar.zst + fetches: + fetch: + - clang-trunk + - mingw-w64-trunk + - llvm-mingw + - gcc-9.5.0 + toolchain: + - linux64-clang-trunk + +linux64-clang-trunk-stage1: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(clang-trunk-stage1) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/1stage.json' + toolchain-alias: + by-project: + toolchains: linux64-clang-toolchain + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-toolchain-sysroot + +linux64-clang-trunk-profile: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(clang-trunk-profile) + run: + using: toolchain-script + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-stage-1.json' + - 'build/build-clang/profile.json' + toolchain-artifact: public/build/merged.profdata + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-x64-compiler-rt-trunk + - linux64-toolchain-sysroot + +linux64-clang-trunk-raw: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(clang-trunk-raw) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/linux64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-clang-trunk-profile + - linux64-toolchain-sysroot + +linux64-clang-trunk: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(clang-trunk) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: linux64-clang + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang-trunk-raw + - android-aarch64-compiler-rt-trunk + - android-aarch64-libunwind-trunk + - android-arm-compiler-rt-trunk + - android-arm-libunwind-trunk + - android-x64-compiler-rt-trunk + - android-x64-libunwind-trunk + - android-x86-compiler-rt-trunk + - android-x86-libunwind-trunk + - linux64-aarch64-compiler-rt-trunk + - linux64-x64-compiler-rt-trunk + - linux64-x86-compiler-rt-trunk + - macosx64-aarch64-compiler-rt-trunk + - macosx64-x64-compiler-rt-trunk + - wasm32-wasi-compiler-rt-trunk + - win32-compiler-rt-trunk + - win64-compiler-rt-trunk + +macosx64-clang-trunk-raw: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TM(clang-trunk-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-clang-trunk-profile + - macosx64-sdk-toolchain + - macosx64-x64-compiler-rt-trunk + +macosx64-clang-trunk: + description: "Clang trunk toolchain repack with MacOS Compiler RT libs" + attributes: + cached_task: false + treeherder: + symbol: TM(clang-trunk) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: + - macosx64-clang + - macosx64-clang-toolchain + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-clang-trunk-raw + - android-aarch64-compiler-rt-trunk + - android-aarch64-libunwind-trunk + - android-arm-compiler-rt-trunk + - android-arm-libunwind-trunk + - android-x64-compiler-rt-trunk + - android-x64-libunwind-trunk + - android-x86-compiler-rt-trunk + - android-x86-libunwind-trunk + - linux64-aarch64-compiler-rt-trunk + - linux64-x64-compiler-rt-trunk + - linux64-x86-compiler-rt-trunk + - macosx64-aarch64-compiler-rt-trunk + - macosx64-x64-compiler-rt-trunk + - wasm32-wasi-compiler-rt-trunk + - win32-compiler-rt-trunk + - win64-compiler-rt-trunk + +macosx64-aarch64-clang-trunk-raw: + description: "Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TM(clang-trunk-aarch64-raw) + worker-type: b-linux-large-gcp + worker: + max-run-time: 3600 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/macosx64.json' + - 'build/build-clang/macosx64-aarch64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-3-stages.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-clang-trunk-profile + - macosx64-sdk-toolchain + - macosx64-aarch64-compiler-rt-trunk + +macosx64-aarch64-clang-trunk: + description: "Clang trunk toolchain repack with MacOS Compiler RT libs" + attributes: + cached_task: false + treeherder: + symbol: TM(clang-trunk-aarch64) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: macosx64-aarch64-clang + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - macosx64-aarch64-clang-trunk-raw + - android-aarch64-compiler-rt-trunk + - android-aarch64-libunwind-trunk + - android-arm-compiler-rt-trunk + - android-arm-libunwind-trunk + - android-x64-compiler-rt-trunk + - android-x64-libunwind-trunk + - android-x86-compiler-rt-trunk + - android-x86-libunwind-trunk + - linux64-aarch64-compiler-rt-trunk + - linux64-x64-compiler-rt-trunk + - linux64-x86-compiler-rt-trunk + - macosx64-aarch64-compiler-rt-trunk + - macosx64-x64-compiler-rt-trunk + - wasm32-wasi-compiler-rt-trunk + - win32-compiler-rt-trunk + - win64-compiler-rt-trunk + +win64-clang-trunk-stage1: + description: "Clang-cl trunk toolchain build stage 1" + attributes: + cached_task: false + treeherder: + symbol: TW64(clang-trunk-stage1) + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/1stage.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/1stage.json' + toolchain-alias: + by-project: + toolchains: win64-clang-toolchain + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - vs-toolchain + +win64-clang-trunk-raw: + description: "Clang-cl trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TW64(clang-trunk-raw) + worker-type: b-win2022 + worker: + max-run-time: 9000 + run: + script: build-clang.sh + arguments: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + resources: + - 'build/build-clang/win64.json' + - 'build/build-clang/clang-trunk.json' + - 'build/build-clang/skip-stage-1-win64.json' + - 'build/build-clang/4stages-pgo.json' + toolchain-artifact: public/build/clang.tar.zst + fetches: + fetch: + - clang-trunk + - cmake + - ninja + toolchain: + - win64-clang-trunk-stage1 + - win64-compiler-rt-trunk + - vs-toolchain + +win64-clang-trunk: + description: "Clang-cl trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TW64(clang-trunk) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: win64-clang + default: null + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - win64-clang-trunk-raw + - android-aarch64-compiler-rt-trunk + - android-aarch64-libunwind-trunk + - android-arm-compiler-rt-trunk + - android-arm-libunwind-trunk + - android-x64-compiler-rt-trunk + - android-x64-libunwind-trunk + - android-x86-compiler-rt-trunk + - android-x86-libunwind-trunk + - linux64-aarch64-compiler-rt-trunk + - linux64-x64-compiler-rt-trunk + - linux64-x86-compiler-rt-trunk + - macosx64-aarch64-compiler-rt-trunk + - macosx64-x64-compiler-rt-trunk + - wasm32-wasi-compiler-rt-trunk + - win32-compiler-rt-trunk + - win64-compiler-rt-trunk diff --git a/taskcluster/ci/toolchain/compiler-rt.yml b/taskcluster/ci/toolchain/compiler-rt.yml new file mode 100644 index 0000000000..631024fbf9 --- /dev/null +++ b/taskcluster/ci/toolchain/compiler-rt.yml @@ -0,0 +1,705 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + using: toolchain-script + script: build-compiler-rt.sh + +wasm32-wasi-compiler-rt-8.0: + description: "wasm32-wasi Compiler-rt for Clang 8 toolchain build" + treeherder: + symbol: TL(wasi-crt-8) + worker-type: b-linux-xlarge-gcp + run: + script: build-compiler-rt-wasi.sh + arguments: + - wasi-sdk-11.patch + resources: + - taskcluster/scripts/misc/wasi-sdk-11.patch + toolchain-artifact: public/build/compiler-rt-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-8.0 + - wasi-sdk-11 + toolchain: + - linux64-clang-8.0-raw + +android-aarch64-compiler-rt-17: + description: "android aarch64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TA(aarch64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-arm-compiler-rt-17: + description: "android arm Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TA(arm-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-x86-compiler-rt-17: + description: "android x86 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TA(x86-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-x64-compiler-rt-17: + description: "android x64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TA(x64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +linux64-x86-compiler-rt-17: + description: "Linux x86 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TL(x86-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - sysroot-i686-linux-gnu + +linux64-x64-compiler-rt-17: + description: "Linux x64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TL(x64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - sysroot-x86_64-linux-gnu + +linux64-aarch64-compiler-rt-17: + description: "Linux aarch64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TL(aarch64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - sysroot-aarch64-linux-gnu + +macosx64-x64-compiler-rt-17: + description: "macOS x64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TM(x64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-apple-darwin.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-compiler-rt-17: + description: "macOS aarch64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TM(aarch64-crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-apple-darwin.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - macosx64-sdk-toolchain + +win32-compiler-rt-17: + description: "win32 x86 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TW32(crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - vs-toolchain + +win64-compiler-rt-17: + description: "win64 x64 Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TW64(crt-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - vs-toolchain + +wasm32-wasi-compiler-rt-17: + description: "wasm32-wasi Compiler-rt for Clang 17 toolchain build" + treeherder: + symbol: TL(wasi-crt-17) + worker-type: b-linux-xlarge-gcp + run: + script: build-compiler-rt-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-artifact: public/build/compiler-rt-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-17 + - wasi-sdk + toolchain: + - linux64-clang-17-stage1 + +android-aarch64-compiler-rt-18: + description: "android aarch64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TA(aarch64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-arm-compiler-rt-18: + description: "android arm Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TA(arm-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-x86-compiler-rt-18: + description: "android x86 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TA(x86-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-x64-compiler-rt-18: + description: "android x64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TA(x64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +linux64-x86-compiler-rt-18: + description: "Linux x86 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TL(x86-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - sysroot-i686-linux-gnu + +linux64-x64-compiler-rt-18: + description: "Linux x64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TL(x64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - sysroot-x86_64-linux-gnu + +linux64-aarch64-compiler-rt-18: + description: "Linux aarch64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TL(aarch64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - sysroot-aarch64-linux-gnu + +macosx64-x64-compiler-rt-18: + description: "macOS x64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TM(x64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-apple-darwin.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-compiler-rt-18: + description: "macOS aarch64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TM(aarch64-crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-apple-darwin.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - macosx64-sdk-toolchain + +win32-compiler-rt-18: + description: "win32 x86 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TW32(crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - vs-toolchain + +win64-compiler-rt-18: + description: "win64 x64 Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TW64(crt-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - vs-toolchain + +wasm32-wasi-compiler-rt-18: + description: "wasm32-wasi Compiler-rt for Clang 18 toolchain build" + treeherder: + symbol: TL(wasi-crt-18) + worker-type: b-linux-xlarge-gcp + run: + script: build-compiler-rt-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-artifact: public/build/compiler-rt-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-18 + - wasi-sdk + toolchain: + - linux64-clang-18-stage1 + +android-aarch64-compiler-rt-trunk: + description: "android aarch64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(aarch64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-arm-compiler-rt-trunk: + description: "android arm Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(arm-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-x86-compiler-rt-trunk: + description: "android x86 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(x86-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-x64-compiler-rt-trunk: + description: "android x64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(x64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +linux64-x86-compiler-rt-trunk: + description: "Linux x86 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(x86-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - sysroot-i686-linux-gnu + +linux64-x64-compiler-rt-trunk: + description: "Linux x64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(x64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - sysroot-x86_64-linux-gnu + +linux64-aarch64-compiler-rt-trunk: + description: "Linux aarch64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(aarch64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-unknown-linux-gnu.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - sysroot-aarch64-linux-gnu + +macosx64-x64-compiler-rt-trunk: + description: "macOS x64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TM(x64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-apple-darwin.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-compiler-rt-trunk: + description: "macOS aarch64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TM(aarch64-crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-aarch64-apple-darwin.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - macosx64-sdk-toolchain + +win32-compiler-rt-trunk: + description: "win32 x86 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TW32(crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-i686-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - vs-toolchain + +win64-compiler-rt-trunk: + description: "win64 x64 Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TW64(crt-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/compiler-rt-x86_64-pc-windows-msvc.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - vs-toolchain + +wasm32-wasi-compiler-rt-trunk: + description: "wasm32-wasi Compiler-rt for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TL(wasi-crt-trunk) + worker-type: b-linux-xlarge-gcp + run: + script: build-compiler-rt-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-artifact: public/build/compiler-rt-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-trunk + - wasi-sdk + toolchain: + - linux64-clang-trunk-stage1 diff --git a/taskcluster/ci/toolchain/dist-toolchains.yml b/taskcluster/ci/toolchain/dist-toolchains.yml new file mode 100644 index 0000000000..31b60918e6 --- /dev/null +++ b/taskcluster/ci/toolchain/dist-toolchains.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "sccache-dist toolchain archive build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + run: + using: toolchain-script + script: build-dist-toolchains.sh + +clang-dist-toolchain: + treeherder: + symbol: TL(clang-dist) + worker: + max-run-time: 1800 + run: + arguments: ['clang'] + toolchain-artifact: public/build/clang-dist-toolchain.tar.xz + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-sccache + +rustc-dist-toolchain: + treeherder: + symbol: TL(rustc-dist) + worker: + max-run-time: 1800 + run: + arguments: ['rustc'] + toolchain-artifact: public/build/rustc-dist-toolchain.tar.xz + use-sccache: true + fetches: + toolchain: + - linux64-rust-cross + - linux64-sccache diff --git a/taskcluster/ci/toolchain/dump-syms.yml b/taskcluster/ci/toolchain/dump-syms.yml new file mode 100644 index 0000000000..95e90b5984 --- /dev/null +++ b/taskcluster/ci/toolchain/dump-syms.yml @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + attributes: + local-toolchain: true + description: "dump_syms toolchain build" + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/dump_syms.tar.zst + fetches: + fetch: + - dump-syms + +linux64-dump_syms: + treeherder: + symbol: TL(dump_syms) + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-dump_syms: + treeherder: + symbol: TM(dump_syms) + worker-type: b-linux-gcp + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-dump_syms: + treeherder: + symbol: TM(dump_syms-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win64-dump_syms: + treeherder: + symbol: TW64(dump_syms) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/fix-stacks.yml b/taskcluster/ci/toolchain/fix-stacks.yml new file mode 100644 index 0000000000..e11bf0b166 --- /dev/null +++ b/taskcluster/ci/toolchain/fix-stacks.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "fix-stacks toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/fix-stacks.tar.zst + fetches: + fetch: + - fix-stacks + +linux64-fix-stacks: + treeherder: + symbol: TL(fix-stacks) + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-fix-stacks: + treeherder: + symbol: TM(fix-stacks) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-fix-stacks: + treeherder: + symbol: TM(fix-stacks-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos-toolchain + - macosx64-sdk-toolchain + +win32-fix-stacks: + treeherder: + symbol: TW32(fix-stacks) + run: + arguments: ['i686-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain + +win64-fix-stacks: + treeherder: + symbol: TW64(fix-stacks) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/gcc.yml b/taskcluster/ci/toolchain/gcc.yml new file mode 100644 index 0000000000..2a09c37e6c --- /dev/null +++ b/taskcluster/ci/toolchain/gcc.yml @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + run: + resources: + - 'build/unix/build-gcc/build-gcc.sh' + toolchain-artifact: public/build/gcc.tar.zst + fetches: + toolchain: + - linux64-toolchain-sysroot + +linux64-gcc-8: + description: "GCC 8 toolchain build" + treeherder: + symbol: TL(gcc8) + run: + script: build-gcc-linux.sh + toolchain-alias: + - linux64-gcc + - linux64-gcc-base + fetches: + fetch: + - gcc-8.5.0 + - gmp-6.1.0 + - isl-0.16.1 + - mpc-1.0.3 + - mpfr-3.1.4 + +linux64-gcc-9: + description: "GCC 9 toolchain build" + treeherder: + symbol: TL(gcc9) + run: + script: build-gcc-linux.sh + fetches: + fetch: + - gcc-9.5.0 + - gmp-6.1.0 + - isl-0.16.1 + - mpc-1.0.3 + - mpfr-3.1.4 + +linux64-gcc-11: + description: "GCC 11 toolchain build" + treeherder: + symbol: TL(gcc11) + run: + script: build-gcc-linux.sh + fetches: + fetch: + - gcc-11.4.0 + - gmp-6.1.0 + - isl-0.16.1 + - mpc-1.0.3 + - mpfr-3.1.4 + +linux64-gcc-sixgill: + description: "sixgill GCC plugin build" + treeherder: + symbol: TL(sixgill) + run: + script: build-gcc-sixgill-plugin-linux.sh + resources: + - 'taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh' + toolchain-artifact: public/build/sixgill.tar.xz + fetches: + fetch: + - gcc-9.5.0 + - gmp-6.1.0 + - isl-0.16.1 + - mpc-1.0.3 + - mpfr-3.1.4 + toolchain: + - linux64-gcc-9 diff --git a/taskcluster/ci/toolchain/geckodriver.yml b/taskcluster/ci/toolchain/geckodriver.yml new file mode 100644 index 0000000000..252befa3fb --- /dev/null +++ b/taskcluster/ci/toolchain/geckodriver.yml @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "geckodriver toolchain build" + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run-on-projects: ['trunk'] + attributes: + build_type: opt + geckodriver: true + shippable: true + rebuild-on-release: true + run: + script: build-geckodriver.sh + toolchain-artifact: public/build/geckodriver.tar.gz + sparse-profile: null + resources: + - 'testing/geckodriver' + - 'testing/mozbase/rust' + - 'testing/webdriver' + # Changes to 'third_party/rust' should be captured by + # 'testing/geckodriver/Cargo.toml'. + +# For Linux platforms target "*-musl" so that the binary will be statically linked. + +linux32-geckodriver: + treeherder: + symbol: TL32(gd) + platform: linux32/opt + run: + arguments: ['i686-unknown-linux-musl'] + attributes: + build_platform: linux-geckodriver + fetches: + toolchain: + - linux64-rust-static + +linux64-geckodriver: + treeherder: + symbol: TL(gd) + platform: linux64/opt + run: + arguments: ['x86_64-unknown-linux-musl'] + attributes: + build_platform: linux64-geckodriver + fetches: + toolchain: + - linux64-rust-static + +linux64-aarch64-geckodriver: + treeherder: + symbol: TL(gd) + platform: linux64-aarch64/opt + run: + arguments: ['aarch64-unknown-linux-musl'] + attributes: + build_platform: linux64-aarch64-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-static + +macosx64-geckodriver: + treeherder: + symbol: TM(gd) + platform: macosx64/opt + run: + arguments: ['x86_64-apple-darwin'] + attributes: + build_platform: macosx64-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos + - macosx64-sdk-toolchain + +macosx64-aarch64-geckodriver: + treeherder: + symbol: TM(gd) + platform: macosx64-aarch64/opt + run: + arguments: ['aarch64-apple-darwin'] + attributes: + build_platform: macosx64-aarch64-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-macos + - macosx64-sdk-toolchain + +win32-geckodriver: + treeherder: + symbol: TW32(gd) + platform: win32/opt + run: + arguments: ['i686-pc-windows-msvc'] + toolchain-artifact: public/build/geckodriver.zip + attributes: + build_platform: win32-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows + - vs-toolchain + +win64-aarch64-geckodriver: + treeherder: + symbol: TW64(gd) + platform: windows2012-aarch64/opt + run: + arguments: ['aarch64-pc-windows-msvc'] + toolchain-artifact: public/build/geckodriver.zip + attributes: + build_platform: win64-aarch64-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows + - vs-toolchain + +win64-geckodriver: + treeherder: + symbol: TW64(gd) + platform: win64/opt + run: + arguments: ['x86_64-pc-windows-msvc'] + toolchain-artifact: public/build/geckodriver.zip + attributes: + build_platform: win64-geckodriver + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows + - vs-toolchain diff --git a/taskcluster/ci/toolchain/gn.yml b/taskcluster/ci/toolchain/gn.yml new file mode 100644 index 0000000000..d12b0f19c5 --- /dev/null +++ b/taskcluster/ci/toolchain/gn.yml @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "gn toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + resources: + - 'taskcluster/scripts/misc/build-gn-common.sh' + toolchain-artifact: public/build/gn.tar.zst + fetches: + fetch: + - gn + +linux64-gn: + treeherder: + symbol: TL(gn) + run: + script: build-gn-linux.sh + +macosx64-gn: + treeherder: + symbol: TM(gn) + run: + script: build-gn-macosx.sh + toolchain-alias: macosx64-aarch64-gn + fetches: + toolchain: + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win64-gn: + treeherder: + symbol: TW64(gn) + worker-type: b-win2022 + worker: + max-run-time: 3600 + run: + script: build-gn-win64.sh + fetches: + fetch: + - ninja + toolchain: + - vs-toolchain diff --git a/taskcluster/ci/toolchain/grcov.yml b/taskcluster/ci/toolchain/grcov.yml new file mode 100644 index 0000000000..15fed68732 --- /dev/null +++ b/taskcluster/ci/toolchain/grcov.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "grcov toolchain build" + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/grcov.tar.zst + fetches: + fetch: + - grcov + +linux64-grcov: + treeherder: + symbol: TL(grcov) + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-rust-toolchain + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-grcov: + treeherder: + symbol: TM(grcov) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win64-grcov: + treeherder: + symbol: TW64(grcov) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml new file mode 100644 index 0000000000..d9dd52c980 --- /dev/null +++ b/taskcluster/ci/toolchain/kind.yml @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + - packages + +transforms: + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.cached_tasks:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + run-on-projects: [] + run: + using: toolchain-script + +jobs-from: + - android.yml + - cargo-vet.yml + - cbindgen.yml + - clang-tidy.yml + - clang.yml + - compiler-rt.yml + - dump-syms.yml + - fix-stacks.yml + - gcc.yml + - geckodriver.yml + - gn.yml + - grcov.yml + - libunwind.yml + - llvm-symbolizer.yml + - macos-sdk.yml + - mingw.yml + - minidump-stackwalk.yml + - misc.yml + - nasm.yml + - node.yml + - pkgconf.yml + - python.yml + - resourcemonitor.yml + - rust.yml + - rust-size.yml + - sccache.yml + - sysroot.yml + - dist-toolchains.yml diff --git a/taskcluster/ci/toolchain/libunwind.yml b/taskcluster/ci/toolchain/libunwind.yml new file mode 100644 index 0000000000..5a1972f3f6 --- /dev/null +++ b/taskcluster/ci/toolchain/libunwind.yml @@ -0,0 +1,247 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + using: toolchain-script + script: build-libunwind.sh + +android-aarch64-libunwind-17: + description: "android aarch64 libunwind for Clang 17 toolchain build" + treeherder: + symbol: TA(aarch64-unwind-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - android-aarch64-compiler-rt-17 + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-arm-libunwind-17: + description: "android arm libunwind for Clang 17 toolchain build" + treeherder: + symbol: TA(arm-unwind-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - android-arm-compiler-rt-17 + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-x86-libunwind-17: + description: "android x86 libunwind for Clang 17 toolchain build" + treeherder: + symbol: TA(x86-unwind-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-i686-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - android-x86-compiler-rt-17 + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-x64-libunwind-17: + description: "android x64 libunwind for Clang 17 toolchain build" + treeherder: + symbol: TA(x64-unwind-17) + run: + arguments: + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-17 + toolchain: + - android-x64-compiler-rt-17 + - linux64-clang-17-stage1 + - linux64-android-ndk-linux-repack + +android-aarch64-libunwind-18: + description: "android aarch64 libunwind for Clang 18 toolchain build" + treeherder: + symbol: TA(aarch64-unwind-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - android-aarch64-compiler-rt-18 + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-arm-libunwind-18: + description: "android arm libunwind for Clang 18 toolchain build" + treeherder: + symbol: TA(arm-unwind-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - android-arm-compiler-rt-18 + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-x86-libunwind-18: + description: "android x86 libunwind for Clang 18 toolchain build" + treeherder: + symbol: TA(x86-unwind-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-i686-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - android-x86-compiler-rt-18 + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-x64-libunwind-18: + description: "android x64 libunwind for Clang 18 toolchain build" + treeherder: + symbol: TA(x64-unwind-18) + run: + arguments: + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-18 + toolchain: + - android-x64-compiler-rt-18 + - linux64-clang-18-stage1 + - linux64-android-ndk-linux-repack + +android-aarch64-libunwind-trunk: + description: "android aarch64 libunwind for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(aarch64-unwind-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-aarch64-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - android-aarch64-compiler-rt-trunk + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-arm-libunwind-trunk: + description: "android arm libunwind for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(arm-unwind-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-armv7-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - android-arm-compiler-rt-trunk + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-x86-libunwind-trunk: + description: "android x86 libunwind for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(x86-unwind-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-i686-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - android-x86-compiler-rt-trunk + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack + +android-x64-libunwind-trunk: + description: "android x64 libunwind for Clang trunk toolchain build" + attributes: + cached_task: false + treeherder: + symbol: TA(x64-unwind-trunk) + run: + arguments: + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-artifact: public/build/libunwind-x86_64-linux-android.tar.zst + fetches: + fetch: + - clang-trunk + toolchain: + - android-x64-compiler-rt-trunk + - linux64-clang-trunk-stage1 + - linux64-android-ndk-linux-repack diff --git a/taskcluster/ci/toolchain/llvm-symbolizer.yml b/taskcluster/ci/toolchain/llvm-symbolizer.yml new file mode 100644 index 0000000000..8864a81cc1 --- /dev/null +++ b/taskcluster/ci/toolchain/llvm-symbolizer.yml @@ -0,0 +1,331 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-large-gcp + worker: + max-run-time: 1800 + run: + script: build-llvm-symbolizer.sh + toolchain-artifact: public/build/llvm-symbolizer.tar.zst + +linux32-llvm-symbolizer-17: + description: "llvm-symbolizer for Linux" + treeherder: + symbol: TL32(llvm-symbolizer) + run: + arguments: + - i686-unknown-linux-gnu + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: null + default: linux32-llvm-symbolizer + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux32-toolchain-sysroot + +linux64-llvm-symbolizer-17: + description: "llvm-symbolizer for Linux" + treeherder: + symbol: TL(llvm-symbolizer) + run: + arguments: + - x86_64-unknown-linux-gnu + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: null + default: linux64-llvm-symbolizer + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - linux64-toolchain-sysroot + +macosx64-llvm-symbolizer-17: + description: "llvm-symbolizer for macOS x64" + treeherder: + symbol: TM(llvm-symbolizer) + run: + arguments: + - x86_64-apple-darwin + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: null + default: macosx64-llvm-symbolizer + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-llvm-symbolizer-17: + description: "llvm-symbolizer for macOS aarch64" + treeherder: + symbol: TM(llvm-symbolizer-aarch64) + run: + arguments: + - aarch64-apple-darwin + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: null + default: macosx64-aarch64-llvm-symbolizer + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - macosx64-sdk-toolchain + +win64-llvm-symbolizer-17: + description: "llvm-symbolizer for Windows" + treeherder: + symbol: TW64(llvm-symbolizer) + run: + arguments: + - x86_64-pc-windows-msvc + - build/build-clang/clang-17.json + resources: + - build/build-clang/clang-17.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: null + default: win64-llvm-symbolizer + fetches: + fetch: + - clang-17 + toolchain: + - linux64-clang-17-stage1 + - vs-toolchain + +linux32-llvm-symbolizer-18: + description: "llvm-symbolizer for Linux" + treeherder: + symbol: TL32(llvm-symbolizer-18) + run: + arguments: + - i686-unknown-linux-gnu + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux32-toolchain-sysroot + +linux64-llvm-symbolizer-18: + description: "llvm-symbolizer for Linux" + treeherder: + symbol: TL(llvm-symbolizer-18) + run: + arguments: + - x86_64-unknown-linux-gnu + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - linux64-toolchain-sysroot + +macosx64-llvm-symbolizer-18: + description: "llvm-symbolizer for macOS x64" + treeherder: + symbol: TM(llvm-symbolizer-18) + run: + arguments: + - x86_64-apple-darwin + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-llvm-symbolizer-18: + description: "llvm-symbolizer for macOS aarch64" + treeherder: + symbol: TM(llvm-sym-aarch64-18) + run: + arguments: + - aarch64-apple-darwin + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - macosx64-sdk-toolchain + +win64-llvm-symbolizer-18: + description: "llvm-symbolizer for Windows" + treeherder: + symbol: TW64(llvm-symbolizer-18) + run: + arguments: + - x86_64-pc-windows-msvc + - build/build-clang/clang-18.json + resources: + - build/build-clang/clang-18.json + - taskcluster/scripts/misc/build-llvm-common.sh + fetches: + fetch: + - clang-18 + toolchain: + - linux64-clang-18-stage1 + - vs-toolchain + +linux32-llvm-symbolizer-trunk: + description: "llvm-symbolizer for Linux" + attributes: + cached_task: false + treeherder: + symbol: TL32(llvm-symbolizer-trunk) + run: + arguments: + - i686-unknown-linux-gnu + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: linux32-llvm-symbolizer + default: null + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux32-toolchain-sysroot + +linux64-llvm-symbolizer-trunk: + description: "llvm-symbolizer for Linux" + attributes: + cached_task: false + treeherder: + symbol: TL(llvm-symbolizer-trunk) + run: + arguments: + - x86_64-unknown-linux-gnu + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: linux64-llvm-symbolizer + default: null + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - linux64-toolchain-sysroot + +macosx64-llvm-symbolizer-trunk: + description: "llvm-symbolizer for macOS x64" + attributes: + cached_task: false + treeherder: + symbol: TM(llvm-symbolizer-trunk) + run: + arguments: + - x86_64-apple-darwin + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: macosx64-llvm-symbolizer + default: null + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - macosx64-sdk-toolchain + +macosx64-aarch64-llvm-symbolizer-trunk: + description: "llvm-symbolizer for macOS aarch64" + attributes: + cached_task: false + treeherder: + symbol: TM(llvm-sym-aarch64-trunk) + run: + arguments: + - aarch64-apple-darwin + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: macosx64-aarch64-llvm-symbolizer + default: null + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - macosx64-sdk-toolchain + +win64-llvm-symbolizer-trunk: + description: "llvm-symbolizer for Windows" + attributes: + cached_task: false + treeherder: + symbol: TW64(llvm-symbolizer-trunk) + run: + arguments: + - x86_64-pc-windows-msvc + - build/build-clang/clang-trunk.json + resources: + - build/build-clang/clang-trunk.json + - taskcluster/scripts/misc/build-llvm-common.sh + toolchain-alias: + by-project: + toolchains: win64-llvm-symbolizer + default: null + fetches: + fetch: + - clang-trunk + toolchain: + - linux64-clang-trunk-stage1 + - vs-toolchain diff --git a/taskcluster/ci/toolchain/macos-sdk.yml b/taskcluster/ci/toolchain/macos-sdk.yml new file mode 100644 index 0000000000..d3e2bc6493 --- /dev/null +++ b/taskcluster/ci/toolchain/macos-sdk.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + env: + PATH: "/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin" + run: + script: unpack-sdk.py + resources: + - python/mozbuild/mozpack/macpkg.py + - taskcluster/scripts/misc/unpack-sdk.py + tooltool-downloads: internal + # Expiry similar to that of fetches + expires-after: 1000 years + +macosx64-sdk-13.3: + description: "MacOSX13.3 SDK" + treeherder: + symbol: TM(sdk13.3) + run: + arguments: + - https://swcdn.apple.com/content/downloads/15/62/032-84673-A_7A1TG1RF8Z/xpc8q44ggn2pkn82iwr0fi1zeb9cxi8ath/CLTools_macOSNMOS_SDK.pkg + - 8de3306508dca31040b3b49ff370e96c94f4546bcf22cb4069abbb18e2d6bce8e20d40a6903f00ae75558175b624a979ea0444f54bc6d9d67e841f2000b0d863 + - Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk + toolchain-artifact: project/gecko/mac-sdk/MacOSX13.3.sdk.tar.zst + toolchain-alias: + - MacOSX13.3.sdk + +macosx64-sdk-14.2: + description: "MacOSX14.2 SDK" + treeherder: + symbol: TM(sdk14.2) + run: + arguments: + - https://swcdn.apple.com/content/downloads/35/47/042-23550-A_M34N50O9N3/sua6ev7bjcexvorh4709vsgk64zgvsl42p/CLTools_macOSNMOS_SDK.pkg + - 22ce0acc9ab5c28845218b8c686d0208d6bf66bef64a369d3c0622247ba1258b438c9e44c64bb2bacf5e99eb398a6e7a337f2730cd4a969a7f0d2d5d3107306a + - Library/Developer/CommandLineTools/SDKs/MacOSX14.2.sdk + toolchain-artifact: project/gecko/mac-sdk/MacOSX14.2.sdk.tar.zst + toolchain-alias: + - macosx64-sdk + - macosx64-sdk-toolchain + - MacOSX14.2.sdk diff --git a/taskcluster/ci/toolchain/mingw.yml b/taskcluster/ci/toolchain/mingw.yml new file mode 100644 index 0000000000..a14413669f --- /dev/null +++ b/taskcluster/ci/toolchain/mingw.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + treeherder: + tier: 2 + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + +linux64-mingw32-nsis: + description: "NSIS build for MinGW32 Cross Compile" + treeherder: + symbol: TMW(mingw32-nsis) + run: + script: build-mingw32-nsis.sh + toolchain-artifact: public/build/nsis.tar.zst + fetches: + fetch: + - nsis-3.07 + - zlib-1.3.1 + toolchain: + - linux64-binutils + - linux64-clang-mingw-x86 + - sysroot-x86_64-linux-gnu + +linux64-mingw-fxc2-x86: + description: "fxc2.exe x86 build for MinGW Cross Compile" + treeherder: + symbol: TMW(mingw-fxc2-x86) + worker: + max-run-time: 1800 + run: + using: toolchain-script + script: build-mingw-fxc2-x86.sh + toolchain-artifact: public/build/fxc2.tar.zst + fetches: + fetch: + - fxc2 + toolchain: + - linux64-clang-mingw-x86 diff --git a/taskcluster/ci/toolchain/minidump-stackwalk.yml b/taskcluster/ci/toolchain/minidump-stackwalk.yml new file mode 100644 index 0000000000..f0f9b3e70c --- /dev/null +++ b/taskcluster/ci/toolchain/minidump-stackwalk.yml @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "minidump-stackwalk toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + env: + FETCH: rust-minidump/minidump-stackwalk + run: + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/minidump-stackwalk.tar.zst + fetches: + fetch: + - rust-minidump + +linux64-minidump-stackwalk: + treeherder: + symbol: TL(stackwalk) + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-minidump-stackwalk: + treeherder: + symbol: TM(stackwalk) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-minidump-stackwalk: + treeherder: + symbol: TM(stackwalk-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win32-minidump-stackwalk: + treeherder: + symbol: TW32(stackwalk) + run: + arguments: ['i686-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain + +win64-minidump-stackwalk: + treeherder: + symbol: TW64(stackwalk) + run: + arguments: ['x86_64-pc-windows-msvc'] + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain + +# This is a totally different thing but it used to share common code with +# minidump-stackwalk, so it lives here for historical reasons. +linux64-breakpad-injector: + description: "breakpad injector toolchain build" + run: + script: build-breakpad-injector.sh + toolchain-artifact: public/build/injector.tar.zst + sparse-profile: null + resources: + - 'build/moz.configure' + - 'config/external/zlib' + - 'moz.configure' + - 'toolkit/crashreporter' + - 'toolkit/crashreporter/google-breakpad/src/processor' + - 'tools/crashreporter/' + treeherder: + symbol: TL(injector) + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot diff --git a/taskcluster/ci/toolchain/misc.yml b/taskcluster/ci/toolchain/misc.yml new file mode 100644 index 0000000000..f19201bc1e --- /dev/null +++ b/taskcluster/ci/toolchain/misc.yml @@ -0,0 +1,605 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + +linux64-binutils-2.31.1: + description: "Binutils toolchain build" + treeherder: + symbol: TL(binutil2.31) + run: + script: build-binutils-linux.sh + resources: + - 'build/unix/build-binutils/build-binutils.sh' + toolchain-artifact: public/build/binutils.tar.zst + fetches: + fetch: + - binutils-2.31.1 + toolchain: + # C++ code in gold doesn't compile with recent versions of GCC. + - linux64-gcc-8 + - linux64-toolchain-sysroot + +linux64-binutils: + description: "Binutils toolchain build" + treeherder: + symbol: TL(binutil) + run: + script: build-binutils-linux.sh + resources: + - 'build/unix/build-binutils/build-binutils.sh' + toolchain-artifact: public/build/binutils.tar.zst + fetches: + fetch: + - binutils-2.41 + toolchain: + - linux64-toolchain-sysroot + +linux64-cctools-port: + description: "cctools-port toolchain build" + treeherder: + symbol: TL(cctools-clang) + run: + script: build-cctools-port.sh + toolchain-artifact: public/build/cctools.tar.zst + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + fetch: + - cctools-port + - libtapi + - ldid + +linux64-hfsplus: + description: "hfsplus toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(hfs+) + run: + script: build-hfsplus-linux.sh + resources: + - 'build/unix/build-hfsplus/build-hfsplus.sh' + toolchain-artifact: public/build/hfsplus.tar.zst + fetches: + fetch: + - hfsplus-tools + toolchain: + - linux64-clang-toolchain + +linux64-libdmg: + description: "libdmg-hfsplus toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(libdmg-hfs+) + run: + script: build-dmg-hfsplus.sh + toolchain-artifact: public/build/dmg.tar.zst + toolchain-alias: linux64-dmg + fetches: + fetch: + - libdmg-hfsplus + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + +linux64-mkbom: + description: "mkbom toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(mkbom) + run: + script: build-mkbom-linux.sh + toolchain-artifact: public/build/mkbom.tar.zst + fetches: + fetch: + - bomutils + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + +linux64-xar: + description: "xar toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(xar) + run: + script: build-xar-linux.sh + toolchain-artifact: public/build/xar.tar.zst + fetches: + fetch: + - xar + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + +linux64-mar-tools: + description: "mar-tools toolchain build" + treeherder: + symbol: TL(mar-tools) + fetches: + toolchain: + - linux64-clang-toolchain + run: + script: build-mar-tools.sh + sparse-profile: null + toolchain-artifact: public/build/mar-tools.tar.zst + resources: + - build/moz.configure + - modules/libmar/ + - moz.configure + - other-licenses/bsdiff/ + - other-licenses/nsis/Contrib/CityHash/cityhash/ + - toolkit/mozapps/update/updater/bspatch/ + - tools/update-packaging/ + +linux64-upx: + description: "UPX build" + treeherder: + symbol: TL(upx) + tier: 1 + run: + script: build-upx.sh + toolchain-artifact: public/build/upx.tar.zst + +linux64-msix-packaging: + description: "msix-packaging build" + treeherder: + symbol: TL(msix-packaging) + tier: 1 + run: + script: build-msix-packaging.sh + toolchain-artifact: public/build/msix-packaging.tar.zst + fetches: + fetch: + - msix-packaging + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + +linux64-custom-v8: + description: "Custom v8 build" + treeherder: + symbol: TL(custom-v8) + worker: + docker-image: {in-tree: custom-v8} + max-run-time: 3600 + run: + script: build-custom-v8.sh + arguments: [ + d8.tar.zst, + is_debug=false, + target_cpu="x64" + ] + toolchain-artifact: public/build/d8.tar.zst + +macosx64-xz: + description: "xz development library" + worker-type: b-osx-1015 + treeherder: + symbol: TM(xz) + worker: + max-run-time: 30000 + run: + script: build-xz.sh + toolchain-artifact: public/build/xz.tar.zst + fetches: + fetch: + - xz-5.4.4 + toolchain: + - macosx64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-custom-car: + description: "Custom chromium-as-release build for mac" + worker-type: b-osx-1015 + attributes: + cached_task: false + treeherder: + symbol: TM(custom-car) + worker: + max-run-time: 35000 + use-system-python: false + run-on-projects: [] + run: + script: build-custom-car.sh + arguments: [ + car_macosx.tar.zst, + blink_symbol_level=0, + chrome_pgo_phase=2, + dcheck_always_on=false, + enable_nacl=false, + enable_resource_allowlist_generation=false, + is_chrome_branded=false, + is_clang=true, + is_component_build=false, + is_debug=false, + is_official_build=true, + symbol_level=0, + target_cpu="x64", + use_v8_context_snapshot=false, + v8_symbol_level=0 + ] + toolchain-artifact: public/build/car_macosx.tar.zst + fetches: + toolchain: + - macosx64-sdk-14.2 + +macosx-arm64-custom-car: + description: "Custom chromium-as-release build for mac (apple silicon)" + worker-type: b-osx-1015 + attributes: + cached_task: false + treeherder: + symbol: TM(custom-car-arm) + tier: 2 + worker: + max-run-time: 50000 + use-system-python: false + run-on-projects: [] + run: + script: build-custom-car.sh + arguments: [ + car_macosx_arm.tar.zst, + blink_symbol_level=0, + chrome_pgo_phase=2, + dcheck_always_on=false, + enable_nacl=false, + enable_resource_allowlist_generation=false, + is_chrome_branded=false, + is_clang=true, + is_component_build=false, + is_debug=false, + is_official_build=true, + symbol_level=0, + target_cpu="arm64", + use_v8_context_snapshot=false, + v8_symbol_level=0 + ] + toolchain-artifact: public/build/car_macosx_arm.tar.zst + fetches: + toolchain: + - macosx64-sdk-14.2 + +android-custom-car: + description: "Custom chromium-as-release build for Android" + attributes: + cached_task: false + treeherder: + symbol: TL(custom-car-android) + worker: + docker-image: {in-tree: custom-car-android} + max-run-time: 20000 + use-system-python: false + run-on-projects: [] + run: + script: build-custom-car.sh + arguments: [ + car_android.tar.zst, + blink_symbol_level=0, + chrome_pgo_phase=2, + dcheck_always_on=false, + enable_nacl=false, + enable_resource_allowlist_generation=false, + is_chrome_branded=false, + is_clang=true, + is_component_build=false, + is_debug=false, + is_official_build=true, + symbol_level=0, + target_cpu="arm64", + target_os="android", + is_android=true, + is_high_end_android=true, + use_v8_context_snapshot=false, + v8_symbol_level=0 + ] + toolchain-artifact: public/build/car_android.tar.zst + +linux64-custom-car: + description: "Custom chromium-as-release build for Linux" + attributes: + cached_task: false + treeherder: + symbol: TL(custom-car) + worker: + docker-image: {in-tree: custom-car-linux} + max-run-time: 16200 + use-system-python: false + run-on-projects: [] + run: + script: build-custom-car.sh + arguments: [ + car_linux.tar.zst, + blink_symbol_level=0, + chrome_pgo_phase=2, + dcheck_always_on=false, + enable_nacl=false, + enable_resource_allowlist_generation=false, + is_chrome_branded=false, + is_clang=true, + is_component_build=false, + is_debug=false, + is_official_build=true, + symbol_level=0, + target_cpu="x64", + use_v8_context_snapshot=false, + v8_symbol_level=0 + ] + toolchain-artifact: public/build/car_linux.tar.zst + +win64-custom-car: + description: "Custom chromium-as-release build for Windows" + attributes: + cached_task: false + treeherder: + symbol: TW64(custom-car) + worker-type: b-win2022 + worker: + max-run-time: 16200 + env: + MSYSTEM: "MINGW64" + use-system-python: false + run-on-projects: [] + run: + script: build-custom-car.sh + arguments: [ + car_win64.tar.zst, + blink_symbol_level=0, + chrome_pgo_phase=2, + dcheck_always_on=false, + enable_nacl=false, + enable_resource_allowlist_generation=false, + is_chrome_branded=false, + is_clang=true, + is_component_build=false, + is_debug=false, + is_official_build=true, + symbol_level=0, + target_cpu=\"x64\", + use_v8_context_snapshot=false, + v8_symbol_level=0 + ] + toolchain-artifact: public/build/car_win64.tar.zst + fetches: + toolchain: + - win64-vs2022 + +browsertime: + description: "npm install browsertime node_modules" + treeherder: + symbol: TL(browsertime) + run: + script: browsertime.sh + sparse-profile: null + resources: + - 'tools/browsertime/package.json' + - 'tools/browsertime/package-lock.json' + - 'tools/browsertime/mach_commands.py' + toolchain-artifact: public/build/browsertime.tar.zst + fetches: + toolchain: + - linux64-node-16 + +wrench-deps: + description: "Downloads all the crates needed for building wrench" + treeherder: + symbol: WR(wrench-deps) + worker: + docker-image: {in-tree: webrender} + run: + script: wrench-deps-vendoring.sh + sparse-profile: null + resources: + - 'gfx/wr/**/Cargo.toml' + - 'gfx/wr/Cargo.lock' + - 'gfx/wr/ci-scripts/install-meson.sh' + toolchain-artifact: public/build/wrench-deps.tar.zst + fetches: + toolchain: + - linux64-rust # whatever m-c is built with + +linux64-wine: + description: "Wine build for Windows Cross Compile" + attributes: + local-toolchain: true + treeherder: + symbol: TL(wine) + tier: 1 + worker: + max-run-time: 10800 + run: + script: build-wine.sh + toolchain-artifact: public/build/wine.tar.zst + fetches: + fetch: + - wine + +linux64-winchecksec: + description: "winchecksec" + attributes: + local-toolchain: true + treeherder: + symbol: TL(winchecksec) + run: + script: build-winchecksec.sh + arguments: ['x86_64-unknown-linux-gnu'] + toolchain-artifact: public/build/winchecksec.tar.zst + tooltool-downloads: internal + fetches: + fetch: + - winchecksec + +win64-winchecksec: + description: "winchecksec" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(winchecksec) + run: + script: build-winchecksec.sh + arguments: ['x86_64-pc-windows-msvc'] + toolchain-artifact: public/build/winchecksec.tar.zst + tooltool-downloads: internal + fetches: + fetch: + - winchecksec + toolchain: + - linux64-clang-toolchain + - vs-toolchain + +linux64-makecab: + description: "makecab toolchain build" + treeherder: + symbol: TL(cab) + worker: + env: + FETCH: rust-makecab + run: + script: build-rust-based-toolchain.sh + arguments: ['x86_64-unknown-linux-gnu'] + toolchain-artifact: public/build/makecab.tar.zst + fetches: + fetch: + - makecab + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +win64-mozmake: + description: "mozmake for Windows" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(mozmake) + run: + script: build-mozmake.sh + toolchain-artifact: public/build/mozmake.tar.zst + fetches: + fetch: + - gnumake + toolchain: + - linux64-clang-toolchain + - vs-toolchain + +nsis: + description: "NSIS for Linux and Windows" + attributes: + local-toolchain: true + treeherder: + symbol: TL(nsis) + run: + script: build-nsis.sh + toolchain-artifact: public/build/nsis.tar.zst + fetches: + fetch: + - nsis-3.07 + - nsis-3.07-win + toolchain: + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu + +win64-vs2019: + description: "VS 2019" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(msvc2019) + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + run: + script: get_vs.py + arguments: + - build/vs/vs2019.yaml + resources: + - build/vs/vs2019.yaml + toolchain-artifact: project/gecko/vs/vs.tar.zst + toolchain-alias: + - vs + - vs-toolchain + +win64-vs2022: + description: "VS 2022" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(msvc2022) + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + run: + script: get_vs.py + arguments: + - build/vs/vs2022.yaml + resources: + - build/vs/vs2022.yaml + toolchain-artifact: project/gecko/vs/vs.tar.zst + +linux64-afl-instrumentation-4.0: + description: "AFL++ instrumentation toolchain build" + treeherder: + symbol: TL(afl++) + worker: + max-run-time: 3600 + run: + script: build-afl.sh + resources: + - taskcluster/scripts/misc/afl-nyx.patch + toolchain-artifact: public/build/afl-instrumentation.tar.zst + toolchain-alias: linux64-afl-instrumentation + fetches: + fetch: + - afl-plus-plus-4.0 + toolchain: + - linux64-clang + - linux64-toolchain-sysroot + +linux64-cargo-apk: + description: "cargo-apk toolchain build" + treeherder: + symbol: TL(cargo-apk) + worker: + env: + FETCH: android-ndk-rs/cargo-apk + run: + script: build-rust-based-toolchain.sh + arguments: ['x86_64-unknown-linux-gnu'] + resources: + - taskcluster/scripts/misc/cargo-apk-Cargo.lock + toolchain-artifact: public/build/cargo-apk.tar.zst + fetches: + fetch: + - android-ndk-rs + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +windows-rs: + # Bootstrap cannot use fetches directly, so we have this task to + # expose the corresponding fetch as a toolchain artifact. + description: Source of the windows rust crate + attributes: + local-toolchain: true + treeherder: + symbol: TW64(windows-rs) + run: + script: dummy.sh + toolchain-artifact: public/build/windows-rs.tar.zst + dependencies: + fetch-windows-rs: fetch-windows-rs + fetches: + fetch-windows-rs: + - artifact: windows-rs.tar.zst + extract: false + dest: ../artifacts diff --git a/taskcluster/ci/toolchain/nasm.yml b/taskcluster/ci/toolchain/nasm.yml new file mode 100644 index 0000000000..866aa5b0f6 --- /dev/null +++ b/taskcluster/ci/toolchain/nasm.yml @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + toolchain-artifact: public/build/nasm.tar.zst + +win64-nasm: + description: "nasm win64 build" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(nasm) + run: + script: build-nasm.sh + arguments: ['win64'] + fetches: + fetch: + - nasm-2.15.05 + toolchain: + - linux64-clang-mingw-x64 + +macosx64-nasm: + description: "nasm macosx64 build" + attributes: + local-toolchain: true + treeherder: + symbol: TM(nasm) + run: + script: build-nasm.sh + arguments: ['macosx64'] + fetches: + fetch: + - nasm-2.15.05 + toolchain: + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-nasm: + description: "nasm macosx64 build" + attributes: + local-toolchain: true + treeherder: + symbol: TM(nasm-aarch64) + run: + script: build-nasm.sh + arguments: ['macosx64-aarch64'] + fetches: + fetch: + - nasm-2.14.02 + toolchain: + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +linux64-nasm: + description: "nasm linux64 build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(nasm) + run: + script: build-nasm.sh + arguments: ['linux64'] + fetches: + fetch: + - nasm-2.15.05 + toolchain: + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu + +linux64-nasm-2.14.02: + description: "nasm linux64 minimum supported version build" + treeherder: + symbol: TL(nasm-2.14.02) + run: + script: build-nasm.sh + arguments: ['linux64'] + fetches: + fetch: + - nasm-2.14.02 + toolchain: + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu diff --git a/taskcluster/ci/toolchain/node.yml b/taskcluster/ci/toolchain/node.yml new file mode 100644 index 0000000000..c1bc86254d --- /dev/null +++ b/taskcluster/ci/toolchain/node.yml @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "Node repack toolchain build" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: repack-node.sh + toolchain-artifact: public/build/node.tar.zst + +linux64-node-12: + treeherder: + symbol: TL(node-12) + fetches: + fetch: + - nodejs-12-linux64 + +linux64-node-16: + treeherder: + symbol: TL(node-16) + run: + toolchain-alias: + - linux64-node + fetches: + fetch: + - nodejs-16-linux64 + +linux64-node-18: + # The upstream prebuilt binary requires glibc 2.28, which is too recent for Ubuntu 18.04, + # so we build our own that won't have that requirement. + description: "Node toolchain build" + treeherder: + symbol: TL(node-18) + run: + script: build-nodejs.sh + fetches: + fetch: + - nodejs-18-source + toolchain: + - linux64-gcc-11 + - linux64-toolchain-sysroot + +macosx64-node-16: + treeherder: + symbol: TM(node-16) + run: + toolchain-alias: + - macosx64-node + fetches: + fetch: + - nodejs-16-macosx64 + +macosx64-node-18: + treeherder: + symbol: TM(node-18) + fetches: + fetch: + - nodejs-18-macosx64 + +macosx64-aarch64-node-16: + treeherder: + symbol: TM(node-16-arm64) + run: + toolchain-alias: + - macosx64-aarch64-node + fetches: + fetch: + - nodejs-16-macosx64-arm64 + +macosx64-aarch64-node-18: + treeherder: + symbol: TM(node-18-arm64) + fetches: + fetch: + - nodejs-18-macosx64-arm64 + +win64-node-16: + treeherder: + symbol: TW64(node-16) + run: + toolchain-alias: + - win64-node + fetches: + fetch: + - nodejs-16-win64 + +win64-node-18: + treeherder: + symbol: TW64(node-18) + fetches: + fetch: + - nodejs-18-win64 + +win32-node-16: + treeherder: + symbol: TW32(node-16) + run: + toolchain-alias: + - win32-node + fetches: + fetch: + - nodejs-16-win32 + +win32-node-18: + treeherder: + symbol: TW32(node-18) + fetches: + fetch: + - nodejs-18-win32 diff --git a/taskcluster/ci/toolchain/pkgconf.yml b/taskcluster/ci/toolchain/pkgconf.yml new file mode 100644 index 0000000000..c97ead7acc --- /dev/null +++ b/taskcluster/ci/toolchain/pkgconf.yml @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + attributes: + local-toolchain: true + description: "pkgconf toolchain build" + worker: + max-run-time: 1800 + worker-type: b-linux-gcp + run: + script: build-pkgconf.sh + toolchain-artifact: public/build/pkgconf.tar.zst + fetches: + fetch: + - pkgconf + +linux64-pkgconf: + treeherder: + symbol: TL(pkgconf) + run: + arguments: ['x86_64-unknown-linux-gnu'] + fetches: + toolchain: + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-pkgconf: + treeherder: + symbol: TM(pkgconf) + run: + arguments: ['x86_64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-pkgconf: + treeherder: + symbol: TM(pkgconf-arm64) + run: + arguments: ['aarch64-apple-darwin'] + fetches: + toolchain: + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win64-pkgconf: + treeherder: + symbol: TW64(pkgconf) + run: + arguments: ['x86_64-pc-windows-gnu'] + fetches: + toolchain: + - linux64-clang-mingw-x64 diff --git a/taskcluster/ci/toolchain/python.yml b/taskcluster/ci/toolchain/python.yml new file mode 100644 index 0000000000..e610ee6e4e --- /dev/null +++ b/taskcluster/ci/toolchain/python.yml @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "python repack" + attributes: + local-toolchain: true + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run: + script: build-cpython.sh + resources: + - 'build/psutil_requirements.txt' + - 'build/zstandard_requirements.txt' + +linux64-python-3.8: + treeherder: + symbol: TL(python) + run: + toolchain-alias: linux64-python + toolchain-artifact: public/build/python.tar.zst + fetches: + fetch: + - cpython-3.8.10 + worker: + docker-image: {in-tree: ubuntu1804-build-python} + +win64-python-3.8: + description: "cpython 3.8.10" + treeherder: + symbol: TW64(python) + run: + script: pack-cpython.sh + toolchain-alias: win64-python + toolchain-artifact: public/build/python.tar.zst + fetches: + toolchain: + - linux64-wine + fetch: + - cpython-3.8.10.exe + +macosx64-python-3.8: + treeherder: + symbol: TM(python) + worker-type: b-osx-1015 + run: + toolchain-alias: macosx64-python + toolchain-artifact: public/build/python.tar.zst + fetches: + fetch: + - cpython-3.8.10 + toolchain: + - macosx64-clang-toolchain + - macosx64-sdk-toolchain + - macosx64-xz + +### + +linux64-python-3.11: + treeherder: + symbol: TL(python3.11) + run: + toolchain-artifact: public/build/python.tar.zst + fetches: + fetch: + - cpython-3.11.7 + worker: + docker-image: {in-tree: ubuntu1804-build-python} + +win64-python-3.11: + description: "cpython 3.11.7" + treeherder: + symbol: TW64(python3.11) + run: + script: pack-cpython.sh + toolchain-artifact: public/build/python.tar.zst + fetches: + toolchain: + - linux64-wine + fetch: + - cpython-3.11.7.exe + +macosx64-python-3.11: + treeherder: + symbol: TM(python3.11) + worker-type: b-osx-1015 + run: + toolchain-artifact: public/build/python.tar.zst + fetches: + fetch: + - cpython-3.11.7 + toolchain: + - macosx64-clang-toolchain + - macosx64-sdk-toolchain + - macosx64-xz diff --git a/taskcluster/ci/toolchain/resourcemonitor.yml b/taskcluster/ci/toolchain/resourcemonitor.yml new file mode 100644 index 0000000000..f4ef9a41c4 --- /dev/null +++ b/taskcluster/ci/toolchain/resourcemonitor.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + fetches: + fetch: + - go-1.14.4 + - resource-monitor + run: + script: build-resourcemonitor.sh + sparse-profile: null + toolchain-artifact: public/build/resource-monitor.tar.zst + +linux64-resource-monitor: + description: "linux64 resourcemonitor toolchain build" + treeherder: + symbol: TL(rs-monitor) + run: + arguments: ['linux64'] + +macosx64-resource-monitor: + description: "macos64 resourcemonitor toolchain build" + treeherder: + symbol: TM(rs-monitor) + run: + arguments: ['macos64'] + +windows32-resource-monitor: + description: "windows32 resourcemonitor toolchain build" + treeherder: + symbol: TW32(rs-monitor) + run: + arguments: ['windows32'] + +windows64-resource-monitor: + description: "windows64 resourcemonitor toolchain build" + treeherder: + symbol: TW64(rs-monitor) + run: + arguments: ['windows64'] diff --git a/taskcluster/ci/toolchain/rust-size.yml b/taskcluster/ci/toolchain/rust-size.yml new file mode 100644 index 0000000000..09963589a6 --- /dev/null +++ b/taskcluster/ci/toolchain/rust-size.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "rust-size toolchain build" + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + script: build-rust-based-toolchain.sh + fetches: + fetch: + - rust-size + +linux64-rust-size: + treeherder: + symbol: TL(rust-size) + run: + arguments: ['x86_64-unknown-linux-gnu'] + toolchain-artifact: public/build/rust-size.tar.zst + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-toolchain + - sysroot-x86_64-linux-gnu + +win64-rust-size: + treeherder: + symbol: TW64(rust-size) + run: + arguments: ['x86_64-pc-windows-msvc'] + toolchain-artifact: public/build/rust-size.tar.zst + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/rust.yml b/taskcluster/ci/toolchain/rust.yml new file mode 100644 index 0000000000..050e581247 --- /dev/null +++ b/taskcluster/ci/toolchain/rust.yml @@ -0,0 +1,219 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "rust repack" + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + run: + script: repack_rust.py + toolchain-artifact: public/build/rustc.tar.zst + +linux64-rust-1.70: + treeherder: + symbol: TL(rust-1.70) + run: + arguments: [ + '--channel', '1.70.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'i686-unknown-linux-gnu', + '--target', 'wasm32-wasi', + ] + toolchain-alias: + - linux64-rust-base + - linux64-rust-toolchain + +linux64-rust-1.76: + treeherder: + symbol: TL(rust) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'i686-unknown-linux-gnu', + '--target', 'aarch64-unknown-linux-gnu', + '--target', 'wasm32-wasi', + ] + toolchain-alias: linux64-rust + +# A patched rust toolchain that allows us to use sanitizers in our vendored +# build environment. See the rust fetch's comments for more details. +linux64-rust-dev: + description: "build rust from source" + worker-type: b-linux-large-gcp + treeherder: + symbol: TL(rust-dev) + worker: + env: + # Work around https://github.com/rust-lang/rust/issues/105967 + RUSTFLAGS_BOOTSTRAP: '-Clink-arg=-Wl,--undefined-version' + RUSTFLAGS_NOT_BOOTSTRAP: '-Clink-arg=-Wl,--undefined-version' + run: + arguments: [ + '--patch', 'rust-vendor-std.patch', + '--patch', 'src/tools/cargo:cargo-vendor-std.patch', + '--channel', 'dev', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + ] + fetches: + fetch: + - rust-1.76.0 + toolchain: + - linux64-clang-toolchain + - linux64-toolchain-sysroot + +linux64-rust-cross-1.76: + description: "rust repack with macos and windows cross support" + treeherder: + symbol: TL(rust-cross) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'i686-unknown-linux-gnu', + '--target', 'x86_64-apple-darwin', + '--target', 'aarch64-apple-darwin', + '--target', 'x86_64-pc-windows-msvc', + '--target', 'i686-pc-windows-msvc', + '--target', 'aarch64-pc-windows-msvc', + '--target', 'armv7-linux-androideabi', + '--target', 'thumbv7neon-linux-androideabi', + '--target', 'aarch64-linux-android', + '--target', 'i686-linux-android', + '--target', 'x86_64-linux-android', + ] + toolchain-alias: linux64-rust-cross + +linux64-rust-static-1.76: + description: "rust repack with static linking support" + treeherder: + symbol: TL(rust-static) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'aarch64-unknown-linux-musl', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-musl', + '--target', 'i686-unknown-linux-gnu', + '--target', 'i686-unknown-linux-musl', + ] + toolchain-alias: linux64-rust-static + +linux64-rust-macos-1.70: + description: "rust repack with macos-cross support" + treeherder: + symbol: TL(rust-macos-1.70) + run: + arguments: [ + '--channel', '1.70.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-apple-darwin', + '--target', 'aarch64-apple-darwin', + ] + toolchain-alias: linux64-rust-macos-toolchain + +linux64-rust-macos-1.76: + description: "rust repack with macos-cross support" + treeherder: + symbol: TL(rust-macos) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-apple-darwin', + '--target', 'aarch64-apple-darwin', + ] + toolchain-alias: linux64-rust-macos + +linux64-rust-android-1.76: + description: "rust repack with android-cross support" + treeherder: + symbol: TL(rust-android) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'armv7-linux-androideabi', + '--target', 'thumbv7neon-linux-androideabi', + '--target', 'aarch64-linux-android', + '--target', 'i686-linux-android', + '--target', 'x86_64-linux-android', + ] + toolchain-alias: linux64-rust-android + +linux64-rust-windows-1.70: + description: "rust repack with windows-cross support" + treeherder: + symbol: TL(rust-win-1.70) + run: + arguments: [ + '--channel', '1.70.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-pc-windows-msvc', + '--target', 'i686-pc-windows-msvc', + ] + toolchain-alias: linux64-rust-windows-toolchain + +linux64-rust-windows-1.76: + description: "rust repack with windows-cross support" + treeherder: + symbol: TL(rust-win) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-unknown-linux-gnu', + '--target', 'x86_64-pc-windows-msvc', + '--target', 'i686-pc-windows-msvc', + '--target', 'aarch64-pc-windows-msvc', + ] + toolchain-alias: linux64-rust-windows + +win64-rust-1.76: + treeherder: + symbol: TW64(rust) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-pc-windows-msvc', + '--target', 'x86_64-pc-windows-msvc', + '--target', 'i686-pc-windows-msvc', + '--target', 'aarch64-pc-windows-msvc', + ] + toolchain-alias: win64-rust + +macosx64-rust-1.76: + treeherder: + symbol: TM(rust) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-apple-darwin', + '--target', 'x86_64-apple-darwin', + ] + toolchain-alias: macosx64-rust + +mingw32-rust-1.76: + treeherder: + symbol: TMW(rust) + run: + arguments: [ + '--channel', '1.76.0', + '--host', 'x86_64-unknown-linux-gnu', + '--target', 'i686-unknown-linux-gnu', + '--target', 'i686-pc-windows-gnu', + '--target', 'x86_64-pc-windows-gnu', + '--target', 'x86_64-unknown-linux-gnu', + ] + toolchain-alias: mingw32-rust diff --git a/taskcluster/ci/toolchain/sccache.yml b/taskcluster/ci/toolchain/sccache.yml new file mode 100644 index 0000000000..6fa9ab46a5 --- /dev/null +++ b/taskcluster/ci/toolchain/sccache.yml @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + description: "sccache toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(sccache) + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + using: toolchain-script + script: build-rust-based-toolchain.sh + toolchain-artifact: public/build/sccache.tar.zst + fetches: + fetch: + - sccache + +linux64-sccache: + treeherder: + symbol: TL(sccache) + run: + arguments: + - x86_64-unknown-linux-gnu + - s3 + - gcs + - azure + - dist-server + - openssl/vendored + - native-zlib + fetches: + toolchain: + - linux64-rust-toolchain + - linux64-clang-toolchain + - sysroot-x86_64-linux-gnu + +macosx64-sccache: + treeherder: + symbol: TM(sccache) + run: + arguments: + - x86_64-apple-darwin + - all + - native-zlib + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +macosx64-aarch64-sccache: + treeherder: + symbol: TM(sccache-aarch64) + worker: + max-run-time: 1800 + run: + arguments: + - aarch64-apple-darwin + - all + - native-zlib + fetches: + toolchain: + - linux64-rust-macos-toolchain + - linux64-clang-toolchain + - macosx64-sdk-toolchain + +win64-sccache: + treeherder: + symbol: TW64(sccache) + worker: + env: + # Blake3 assembly can't be cross-compiled. The intrinsics version should be good enough. + CARGO_FEATURE_PREFER_INTRINSICS: '1' + run: + arguments: + - x86_64-pc-windows-msvc + - dist-client + - s3 + - gcs + - native-zlib + fetches: + toolchain: + - linux64-clang-toolchain + - linux64-rust-windows-toolchain + - vs-toolchain diff --git a/taskcluster/ci/toolchain/sysroot.yml b/taskcluster/ci/toolchain/sysroot.yml new file mode 100644 index 0000000000..e4479d16e0 --- /dev/null +++ b/taskcluster/ci/toolchain/sysroot.yml @@ -0,0 +1,206 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + run: + using: toolchain-script + script: build-sysroot.sh + toolchain-artifact: public/build/sysroot.tar.zst + +sysroot-i686-linux-gnu: + description: "Sysroot for linux32 builds" + attributes: + local-toolchain: true + worker: + env: + PACKAGES_TASKS: {task-reference: ''} + treeherder: + symbol: TL(sysroot32) + dependencies: + gcc-8: packages-deb8-32-gcc-8 + run: + toolchain-artifact: public/build/sysroot-i686-linux-gnu.tar.zst + arguments: + - i386 + +sysroot-x86_64-linux-gnu: + description: "Sysroot for linux64 builds" + attributes: + local-toolchain: true + worker: + env: + PACKAGES_TASKS: {task-reference: ''} + treeherder: + symbol: TL(sysroot64) + dependencies: + gcc-8: packages-deb8-gcc-8 + run: + toolchain-artifact: public/build/sysroot-x86_64-linux-gnu.tar.zst + arguments: + - amd64 + +sysroot-x86_64-linux-gnu-x11: + description: "Sysroot for linux64 builds (X11-only)" + attributes: + local-toolchain: true + worker: + env: + PACKAGES_TASKS: {task-reference: ' '} + treeherder: + symbol: TL(sysroot64-x11) + dependencies: + gcc-8: packages-deb8-gcc-8 + gtk3: packages-deb8-gtk3 + run: + toolchain-artifact: public/build/sysroot-x86_64-linux-gnu.tar.zst + arguments: + - amd64 + +sysroot-wasm32-wasi-clang-8.0: + description: "Sysroot for wasi" + attributes: + local-toolchain: true + treeherder: + symbol: TL(sysroot-wasi-8) + run: + script: build-sysroot-wasi.sh + toolchain-artifact: public/build/sysroot-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-8.0 + - wasi-sdk-11 + toolchain: + - linux64-clang-8.0 + - wasm32-wasi-compiler-rt-8.0 + +sysroot-wasm32-wasi-clang-17: + description: "Sysroot for wasi" + attributes: + local-toolchain: true + treeherder: + symbol: TL(sysroot-wasi-17) + run: + script: build-sysroot-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-alias: + by-project: + toolchains: null + default: sysroot-wasm32-wasi + toolchain-artifact: public/build/sysroot-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-17 + - wasi-sdk + toolchain: + - linux64-clang-17-stage1 + - wasm32-wasi-compiler-rt-17 + +sysroot-wasm32-wasi-clang-18: + description: "Sysroot for wasi" + attributes: + local-toolchain: true + treeherder: + symbol: TL(sysroot-wasi-18) + run: + script: build-sysroot-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-artifact: public/build/sysroot-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-18 + - wasi-sdk + toolchain: + - linux64-clang-18-stage1 + - wasm32-wasi-compiler-rt-18 + +sysroot-wasm32-wasi-clang-trunk: + description: "Sysroot for wasi" + attributes: + cached_task: false + treeherder: + symbol: TL(sysroot-wasi-trunk) + run: + script: build-sysroot-wasi.sh + arguments: + - wasi-sdk.patch + resources: + - taskcluster/scripts/misc/wasi-sdk.patch + toolchain-alias: + by-project: + toolchains: sysroot-wasm32-wasi + default: null + toolchain-artifact: public/build/sysroot-wasm32-wasi.tar.zst + fetches: + fetch: + - clang-trunk + - wasi-sdk + toolchain: + - linux64-clang-trunk-stage1 + - wasm32-wasi-compiler-rt-trunk + +sysroot-aarch64-linux-gnu: + description: "Sysroot for linux64 aarch64 builds" + attributes: + local-toolchain: true + treeherder: + symbol: TL(sysroot-aarch64) + run: + toolchain-artifact: public/build/sysroot-aarch64-linux-gnu.tar.zst + arguments: + - arm64 + +linux32-toolchain-sysroot: + description: "Sysroot for linux32 toolchain builds" + worker: + env: + PACKAGES_TASKS: {task-reference: ''} + treeherder: + symbol: TL(sysroot32-toolchain) + dependencies: + gcc-8: packages-deb8-32-gcc-8 + run: + arguments: + - i386 + # For clang + - binutils-dev + - libxml2-dev + # For minidump-stackwalk + - libcurl4-openssl-dev + - libssl-dev + - libbz2-dev + # For msix-package + - libicu-dev + +linux64-toolchain-sysroot: + description: "Sysroot for linux64 toolchain builds" + worker: + env: + PACKAGES_TASKS: {task-reference: ''} + treeherder: + symbol: TL(sysroot64-toolchain) + dependencies: + gcc-8: packages-deb8-gcc-8 + run: + arguments: + - amd64 + # For clang + - binutils-dev + - gcc-multilib + - lib32stdc++-8-dev + - libxml2-dev + # For minidump-stackwalk + - libcurl4-openssl-dev + - libssl-dev + - libbz2-dev + # For msix-package + - libicu-dev diff --git a/taskcluster/ci/trigger-comm-central/kind.yml b/taskcluster/ci/trigger-comm-central/kind.yml new file mode 100644 index 0000000000..0b6dfb2c64 --- /dev/null +++ b/taskcluster/ci/trigger-comm-central/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.trigger_comm_central:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + run-on-projects: ['mozilla-central-only'] + treeherder: + kind: other + platform: other/opt + tier: 2 + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: debian12-amd64-build} + max-run-time: 600 + taskcluster-proxy: true + run: + using: run-task + checkout: false + +jobs: + tb-rust: + description: 'Schedule job to verify the comm-central copy of third_party/rust' + treeherder: + symbol: tb-rust + run: + command: '/usr/bin/curl --fail --verbose -X POST -d "{}" -H "Content-Type: application/json" ${TASKCLUSTER_PROXY_URL}/api/hooks/v1/hooks/project-releng/cron-task-comm-central%2ftb-rust-vendor-check/trigger' + when: + files-changed: + - Cargo.lock + - Cargo.toml + - toolkit/library/rust/shared/Cargo.toml + scopes: + by-level: + '3': + - 'hooks:trigger-hook:project-releng/cron-task-comm-central/tb-rust-vendor-check' + default: [] diff --git a/taskcluster/ci/updatebot/kind.yml b/taskcluster/ci/updatebot/kind.yml new file mode 100644 index 0000000000..b62eccc5bc --- /dev/null +++ b/taskcluster/ci/updatebot/kind.yml @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + - toolchain + +transforms: + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + cron: + description: Cron Job for Updatebot + run-on-projects: [] + treeherder: + kind: other + platform: updatebot/all + symbol: cron + tier: 1 + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: updatebot} + max-run-time: 3600 + taskcluster-proxy: true + scopes: + - secrets:get:project/updatebot/{level}/try-sshkey + - secrets:get:project/updatebot/{level}/phabricator-token + - secrets:get:project/updatebot/{level}/bugzilla-api-key + - secrets:get:project/updatebot/{level}/database-password + - secrets:get:project/updatebot/{level}/sentry-url + - secrets:get:project/updatebot/{level}/sql-proxy-config + - hooks:trigger-hook:project-gecko/in-tree-action-1-generic/* + run: + using: run-task + command: /builds/worker/run.py /builds/worker/checkouts/gecko /builds/worker/updatebot + notifications: + subject: 'updatebot cron failed' + message: 'updatebot cron failed' + status-types: + - on-failed + - on-exception + emails: ["tom@mozilla.com", "jewilde@mozilla.com"] + fetches: + toolchain: + - linux64-rust + - linux64-cargo-vet + + cron-win: + description: Cron Job for Updatebot (Windows) + run-on-projects: [] + treeherder: + kind: other + platform: updatebot/all + symbol: w-cron + tier: 1 + worker-type: b-win2022 + worker: + max-run-time: 3600 + taskcluster-proxy: true + # begin _GENERATE_DEPOT_TOOLS_BINARIES_ + # artifacts: + # - type: directory + # name: public/depot_tools.zip + # path: build\src\obj-build\depot_tools + # end _GENERATE_DEPOT_TOOLS_BINARIES_ + env: + TOOLTOOL_MANIFEST: "taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest" + MSYSTEM: "MINGW64" + run: + using: run-task + cwd: '{checkout}' + command: ./taskcluster/docker/updatebot/windows-setup.sh + tooltool-downloads: internal + scopes: + - secrets:get:project/updatebot/{level}/try-sshkey + - secrets:get:project/updatebot/{level}/phabricator-token + - secrets:get:project/updatebot/{level}/bugzilla-api-key + - secrets:get:project/updatebot/{level}/database-password + - secrets:get:project/updatebot/{level}/sentry-url + - secrets:get:project/updatebot/{level}/sql-proxy-config + - hooks:trigger-hook:project-gecko/in-tree-action-1-generic/* + fetches: + fetch: + - cloud-sql-proxy-1.25.0 + - php-windows-7.4.24 + - arcanist + - depot_tools diff --git a/taskcluster/ci/updatebot/moz.build b/taskcluster/ci/updatebot/moz.build new file mode 100644 index 0000000000..315dc32600 --- /dev/null +++ b/taskcluster/ci/updatebot/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Developer Infrastructure", "Mach Vendor & Updatebot") diff --git a/taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest b/taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest new file mode 100644 index 0000000000..f284d80914 --- /dev/null +++ b/taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest @@ -0,0 +1,29 @@ +[ + { + "version": "vcruntime140.dll from Microsoft Visual C++ Redistributable for Visual Studio 2019 ver 14.29.30133.0", + "size": 97160, + "visibility": "internal", + "digest": "aa6afd6bea27f554e3646152d8c4f96f7bcaaa4933f8b7c04346e410f93f23cfa6d29362fd5d51ccbb8b6223e094cd89e351f072ad0517553703f5bf9de28778", + "algorithm": "sha512", + "filename": "vcruntime140.dll", + "unpack": false + }, + { + "version": "Visual Studio Community 2019, Windows SDK 10.0.19041.0, and a few system dlls Lobotmized for Angle Update", + "filename": "vs-sdk-and-windir-shim.zip", + "size": 463564917, + "algorithm": "sha512", + "digest": "20d93b0014ca44a666ca2b5cc570127f24378454889ff0a036df72213cd18140185d2c2e61df40bf1e4c0d9d2dc83a7796af9d2a13252de3c85faad068291945", + "visibility": "internal", + "unpack": true + }, + { + "version": "Windows binaries for depot_tools retrieved with depot_tools revision 7b3351443198f4e7d068999b169385b55a43679a on 2021-12-18", + "filename": "depot_tools-preloaded-binaries-7b3351443198f4e7d068999b169385b55a43679a-2021-12-18.zip", + "size": 315038144, + "algorithm": "sha512", + "digest": "250f3f3fcf631c53de1f505c53eb5ba782864fa92111eef0d710a1e7d0ad3da66db2fb8908d7c6d83e149674e976984d679f9a641eb9bea0b5c3d7b4f236920d", + "visibility": "internal", + "unpack": true + } +] diff --git a/taskcluster/ci/upload-generated-sources-dummy/kind.yml b/taskcluster/ci/upload-generated-sources-dummy/kind.yml new file mode 100644 index 0000000000..56e6762839 --- /dev/null +++ b/taskcluster/ci/upload-generated-sources-dummy/kind.yml @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.per_platform_dummy + - gecko_taskgraph.transforms.mac_dummy + - gecko_taskgraph.transforms.task + +only-for-build-platforms: + macosx64-shippable/opt + macosx64-devedition/opt + +kind-dependencies: + - build + - upload-generated-sources + +jobs: + upload-generated-sources-dummy: + description: Dummy task to pull in mac x64 and aarch64 upload-generated-symbols tasks + from-deps: + group-by: single-with-filters + worker-type: succeed diff --git a/taskcluster/ci/upload-generated-sources/kind.yml b/taskcluster/ci/upload-generated-sources/kind.yml new file mode 100644 index 0000000000..86123d336d --- /dev/null +++ b/taskcluster/ci/upload-generated-sources/kind.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.upload_generated_sources + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + +only-for-attributes: + - nightly + - shippable + - upload-generated-sources + +not-for-build-platforms: + - macosx64-shippable/opt + - macosx64-devedition/opt + +jobs: + upload-generated-sources: + description: Upload generated source files from build + from-deps: + group-by: single-with-filters + worker-type: t-linux-xlarge + treeherder: + symbol: Ugs + kind: build + worker: + docker-image: {in-tree: "lint"} + max-run-time: 600 + run: + using: mach + mach: {artifact-reference: "python --virtualenv upload-generated-sources build/upload_generated_sources.py "} + sparse-profile: upload-generated-sources + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload diff --git a/taskcluster/ci/upload-symbols-dummy/kind.yml b/taskcluster/ci/upload-symbols-dummy/kind.yml new file mode 100644 index 0000000000..d9202ef373 --- /dev/null +++ b/taskcluster/ci/upload-symbols-dummy/kind.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.per_platform_dummy + - gecko_taskgraph.transforms.mac_dummy + - gecko_taskgraph.transforms.task + +only-for-build-platforms: + macosx64-shippable/opt + macosx64-devedition/opt + +kind-dependencies: + - build + - upload-symbols + +jobs: + upload-symbols-dummy: + description: Dummy task to pull in mac x64 and aarch64 upload-symbols tasks + from-deps: + group-by: single-with-filters + unique-kinds: false + worker-type: succeed diff --git a/taskcluster/ci/upload-symbols/kind.yml b/taskcluster/ci/upload-symbols/kind.yml new file mode 100644 index 0000000000..0421835607 --- /dev/null +++ b/taskcluster/ci/upload-symbols/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.upload_symbols + - gecko_taskgraph.transforms.job + - gecko_taskgraph.transforms.task + +kind-dependencies: + - build + - toolchain + +not-for-build-platforms: + # Address Sanitizer builds don't have symbols to upload. + - linux64-asan-reporter-shippable/opt + - win64-asan-reporter-shippable/opt + - linux-geckodriver/opt + - linux64-geckodriver/opt + - linux64-aarch64-geckodriver/opt + - macosx64-geckodriver/opt + - macosx64-aarch64-geckodriver/opt + - win32-geckodriver/opt + - win64-geckodriver/opt + - win64-aarch64-geckodriver/opt + +jobs: + upload-symbols: + description: Upload Symbols + from-deps: + group-by: platform + worker-type: b-linux-gcp + use-system-python: false + worker: + docker-image: {in-tree: "debian12-base"} + max-run-time: 1800 + env: + # {level} gets replaced in the upload_symbols transform + SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload" + MAKECAB: /builds/worker/fetches/makecab/makecab + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + run: + using: mach + mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py "} + sparse-profile: upload-symbols + scopes: + - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload + fetches: + toolchain: + - linux64-makecab diff --git a/taskcluster/ci/valgrind/kind.yml b/taskcluster/ci/valgrind/kind.yml new file mode 100644 index 0000000000..754616e611 --- /dev/null +++ b/taskcluster/ci/valgrind/kind.yml @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + use-system-python: false + +jobs: + linux64-valgrind-qr/opt-swr: + description: "Linux64 Valgrind Opt WebRender" + index: + product: firefox + job-name: linux64-valgrind-qr/opt-swr + treeherder: + platform: linux64-qr/opt + symbol: V-swr + kind: build + tier: 1 + worker-type: t-linux-kvm + worker: + docker-image: {in-tree: valgrind-build} + max-run-time: 72000 + env: + NEED_XVFB: "true" + PERFHERDER_EXTRA_OPTIONS: valgrind + run: + using: mozharness + actions: [get-secrets, build, valgrind-test] + custom-build-variant-cfg: valgrind + config: + - builds/releng_base_firefox.py + - builds/releng_base_linux_64_builds.py + extra-config: + extra_prefs: [gfx.webrender.software=true] + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + use-sccache: true + fetches: + toolchain: + - linux64-clang + - linux64-rust + - linux64-cbindgen + - linux64-dump_syms + - linux64-sccache + - linux64-nasm + - linux64-node + - linux64-pkgconf + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + optimization: + skip-unless-expanded: null diff --git a/taskcluster/ci/webrender/kind.yml b/taskcluster/ci/webrender/kind.yml new file mode 100644 index 0000000000..84a938cf67 --- /dev/null +++ b/taskcluster/ci/webrender/kind.yml @@ -0,0 +1,634 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + attributes: + retrigger: true + run-on-projects: ['integration', 'release'] + treeherder: + tier: 1 + kind: other + worker: + max-run-time: 3600 + env: + RUST_BACKTRACE: 'full' + RUSTFLAGS: '--deny warnings' + run: + sparse-profile: webrender + use-system-python: false + +jobs: + lint-tidy: + description: Runs linting checks on WebRender code + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: webrender} + run: + using: run-task + cwd: '{checkout}/gfx/wr' + command: >- + pushd /tmp && + python3 -m pip install six && + python3 -m pip install -e 'git+https://github.com/servo/servo.git@65a4d1646da46c37fe748add6dcf24b62ebb602a#egg=servo_tidy&subdirectory=python/tidy' && + popd && + servo-tidy + treeherder: + platform: linux64-qr/opt + symbol: WR(tidy) + when: + files-changed: + - 'gfx/wr/**' + + linux64-release: + description: Runs release-mode WebRender CI tests on a Linux worker + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: webrender} + fetches: + toolchain: + - linux64-rust + - wrench-deps + run: + using: run-task + command: >- + export PATH=$PATH:$MOZ_FETCHES_DIR/rustc/bin:${MOZ_FETCHES_DIR}/wrench-deps/meson && + cd $HOME/checkouts/gecko/gfx/wr && + mv $MOZ_FETCHES_DIR/wrench-deps/{vendor,.cargo} ./ && + CARGOFLAGS="--verbose --frozen" ci-scripts/linux-release-tests.sh + treeherder: + platform: linux64-qr/opt + symbol: WR(wrench) + kind: other + when: + files-changed: + - 'gfx/wr/**' + + linux64-debug: + description: Runs debug-mode WebRender CI tests on a Linux worker + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: webrender} + fetches: + toolchain: + - linux64-rust + - wrench-deps + run: + using: run-task + command: >- + export PATH=$PATH:$MOZ_FETCHES_DIR/rustc/bin:${MOZ_FETCHES_DIR}/wrench-deps/meson && + cd $HOME/checkouts/gecko/gfx/wr && + mv $MOZ_FETCHES_DIR/wrench-deps/{vendor,.cargo} ./ && + CARGOFLAGS="--verbose --frozen" ci-scripts/linux-debug-tests.sh + treeherder: + platform: linux64-qr/debug + symbol: WR(wrench) + kind: other + when: + files-changed: + - 'gfx/wr/**' + + wrench-macos-build: + description: Cross compilation of wrench for macOS on Linux + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: webrender} + chain-of-trust: true + artifacts: + - type: file + name: public/build/wrench-macos.tar.bz2 + path: /builds/worker/artifacts/wrench-macos.tar.bz2 + - type: file + name: public/build/wrench-macos-headless.tar.bz2 + path: /builds/worker/artifacts/wrench-macos-headless.tar.bz2 + dependencies: + macosx64-clang-14: toolchain-macosx64-clang-14-raw + fetches: + toolchain: + - linux64-rust-macos + - linux64-clang-14 + - macosx64-sdk + - wrench-deps + macosx64-clang-14: + - artifact: clang.tar.zst + dest: clang-mac + run: + using: run-task + command: '$HOME/checkouts/gecko/taskcluster/scripts/misc/wrench-macos-build.sh' + treeherder: + platform: osx-cross/opt + symbol: WR(B) + when: + files-changed: + - 'gfx/wr/**' + - 'taskcluster/scripts/misc/wr-macos-cross-build-setup.sh' + - 'taskcluster/scripts/misc/wrench-macos-build.sh' + + macos-release: + description: Runs release-mode WebRender CI tests on a macOS worker + worker-type: t-osx-1015-r8 + worker: + # repo cloning has highly variable run time + max-run-time: 7200 + env: + MOZ_FETCHES_DIR: 'checkouts/gecko/gfx/wr' + dependencies: + webrender-wrench-macos-build: webrender-wrench-macos-build + fetches: + webrender-wrench-macos-build: + - 'wrench-macos.tar.bz2' + - 'wrench-macos-headless.tar.bz2' + run: + using: run-task + cwd: '{checkout}/gfx/wr' + command: >- + export WRENCH_HEADLESS_TARGET=$PWD/wrench-macos-headless/ && + export WRENCH_BINARY=$PWD/wrench-macos/bin/wrench && + ci-scripts/macos-release-tests.sh + treeherder: + platform: macosx1015-64-qr/opt + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + + cargotest-macos-build: + description: Cross compilation of cargo tests for macOS on Linux + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: webrender} + chain-of-trust: true + artifacts: + - type: file + name: public/build/cargo-test-binaries.tar.bz2 + path: /builds/worker/artifacts/cargo-test-binaries.tar.bz2 + fetches: + toolchain: + - linux64-rust-macos + - linux64-clang + - macosx64-sdk + - wrench-deps + run: + using: run-task + command: '$HOME/checkouts/gecko/taskcluster/scripts/misc/wr-cargotest-macos-build.sh' + treeherder: + platform: osx-cross/debug + symbol: WR(B) + when: + files-changed: + - 'gfx/wr/**' + - 'taskcluster/scripts/misc/wr-cargotest-macos-build.sh' + - 'taskcluster/scripts/misc/wr-macos-cross-build-setup.sh' + + macos-debug: + description: Runs debug-mode WebRender CI tests on a macOS worker + worker-type: t-osx-1015-r8 + worker: + max-run-time: 3600 + env: + MOZ_FETCHES_DIR: 'checkouts/gecko/gfx/wr' + dependencies: + webrender-cargotest-macos-build: webrender-cargotest-macos-build + fetches: + webrender-cargotest-macos-build: + - 'cargo-test-binaries.tar.bz2' + run: + using: run-task + cwd: '{checkout}/gfx/wr' + command: >- + mv cargo-test-binaries target && + cd target && + for i in debug/deps/*; do $i; done + treeherder: + platform: macosx1015-64-qr/debug + symbol: WR(cargotest) + when: + files-changed: + - 'gfx/wr/**' + + windows: + description: Runs WebRender CI tests on a Windows worker + worker-type: win11-64-2009-source + worker: + max-run-time: 5400 + env: + NODE_OPTIONS: '--max_old_space_size=4096' + fetches: + toolchain: + - win64-rust + - vs + - wrench-deps + run: + using: run-task + use-caches: false + command: '$GECKO_PATH/taskcluster/scripts/misc/wrench-windows-tests.sh' + treeherder: + platform: windows11-64-2009-qr/release + symbol: WR(wrench) + when: + files-changed: + - 'build/win64/mozconfig.vs2019' + - 'gfx/wr/**' + - 'taskcluster/scripts/misc/wrench-windows-tests.sh' + + wrench-android-debug: + description: Debug build of wrench for Android + attributes: + build_platform: android + worker-type: b-linux-gcp + worker: + max-run-time: 5400 + docker-image: {in-tree: webrender} + artifacts: + - type: file + name: public/build/wrench-debug.apk + path: /builds/worker/checkouts/gecko/gfx/wr/target/debug/apk/wrench.apk + - type: file + name: public/build/reftests.tar.gz + path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.gz + - type: file + name: public/build/mozdevice.tar.gz + path: /builds/worker/checkouts/gecko/testing/mozbase/mozdevice.tar.gz + - type: file + name: public/build/six.tar.gz + path: /builds/worker/checkouts/gecko/third_party/python/six.tar.gz + run: + using: run-task + cwd: '{checkout}/gfx/wr/wrench' + command: >- + $GECKO_PATH/taskcluster/scripts/misc/wrench-android-build.sh debug && + tar czf reftests.tar.gz reftests/ && + cd $GECKO_PATH/testing/mozbase && + tar czf mozdevice.tar.gz mozdevice/ && + cd $GECKO_PATH/third_party/python && + tar czf six.tar.gz six/ + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-cargo-apk + - linux64-jdk + - linux64-rust-android + - wrench-deps + treeherder: + platform: android-em-7-0-x86/debug + symbol: WR(B) + when: + files-changed: + - 'build.gradle' + - 'gfx/wr/**' + - 'taskcluster/scripts/misc/wrench-android-build.sh' + + wrench-android-release: + description: Release build of wrench for Android + attributes: + build_platform: android + worker-type: b-linux-gcp + worker: + max-run-time: 5400 + docker-image: {in-tree: webrender} + artifacts: + - type: file + name: public/build/wrench-release.apk + path: /builds/worker/checkouts/gecko/gfx/wr/target/release/apk/wrench.apk + - type: file + name: public/build/reftests.tar.gz + path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.gz + - type: file + name: public/build/mozdevice.tar.gz + path: /builds/worker/checkouts/gecko/testing/mozbase/mozdevice.tar.gz + - type: file + name: public/build/six.tar.gz + path: /builds/worker/checkouts/gecko/third_party/python/six.tar.gz + run: + using: run-task + cwd: '{checkout}/gfx/wr/wrench' + command: >- + $GECKO_PATH/taskcluster/scripts/misc/wrench-android-build.sh release && + tar czf reftests.tar.gz reftests/ && + cd $GECKO_PATH/testing/mozbase && + tar czf mozdevice.tar.gz mozdevice/ && + cd $GECKO_PATH/third_party/python && + tar czf six.tar.gz six/ + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-cargo-apk + - linux64-jdk + - linux64-rust-android + - wrench-deps + treeherder: + platform: android-em-7-0-x86/opt + symbol: WR(B) + when: + files-changed: + - 'build.gradle' + - 'gfx/wr/**' + - 'taskcluster/scripts/misc/wrench-android-build.sh' + + android-emulator-debug: + description: Run debug wrench reftests on Android emulator + attributes: + build_platform: android + worker-type: t-linux-kvm # privileged:true doesn't work on b-linux + worker: + max-run-time: 5400 + docker-image: {in-tree: ubuntu1804-test} + privileged: true # for access to /dev/kvm for hw accel in the emulator + artifacts: + - type: directory + name: public/build/logs + path: /builds/worker/workspace/build/logs + env: + # need hard-coded URLs until bug 1551875 is fixed + TASKCLUSTER_ROOT_URL: 'https://taskcluster.net' + TASKCLUSTER_PROXY_URL: 'http://taskcluster' + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + NEED_WINDOW_MANAGER: 'true' + PYTHON: "/builds/worker/checkouts/gecko/mach python" + dependencies: + webrender-wrench-android-debug: webrender-wrench-android-debug + fetches: + webrender-wrench-android-debug: + - artifact: 'wrench-debug.apk' + extract: false + toolchain: + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-jdk + run: + using: run-task + tooltool-downloads: internal + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-debug.apk && + export WRENCH_REFTESTS=$GECKO_PATH/gfx/wr/wrench/reftests && + export MOZHARNESS_PATH=$GECKO_PATH/testing/mozharness && + $GECKO_PATH/taskcluster/scripts/tester/test-linux.sh + treeherder: + platform: android-em-7-0-x86/debug + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + + android-emulator-release: + description: Run release wrench reftests on Android emulator + attributes: + build_platform: android + worker-type: t-linux-kvm # privileged:true doesn't work on b-linux + worker: + max-run-time: 5400 + docker-image: {in-tree: ubuntu1804-test} + privileged: true # for access to /dev/kvm for hw accel in the emulator + artifacts: + - type: directory + name: public/build/logs + path: /builds/worker/workspace/build/logs + env: + # need hard-coded URLs until bug 1551875 is fixed + TASKCLUSTER_ROOT_URL: 'https://taskcluster.net' + TASKCLUSTER_PROXY_URL: 'http://taskcluster' + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + NEED_WINDOW_MANAGER: 'true' + PYTHON: "/builds/worker/checkouts/gecko/mach python" + dependencies: + webrender-wrench-android-release: webrender-wrench-android-release + fetches: + webrender-wrench-android-release: + - artifact: 'wrench-release.apk' + extract: false + toolchain: + - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux + - linux64-jdk + run: + using: run-task + tooltool-downloads: internal + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-release.apk && + export WRENCH_REFTESTS=$GECKO_PATH/gfx/wr/wrench/reftests && + export MOZHARNESS_PATH=$GECKO_PATH/testing/mozharness && + $GECKO_PATH/taskcluster/scripts/tester/test-linux.sh + treeherder: + platform: android-em-7-0-x86/opt + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + + android-hw-p5-debug: + description: Run debug wrench reftests on Android Pixel5 + attributes: + build_platform: android + worker-type: t-bitbar-gw-unit-p5 + worker: + max-run-time: 5400 + artifacts: + - type: directory + name: public/build/logs + path: build/logs + env: + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + MOZ_FETCHES_DIR: fetches + PYTHONPATH: "fetches/mozdevice:fetches/six" + dependencies: + webrender-wrench-android-debug: webrender-wrench-android-debug + android-build: build-android-arm/debug + fetches: + webrender-wrench-android-debug: + - artifact: 'wrench-debug.apk' + extract: false + - 'reftests.tar.gz' + - 'mozdevice.tar.gz' + - 'six.tar.gz' + android-build: + - artifact: 'mozharness.zip' + dest: 'mozharness' + run: + using: run-task + checkout: false + run-as-root: true + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-debug.apk && + export WRENCH_REFTESTS=$MOZ_FETCHES_DIR/reftests && + export MOZHARNESS_PATH=$MOZ_FETCHES_DIR/mozharness && + python3 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} --config-file ${MOZHARNESS_PATH}/configs/${MOZHARNESS_CONFIG} + treeherder: + platform: android-hw-p5-13-0-arm7/debug + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + - 'testing/mozbase/mozdevice/**' + + android-hw-p5-opt: + description: Run opt wrench reftests on Android Pixel5 + attributes: + build_platform: android + worker-type: t-bitbar-gw-unit-p5 + worker: + max-run-time: 5400 + artifacts: + - type: directory + name: public/build/logs + path: build/logs + env: + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + MOZ_FETCHES_DIR: fetches + PYTHONPATH: "fetches/mozdevice:fetches/six" + dependencies: + webrender-wrench-android-release: webrender-wrench-android-release + android-build: build-android-arm/opt + fetches: + webrender-wrench-android-release: + - artifact: 'wrench-release.apk' + extract: false + - 'reftests.tar.gz' + - 'mozdevice.tar.gz' + - 'six.tar.gz' + android-build: + - artifact: 'mozharness.zip' + dest: 'mozharness' + run: + using: run-task + checkout: false + run-as-root: true + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-release.apk && + export WRENCH_REFTESTS=$MOZ_FETCHES_DIR/reftests && + export MOZHARNESS_PATH=$MOZ_FETCHES_DIR/mozharness && + python3 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} --config-file ${MOZHARNESS_PATH}/configs/${MOZHARNESS_CONFIG} + treeherder: + platform: android-hw-p5-13-0-arm7/opt + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + - 'testing/mozbase/mozdevice/**' + + android-hw-a51-debug: + description: Run debug wrench reftests on Android Samsung Galaxy A51 + attributes: + build_platform: android + worker-type: t-bitbar-gw-perf-a51 + worker: + max-run-time: 5400 + artifacts: + - type: directory + name: public/build/logs + path: build/logs + env: + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + MOZ_FETCHES_DIR: fetches + PYTHONPATH: "fetches/mozdevice:fetches/six" + dependencies: + webrender-wrench-android-debug: webrender-wrench-android-debug + android-build: build-android-arm/debug + fetches: + webrender-wrench-android-debug: + - artifact: 'wrench-debug.apk' + extract: false + - 'reftests.tar.gz' + - 'mozdevice.tar.gz' + - 'six.tar.gz' + android-build: + - artifact: 'mozharness.zip' + dest: 'mozharness' + run: + using: run-task + checkout: false + run-as-root: true + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-debug.apk && + export WRENCH_REFTESTS=$MOZ_FETCHES_DIR/reftests && + export MOZHARNESS_PATH=$MOZ_FETCHES_DIR/mozharness && + python3 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} --config-file ${MOZHARNESS_PATH}/configs/${MOZHARNESS_CONFIG} + treeherder: + platform: android-hw-a51-11-0-arm7/debug + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + - 'testing/mozbase/mozdevice/**' + + android-hw-a51-opt: + description: Run opt wrench reftests on Android Samsung Galaxy A51 + attributes: + build_platform: android + worker-type: t-bitbar-gw-perf-a51 + worker: + max-run-time: 5400 + artifacts: + - type: directory + name: public/build/logs + path: build/logs + env: + MOZHARNESS_SCRIPT: android_wrench.py + MOZHARNESS_CONFIG: android/wrench.py + MOZ_FETCHES_DIR: fetches + PYTHONPATH: "fetches/mozdevice:fetches/six" + dependencies: + webrender-wrench-android-release: webrender-wrench-android-release + android-build: build-android-arm/opt + fetches: + webrender-wrench-android-release: + - artifact: 'wrench-release.apk' + extract: false + - 'reftests.tar.gz' + - 'mozdevice.tar.gz' + - 'six.tar.gz' + android-build: + - artifact: 'mozharness.zip' + dest: 'mozharness' + run: + using: run-task + checkout: false + run-as-root: true + command: >- + export WRENCH_APK=$MOZ_FETCHES_DIR/wrench-release.apk && + export WRENCH_REFTESTS=$MOZ_FETCHES_DIR/reftests && + export MOZHARNESS_PATH=$MOZ_FETCHES_DIR/mozharness && + python3 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} --config-file ${MOZHARNESS_PATH}/configs/${MOZHARNESS_CONFIG} + treeherder: + platform: android-hw-a51-11-0-arm7/opt + symbol: WR(wrench) + when: + files-changed: + - 'gfx/wr/**' + - 'testing/mozharness/scripts/android_*.py' + - 'testing/mozharness/configs/android/*.py' + - 'testing/mozbase/mozdevice/**' diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY new file mode 100644 index 0000000000..cb1e1bb482 --- /dev/null +++ b/taskcluster/docker/REGISTRY @@ -0,0 +1 @@ +taskcluster diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile new file mode 100644 index 0000000000..22eb3c51c8 --- /dev/null +++ b/taskcluster/docker/android-build/Dockerfile @@ -0,0 +1,45 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Nick Alexander + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +# rsync is required for l10n single locale repacks. less, screen, and +# vim, help debugging interactive tasks in Task Cluster. +# git and openssh-client are used to upload GeckoView javadoc to Github. +RUN apt-get update && \ + apt-get install \ + autoconf2.13 \ + build-essential \ + base-files \ + ccache \ + cmake \ + curl \ + file \ + gnupg \ + jq \ + less \ + openssh-client \ + procps \ + python3-dev \ + rsync \ + screen \ + sudo \ + tar \ + unzip \ + uuid \ + valgrind \ + vim \ + x11-utils \ + xvfb \ + wget \ + zip \ + zstd + +# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb. +# %include taskcluster/docker/recipes/xvfb.sh +COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh + +# Back to the workdir, matching desktop-build. +WORKDIR /builds/worker diff --git a/taskcluster/docker/android-build/README.md b/taskcluster/docker/android-build/README.md new file mode 100644 index 0000000000..6096b08368 --- /dev/null +++ b/taskcluster/docker/android-build/README.md @@ -0,0 +1,2 @@ +This is a docker script for fetching Android Gradle dependenices for +use in Mozilla's build clusters. diff --git a/taskcluster/docker/android-build/VERSION b/taskcluster/docker/android-build/VERSION new file mode 100644 index 0000000000..4e379d2bfe --- /dev/null +++ b/taskcluster/docker/android-build/VERSION @@ -0,0 +1 @@ +0.0.2 diff --git a/taskcluster/docker/condprof/Dockerfile b/taskcluster/docker/condprof/Dockerfile new file mode 100644 index 0000000000..76817e7927 --- /dev/null +++ b/taskcluster/docker/condprof/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gregory Mierzwinski + +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/workspace + +RUN apt-get -y update && \ + apt-get install -y python3.7 \ + python3.7-venv \ + python3.7-dev + +CMD ["/bin/bash", "--login"] + diff --git a/taskcluster/docker/custom-car-android/Dockerfile b/taskcluster/docker/custom-car-android/Dockerfile new file mode 100644 index 0000000000..4886012804 --- /dev/null +++ b/taskcluster/docker/custom-car-android/Dockerfile @@ -0,0 +1,25 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER kshampur + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + libbz2-dev \ + libglib2.0-dev \ + libtinfo5 \ + pkg-config \ + tar \ + unzip \ + wget \ + zip \ + lsb-release \ + gperf \ + procps \ + gcc diff --git a/taskcluster/docker/custom-car-linux/Dockerfile b/taskcluster/docker/custom-car-linux/Dockerfile new file mode 100644 index 0000000000..17f24b9042 --- /dev/null +++ b/taskcluster/docker/custom-car-linux/Dockerfile @@ -0,0 +1,24 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER kshampur + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + libbz2-dev \ + libglib2.0-dev \ + libtinfo5 \ + pkg-config \ + tar \ + unzip \ + wget \ + zip \ + lsb-release \ + gperf \ + procps diff --git a/taskcluster/docker/custom-v8/Dockerfile b/taskcluster/docker/custom-v8/Dockerfile new file mode 100644 index 0000000000..810c2eb090 --- /dev/null +++ b/taskcluster/docker/custom-v8/Dockerfile @@ -0,0 +1,22 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Joel Maher + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + libbz2-dev \ + libglib2.0-dev \ + libtinfo5 \ + pkg-config \ + tar \ + unzip \ + wget \ + zip \ + procps diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile new file mode 100644 index 0000000000..2a50b97a29 --- /dev/null +++ b/taskcluster/docker/debian-base/Dockerfile @@ -0,0 +1,66 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +### Add worker user and setup its workspace. +RUN mkdir /builds && \ + groupadd -g 1000 worker && \ + useradd -u 1000 -g 1000 -d /builds/worker -s /bin/bash -m worker && \ + mkdir -p /builds/worker/workspace && \ + chown -R worker:worker /builds + +# Declare default working folder +WORKDIR /builds/worker + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV HOME=/builds/worker \ + SHELL=/bin/bash \ + USER=worker \ + LOGNAME=worker \ + HOSTNAME=taskcluster-worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + git \ + less \ + make \ + mercurial \ + patch \ + python3 \ + python3-minimal \ + python3-zstandard \ + python3-psutil \ + python3-venv \ + vim-tiny \ + xz-utils \ + zstd \ + && /usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES + +# %include testing/mozharness/external_tools/robustcheckout.py +COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + +# Add pip configuration, among other things. +# %include taskcluster/docker/recipes/dot-config +COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config + +# %include taskcluster/scripts/run-task +COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task + +# %include taskcluster/scripts/misc/fetch-content +ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* diff --git a/taskcluster/docker/debian-build/Dockerfile b/taskcluster/docker/debian-build/Dockerfile new file mode 100644 index 0000000000..12f72df130 --- /dev/null +++ b/taskcluster/docker/debian-build/Dockerfile @@ -0,0 +1,53 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +# binutils is needed for base toolchain builds. Ideally, it wouldn't be needed. +# libc6-i386 and lib32gcc1 are needed for wine. +# libgtk-3-0 are needed to run xpcshell during the build. +# lib32atomic1, lib32stdc++6 and lib32z1 are needed to run some 32-bits +# cpio is needed by xar (mac repackaging on linux) +# spidermonkey tests. libasound2 is needed to run xpcshell after we introduced +# the dependencies on alsa via Web MIDI. +RUN apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + bash \ + binutils \ + bzip2 \ + cpio \ + curl \ + file \ + findutils \ + gawk \ + gnupg \ + gzip \ + jq \ + lib32atomic1 \ + 'lib32gcc(1|-s1)$' \ + lib32stdc++6 \ + lib32z1 \ + libasound2 \ + libc6-i386 \ + libgtk-3-0 \ + libucl1 \ + libxml2 \ + m4 \ + make \ + p7zip-full \ + perl \ + procps \ + python3-dev \ + rsync \ + screen \ + tar \ + unzip \ + uuid \ + wget \ + x11-utils \ + zip diff --git a/taskcluster/docker/debian-packages/Dockerfile b/taskcluster/docker/debian-packages/Dockerfile new file mode 100644 index 0000000000..1e651a48b7 --- /dev/null +++ b/taskcluster/docker/debian-packages/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +RUN apt-get install --install-recommends \ + apt-utils \ + aptitude \ + build-essential \ + devscripts \ + equivs \ + fakeroot \ + git diff --git a/taskcluster/docker/debian-raw/Dockerfile b/taskcluster/docker/debian-raw/Dockerfile new file mode 100644 index 0000000000..7dce7ccf21 --- /dev/null +++ b/taskcluster/docker/debian-raw/Dockerfile @@ -0,0 +1,70 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +MAINTAINER Mike Hommey + +ENV DEBIAN_FRONTEND=noninteractive + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + +# %include taskcluster/docker/recipes/setup_packages.sh +COPY topsrcdir/taskcluster/docker/recipes/setup_packages.sh /usr/local/sbin/ +# %include taskcluster/docker/recipes/clean_packages.sh +COPY topsrcdir/taskcluster/docker/recipes/clean_packages.sh /usr/local/sbin/ + +COPY taskcluster-hack.sh /usr/local/sbin +COPY snapshot-hack.py /usr/local/sbin +COPY gpgvnoexpkeysig /usr/local/sbin + +ARG DIST +ARG SNAPSHOT +ARG TASKCLUSTER_ROOT_URL +# Set apt sources list to a snapshot. +# Note: the use of gpgvnoexpkeysig is because the Debian Jessie GPG key expired. +RUN if [ -n "$DIST" ]; then for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST-security debian-debug_$DIST-debug debian-debug_$DIST-proposed-updates-debug debian-debug_$DIST-backports-debug; do \ + case "$s" in \ + debian-debug_jessie*|debian_jessie-updates) \ + : No debian-debug/updates archive for Jessie; \ + ;; \ + debian-security_jessie-security) \ + echo "deb http://archive.debian.org/debian-security/ jessie/updates main"; \ + ;; \ + debian*_jessie*) \ + echo "deb http://archive.debian.org/${s%_*}/ ${s#*_} main"; \ + ;; \ + debian-security_buster-security) \ + d=${s#*_}; \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${d%-security}/updates main"; \ + ;; \ + *) \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \ + ;; \ + esac; \ + done > /etc/apt/sources.list ; fi && \ + rm -f /etc/apt/sources.list.d/* && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + if dpkg --compare-versions $(apt --version | awk '{print $2}') ge 2.1.15; then \ + echo 'dir::bin::methods::https "/usr/local/sbin/taskcluster-hack.sh";'; \ + fi; \ + if [ "$DIST" = "jessie" ]; then \ + echo 'Dir::Bin::gpg "/usr/local/sbin/gpgvnoexpkeysig";'; \ + fi; \ + ) > /etc/apt/apt.conf.d/99taskcluster && \ + ( echo 'Package: *'; \ + echo 'Pin: origin "'$TASKCLUSTER_ROOT_URL'"' | sed 's,https://,,'; \ + echo 'Pin-Priority: 1001'; \ + ) > /etc/apt/preferences.d/99taskcluster + +RUN apt-get update && \ + if grep -q snapshot.debian.org /etc/apt/sources.list; then \ + apt-get install python3-minimal libpython3-stdlib; \ + echo 'dir::bin::methods::http "/usr/local/sbin/snapshot-hack.py";' >> /etc/apt/apt.conf.d/99taskcluster; \ + fi && \ + apt-get dist-upgrade && \ + apt-get install \ + apt-transport-https \ + ca-certificates diff --git a/taskcluster/docker/debian-raw/gpgvnoexpkeysig b/taskcluster/docker/debian-raw/gpgvnoexpkeysig new file mode 100755 index 0000000000..fbbfd7a658 --- /dev/null +++ b/taskcluster/docker/debian-raw/gpgvnoexpkeysig @@ -0,0 +1,52 @@ +#!/bin/sh +# +# Downloaded from https://gitlab.mister-muffin.de/josch/mmdebstrap/raw/branch/main/gpgvnoexpkeysig +# +# This script is in the public domain +# +# Author: Johannes Schauer Marin Rodrigues +# +# This is a wrapper around gpgv as invoked by apt. It turns EXPKEYSIG results +# from gpgv into GOODSIG results. This is necessary for apt to access very old +# timestamps from snapshot.debian.org for which the GPG key is already expired: +# +# Get:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease [242 kB] +# Err:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease +# The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) +# Reading package lists... +# W: GPG error: http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease: The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) +# E: The repository 'http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease' is not signed. +# +# To use this script, call apt with +# +# -o Apt::Key::gpgvcommand=/usr/libexec/mmdebstrap/gpgvnoexpkeysig +# +# Scripts doing similar things can be found here: +# +# * debuerreotype as /usr/share/debuerreotype/scripts/.gpgv-ignore-expiration.sh +# * derivative census: salsa.d.o/deriv-team/census/-/blob/master/bin/fakegpgv + +set -eu + +find_gpgv_status_fd() { + while [ "$#" -gt 0 ]; do + if [ "$1" = '--status-fd' ]; then + echo "$2" + return 0 + fi + shift + done + # default fd is stdout + echo 1 +} +GPGSTATUSFD="$(find_gpgv_status_fd "$@")" + +case $GPGSTATUSFD in + ''|*[!0-9]*) + echo "invalid --status-fd argument" >&2 + exit 1 + ;; +esac + +# we need eval because we cannot redirect a variable fd +eval 'exec gpgv "$@" '"$GPGSTATUSFD"'>&1 | sed "s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /" >&'"$GPGSTATUSFD" diff --git a/taskcluster/docker/debian-raw/snapshot-hack.py b/taskcluster/docker/debian-raw/snapshot-hack.py new file mode 100755 index 0000000000..6e880f0a74 --- /dev/null +++ b/taskcluster/docker/debian-raw/snapshot-hack.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import subprocess +import sys +import threading +import urllib.request +from urllib.parse import urlparse, urlunparse + +# This script interposes between APT and its HTTP method. APT sends queries on +# stdin, and expect responses on stdout. We intercept those and change the +# snapshot.debian.org URLs it requests on the fly, if the equivalent URLs +# exist on deb.debian.org. + +URI_HEADER = "URI: " + + +def url_exists(url): + try: + req = urllib.request.Request(url, method="HEAD") + response = urllib.request.urlopen(req) + return response.getcode() == 200 + except Exception: + return False + + +def write_and_flush(fh, data): + fh.write(data) + fh.flush() + + +def output_handler(proc, url_mapping, lock): + for line in proc.stdout: + if line.startswith(URI_HEADER): + url = line[len(URI_HEADER) :].rstrip() + # APT expects back the original url it requested. + with lock: + original_url = url_mapping.get(url, None) + if original_url: + write_and_flush(sys.stdout, line.replace(url, original_url)) + continue + write_and_flush(sys.stdout, line) + + +def main(): + proc = subprocess.Popen( + ["/usr/lib/apt/methods/http"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + url_mapping = {} + lock = threading.Lock() + output_thread = threading.Thread( + target=output_handler, args=(proc, url_mapping, lock), daemon=True + ) + output_thread.start() + + while True: + try: + line = sys.stdin.readline() + except KeyboardInterrupt: + # When apt cuts the connection, we receive a KeyboardInterrupt. + break + if not line: + break + + if line.startswith(URI_HEADER): + url = line[len(URI_HEADER) :].rstrip() + url_parts = urlparse(url) + # For .deb packages, if we can find the file on deb.debian.org, take it + # from there instead of snapshot.debian.org, because deb.debian.org will + # be much faster. Hopefully, most files will be available on deb.debian.org. + if url_parts.hostname == "snapshot.debian.org" and url_parts.path.endswith( + ".deb" + ): + # The url is assumed to be of the form + # http://snapshot.debian.org/archive/section/yymmddThhmmssZ/... + path_parts = url_parts.path.split("/") + # urlparse().path always starts with a / so path_parts is + # expected to look like ["", "archive", "section", "yymmddThhmmssZ", ...] + # we want to remove "archive" and "yymmddThhmmssZ" to create an url + # on deb.debian.org. + path_parts.pop(3) + path_parts.pop(1) + modified_url = urlunparse( + url_parts._replace( + netloc="deb.debian.org", path="/".join(path_parts) + ) + ) + if url_exists(modified_url): + with lock: + url_mapping[modified_url] = url + write_and_flush(proc.stdin, line.replace(url, modified_url)) + continue + write_and_flush(proc.stdin, line) + + proc.stdin.close() + output_thread.join() + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/debian-raw/taskcluster-hack.sh b/taskcluster/docker/debian-raw/taskcluster-hack.sh new file mode 100755 index 0000000000..eecac021ec --- /dev/null +++ b/taskcluster/docker/debian-raw/taskcluster-hack.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# APT version 2.1.15 and newer changed how they handle quoting in redirections +# in a way that breaks the setup for APT repos in taskcluster artifacts +# (unfortunately, there's also no setup on the taskcluster end that would work +# with both old and newer versions of APT, short of removing redirections +# entirely). +/usr/lib/apt/methods/https | sed -u '/^New-URI:/s/+/%2b/g' diff --git a/taskcluster/docker/debian-repackage/Dockerfile b/taskcluster/docker/debian-repackage/Dockerfile new file mode 100644 index 0000000000..ae257931d4 --- /dev/null +++ b/taskcluster/docker/debian-repackage/Dockerfile @@ -0,0 +1,35 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mozilla Releng + +VOLUME /builds/worker/workspace + +# At the moment the Firefox build baseline is jessie. +# Things in the build/CI system that run in the task's container +# (mach, run-task, etc.) do not support jessie. +# There's also no jessie `base` images or packages generated in the CI. +# To generate Firefox Debian packages compatible with the baseline +# we bootstrap basic i386 and amd64 jessie systems on bullseye. +# We use these to generate shared library dependencies in jessie using chroot. +# python/mozbuild/mozbuild/repackaging/deb.py checks for jessie systems +# bootstrapped under /srv/jessie-i386 and /srv/jessie-amd64 +# If they aren't there, deb.py runs the repackage in the working directory without using chroot. +# To keep the build and repackage enviroments consistent the Debian baseline used +# here (jessie) should be synchronized with the baseline used in +# taskcluster/scripts/misc/build-sysroot.sh +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ + apt-get install debootstrap && set -xe && \ + for arch in i386 amd64; do \ + debootstrap \ + --arch=$arch \ + --variant=buildd \ + --include=debhelper,libasound2,libgtk-3-0,libx11-xcb1,libxtst6 \ + --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg \ + --verbose \ + jessie \ + /srv/jessie-$arch \ + https://archive.debian.org/debian; \ + done && \ + /usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES diff --git a/taskcluster/docker/decision/Dockerfile b/taskcluster/docker/decision/Dockerfile new file mode 100644 index 0000000000..f2aa83903e --- /dev/null +++ b/taskcluster/docker/decision/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Andrew Halberstadt + +RUN mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +ADD system-setup.sh /tmp/system-setup.sh +RUN bash /tmp/system-setup.sh + +ADD comm-task-env /builds/worker/bin/comm-task-env + +ENV PATH=/builds/worker/bin:$PATH + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/decision/HASH b/taskcluster/docker/decision/HASH new file mode 100644 index 0000000000..11ff3dc377 --- /dev/null +++ b/taskcluster/docker/decision/HASH @@ -0,0 +1 @@ +sha256:ea73900129328095da4fb033697690d538a08d935298fef4d5fa4aae80273bce diff --git a/taskcluster/docker/decision/README.md b/taskcluster/docker/decision/README.md new file mode 100644 index 0000000000..4490880be8 --- /dev/null +++ b/taskcluster/docker/decision/README.md @@ -0,0 +1,5 @@ +# Decision Tasks + +The decision image is a "boostrapping" image for the in tree logic it +deals with cloning gecko and the related utilities for providing an +environment where we can run gecko. diff --git a/taskcluster/docker/decision/VERSION b/taskcluster/docker/decision/VERSION new file mode 100644 index 0000000000..ee74734aa2 --- /dev/null +++ b/taskcluster/docker/decision/VERSION @@ -0,0 +1 @@ +4.1.0 diff --git a/taskcluster/docker/decision/comm-task-env b/taskcluster/docker/decision/comm-task-env new file mode 100755 index 0000000000..65481497ae --- /dev/null +++ b/taskcluster/docker/decision/comm-task-env @@ -0,0 +1,199 @@ +#!/usr/bin/python3 -u +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird build environment prep for run-task, +for use with comm-central derived repositories. + +This script is meant to run prior to run-task on repositories like +comm-central that need to check out a copy of a mozilla repository +in order to build. +See bug 1491371 for background on why this is necessary. + +A project will have a file named ".gecko_rev.yml" in it's root. See the +constant "GECKO_REV_CONF" if you want to change that. To download it, the +script uses the project repository URL and the revision number. +Those are defined in the environment variables: +COMM_HEAD_REPOSITORY +COMM_HEAD_REV + +.gecko_rev.yml has a structure like (for comm-central): +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central +GECKO_HEAD_REF: default +``` +or for branches: +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta +GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH +GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910 + +Note about GECKO_HEAD_REV and GECKO_HEAD_REF: +GECKO_HEAD_REF is a branch name or "default". +GECKO_HEAD_REV is a revision hash. +``` +""" + +import sys + +import os +import socket +import time +from datetime import datetime +from pprint import pformat + +import urllib.error +import urllib.request + +import yaml + +if sys.version_info[0:2] < (3, 5): + print('run-task-wrapper requires Python 3.5+') + sys.exit(1) + +GECKO_REV_CONF = ".gecko_rev.yml" +DEBUG = bool(os.environ.get("RTW_DEBUG", False)) + + +def print_message(msg, prefix=__file__, level=""): + """ + Print messages. + :param object msg: message to print, usually a string, but not always + :param str prefix: message prefix + :param str level: message level (DEBUG, ERROR, INFO) + """ + if not isinstance(msg, str): + msg = pformat(msg) + now = datetime.utcnow().isoformat() + # slice microseconds to 3 decimals. + now = now[:-3] if now[-7:-6] == '.' else now + if level: + sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format( + prefix=prefix, now=now, level=level, msg=msg)) + else: + sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format( + prefix=prefix, now=now, msg=msg)) + sys.stdout.flush() + + +def error_exit(msg): + """Print the error message and exit with error.""" + print_message(msg, level="ERROR") + if DEBUG: + raise Exception(msg) + + sys.exit(1) + + +def print_debug(msg): + """Prints a message with DEBUG prefix if DEBUG is enabled + with the environment variable "RTW_DEBUG". + """ + if DEBUG: + print_message(msg, level="DEBUG") + + +def check_environ(): + """Check that the necessary environment variables to find the + comm- repository are defined. (Set in .taskcluster.yml) + :return: tuple(str, str) + """ + print_debug("Checking environment variables...") + project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None) + project_head_rev = os.environ.get("COMM_HEAD_REV", None) + + if project_head_repo is None or project_head_rev is None: + error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format( + project_head_repo, project_head_rev) + + print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format( + project_head_repo, project_head_rev)) + return project_head_repo, project_head_rev + + +def download_url(url, retry=1): + """Downloads the given URL. Naively retries (when asked) upon failure + :param url: str + :param retry: int + :return: str + """ + # Use 1-based counting for display and calculation purposes. + for i in range(1, retry+1): + try: + print_message('Fetching {}. Attempt {} of {}.'.format( + url, i, retry)) + with urllib.request.urlopen(url, timeout=10) as response: + data = response.read().decode("utf-8") + return data + except (urllib.error.URLError, socket.timeout) as exc: + print_message('Unable to retrieve {}'.format(url)) + if isinstance(exc, urllib.error.URLError): + print_message(exc.reason) + else: # socket.timeout + print_message('Connection timed out.') + + if i < retry: # No more retries + wait_time = i * 5 # fail #1: sleep 5s. #2, sleep 10s + print_message('Retrying in {} seconds.'.format(wait_time)) + time.sleep(wait_time) + + error_exit('No more retry attempts! Aborting.') + + +def fetch_gecko_conf(project_head_repo, project_revision): + """Downloads .gecko_rev.yml from the project repository + :param project_head_repo: str + :param project_revision: str + :return: dict + """ + gecko_conf_url = '/'.join( + [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF]) + + gecko_conf_yml = download_url(gecko_conf_url, retry=5) + + try: + gecko_conf = yaml.safe_load(gecko_conf_yml) + return gecko_conf + except yaml.YAMLError as exc: + err_txt = ["Error processing Gecko YAML configuration."] + if hasattr(exc, "problem_mark"): + mark = exc.problem_mark # pylint: disable=no-member + err_txt.append("Error position: line {}, column {}".format( + mark.line + 1, mark.column + 1)) + error_exit('\n'.join(err_txt)) + + +def update_environment(gecko_conf): + """Adds the new variables defined in gecko_conf to the + running environment. + :param gecko_conf: dict + """ + print_message("Updating environment with:") + print_message(gecko_conf) + os.environ.update(gecko_conf) + + print_debug("New environment:") + print_debug(os.environ) + + +def exec_run_task(args): + """Executes run-task with a modified environment.""" + print_message("Executing: {}".format(pformat(args))) + os.execv(args[0], args[0:]) + + +def main(): + """Main function.""" + args = sys.argv[1:] # Remaining args starting with run-task + + project_head_repo, project_revision = check_environ() + gecko_conf = fetch_gecko_conf(project_head_repo, project_revision) + update_environment(gecko_conf) + exec_run_task(args) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/decision/system-setup.sh b/taskcluster/docker/decision/system-setup.sh new file mode 100644 index 0000000000..0f9d04fa50 --- /dev/null +++ b/taskcluster/docker/decision/system-setup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -v -e + +test "$(whoami)" == 'root' + +apt-get update +apt-get install \ + python \ + sudo \ + python3-yaml + +rm -rf /var/lib/apt/lists/ +rm "$0" diff --git a/taskcluster/docker/diffoscope/Dockerfile b/taskcluster/docker/diffoscope/Dockerfile new file mode 100644 index 0000000000..1ac4addbd8 --- /dev/null +++ b/taskcluster/docker/diffoscope/Dockerfile @@ -0,0 +1,34 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV LANG=en_US.UTF-8 + +RUN apt-get install \ + binutils-multiarch \ + bzip2 \ + curl \ + enjarify \ + diffoscope \ + jsbeautifier \ + libc++abi1 \ + locales \ + default-jdk-headless \ + python3-progressbar \ + unzip \ + xxd \ + zip \ + && \ + sed -i '/en_US.UTF-8/s/^# *//' /etc/locale.gen && \ + locale-gen + +COPY get_and_diffoscope /builds/worker/bin/get_and_diffoscope +COPY readelf /builds/worker/bin/readelf +COPY report_error /builds/worker/bin/report_error +COPY test_diffoscope /builds/worker/bin/test_diffoscope + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* && \ + /builds/worker/bin/test_diffoscope diff --git a/taskcluster/docker/diffoscope/get_and_diffoscope b/taskcluster/docker/diffoscope/get_and_diffoscope new file mode 100644 index 0000000000..8f72970f77 --- /dev/null +++ b/taskcluster/docker/diffoscope/get_and_diffoscope @@ -0,0 +1,109 @@ +#!/bin/bash + +set -e +set -x + +cd /builds/worker + +mkdir a b + +# /builds/worker/bin contains wrapper binaries to divert what diffoscope +# needs to use, so it needs to appear first. +export PATH=/builds/worker/bin:$PATH + +# Until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879010 is +# implemented, it's better to first manually extract the data. +# Plus dmg files are not supported yet. + +RESOURCE_DIR=firefox +case "$ORIG_URL" in +*.zip|*.apk) + curl -L "$ORIG_URL" > a.zip + curl -L "$NEW_URL" > b.zip + unzip -d a a.zip + unzip -d b b.zip + ;; +*.tar.zst) + curl -L "$ORIG_URL" | tar -C a --zstd -xf - + curl -L "$NEW_URL" | tar -C b --zstd -xf - + ;; +*.tar.bz2) + curl -L "$ORIG_URL" | tar -C a -jxf - + curl -L "$NEW_URL" | tar -C b -jxf - + ;; +*.tar.gz) + curl -L "$ORIG_URL" | tar -C a -zxf - + curl -L "$NEW_URL" | tar -C b -zxf - + ;; +*.dmg) + for tool in lipo otool; do + ln -s $MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin*-$tool bin/$tool + done + curl -L "$ORIG_URL" > a.dmg + curl -L "$NEW_URL" > b.dmg + for i in a b; do + $MOZ_FETCHES_DIR/dmg/dmg extract $i.dmg $i.hfs + $MOZ_FETCHES_DIR/dmg/hfsplus $i.hfs extractall / $i + done + RESOURCE_DIR=$(cd b; echo */Contents/Resources) + ;; +*) + ARTIFACT=$(basename "${ORIG_URL}") + curl -L "$ORIG_URL" > "a/${ARTIFACT}" + curl -L "$NEW_URL" > "b/${ARTIFACT}" +esac + +case "$ORIG_URL" in +*/target.apk) + OMNIJAR=assets/omni.ja + ;; +*) + OMNIJAR=omni.ja + ;; +esac + +# Builds are 99% of the time differing in some small ways, so it's not +# really useful to report a failure (at least not until we actually +# care about the builds being 100% identical). +POST=true + +fail() { + exit 1 +} + +for option; do + case "$option" in + --unpack) + CURDIR=$PWD + for dir in a b; do + # Need to run mach python from inside the gecko source. + # See bug #1533642. + (cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR "$CURDIR/$dir/$RESOURCE_DIR") + done + ;; + --fail) + POST="fail" + ;; + *) + echo "Unsupported option: $option" >&2 + exit 1 + esac +done + +if [ -n "$PRE_DIFF" ]; then + eval $PRE_DIFF +fi + +if eval diffoscope \ + --html diff.html \ + --text diff.txt \ + --progress \ + $DIFFOSCOPE_ARGS \ + a b +then + # Ok + : +else + $(dirname $0)/report_error diff + $POST +fi diff --git a/taskcluster/docker/diffoscope/readelf b/taskcluster/docker/diffoscope/readelf new file mode 100644 index 0000000000..6b864171d7 --- /dev/null +++ b/taskcluster/docker/diffoscope/readelf @@ -0,0 +1,13 @@ +#!/bin/sh + +case "$1 $2" in +"--wide --symbols") + # When called with --wide --symbols, we remove the first column (which + # is essentially a line number that is not very useful), and then sort, + # which will order symbols by address, making a diff more useful. + /usr/bin/readelf "$@" | awk -F: '{print $2}' | sort + ;; +*) + exec /usr/bin/readelf "$@" + ;; +esac diff --git a/taskcluster/docker/diffoscope/report_error b/taskcluster/docker/diffoscope/report_error new file mode 100755 index 0000000000..9074e33cae --- /dev/null +++ b/taskcluster/docker/diffoscope/report_error @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import sys +stem = sys.argv[1] + +# We "parse" the diff output, so we look at the lines that contain a "tee", like: +# ├── +++ b/firefox +# │ ├── +++ b/firefox/libxul.so +# │ │ ├── readelf --wide --notes {} +# We ignore lines like the last one, to only report file names. And we ignore +# lines for directories such as the first one. +TEE = "├── " +VERTICAL_LINE = "│" +paths = set() +with open(f"{stem}.txt") as fh: + for l in fh: + l = l.rstrip() + before, tee, after = l.partition(TEE) + if not tee: + continue + before = before.split() + assert all(x == VERTICAL_LINE for x in before) + depth = len(before) + _, plus, after = after.partition("+++ ") + if not plus: + continue + _, b, full_path = after.partition("b/") + assert b == "b/" + parent_path = "/".join(full_path.split("/")[:-1]) + if parent_path in paths: + paths.remove(parent_path) + if full_path: + paths.add(full_path) + +for p in sorted(paths): + print(f"TEST-UNEXPECTED-FAIL | {p} differs. See the {stem}.html or {stem}.txt artifact") diff --git a/taskcluster/docker/diffoscope/test_diffoscope b/taskcluster/docker/diffoscope/test_diffoscope new file mode 100644 index 0000000000..ab1c6c1eee --- /dev/null +++ b/taskcluster/docker/diffoscope/test_diffoscope @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +WORKDIR=$(mktemp -d) +cd $WORKDIR + +mkdir -p a/foo/bar/bar +mkdir -p a/foo/bar/baz +mkdir -p b/foo/bar/bar +mkdir -p b/foo/bar/baz + +# A file that is modified +echo qux > a/foo/bar/qux +echo quz > b/foo/bar/qux + +# A binary file that is modified +cp $(which ls) a/foo/bin +cp $(which cat) b/foo/bin + +# A file that is removed +echo hoge > a/foo/bar/bar/hoge + +# A file that is created +echo fuga > b/foo/bar/baz/fuga + +# Also add a zip file with the same contents +(cd a/foo; zip -r bar.zip bar) +(cd b/foo; zip -r bar.zip bar) + +if TERM=linux diffoscope --no-progress --text diff.txt a b; then + echo "diffoscope didn't find differences?" + exit 1 +fi + +cat > expected.txt < + +int close_range(unsigned int first, unsigned int last, unsigned int flags) { + errno = ENOSYS; + return -1; +} diff --git a/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py b/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..b1eb745d7f --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +""" +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +""" + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/taskcluster/docker/firefox-flatpak/firefox-symbolic.svg b/taskcluster/docker/firefox-flatpak/firefox-symbolic.svg new file mode 100644 index 0000000000..1858ced70e --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/firefox-symbolic.svg @@ -0,0 +1,6 @@ + + + + diff --git a/taskcluster/docker/firefox-flatpak/launch-script.sh b/taskcluster/docker/firefox-flatpak/launch-script.sh new file mode 100644 index 0000000000..e41bf66f33 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/launch-script.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export TMPDIR=$XDG_CACHE_HOME/tmp +exec /app/lib/firefox/firefox --name org.mozilla.firefox "$@" diff --git a/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in new file mode 100644 index 0000000000..a3edce20c6 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in @@ -0,0 +1,45 @@ + + + org.mozilla.firefox + org.mozilla.firefox.desktop + Firefox + Mozilla + Fast, Private & Safe Web Browser + CC0-1.0 + MPL-2.0 + +

When it comes to your life online, you have a choice: accept the factory settings or put your privacy first. When you choose Firefox as your default browser, you’re choosing to protect your data while supporting an independent tech company. Firefox is also the only major browser backed by a non-profit fighting to give you more openness, transparency and control of your life online. Join hundreds of millions of people who choose to protect what's important by choosing Firefox - a web browser designed to be fast, easy to use, customizable and private.

+ + + + + + mozilla + internet + web + + + https://www.mozilla.org/firefox/ + https://donate.mozilla.org/ + https://bugzilla.mozilla.org/ + https://support.mozilla.org/ + https://wiki.mozilla.org/L10n:Starting_a_localization + + + + https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image1.png + The “New Tab” page + + + https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image2.png + A Wikipedia article displayed in Firefox + + + https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image3.png + The “Welcome to Firefox” page: “Open up an amazing internet” + + + + https://hg.mozilla.org/mozilla-central/file/tip/taskcluster/docker/firefox-flatpak/runme.sh + + diff --git a/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop new file mode 100644 index 0000000000..21c3e7aec0 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop @@ -0,0 +1,386 @@ +[Desktop Entry] +Version=1.0 +Name=Firefox Web Browser +Name[ar]=متصفح الويب فَيَرفُكْس +Name[ast]=Restolador web Firefox +Name[bn]=ফায়ারফক্স ওয়েব ব্রাউজার +Name[ca]=Navegador web Firefox +Name[cs]=Firefox Webový prohlížeč +Name[da]=Firefox - internetbrowser +Name[el]=Περιηγητής Firefox +Name[es]=Navegador web Firefox +Name[et]=Firefoxi veebibrauser +Name[fa]=مرورگر اینترنتی Firefox +Name[fi]=Firefox-selain +Name[fr]=Navigateur Web Firefox +Name[gl]=Navegador web Firefox +Name[he]=דפדפן האינטרנט Firefox +Name[hr]=Firefox web preglednik +Name[hu]=Firefox webböngésző +Name[it]=Firefox Browser Web +Name[ja]=Firefox ウェブ・ブラウザ +Name[ko]=Firefox 웹 브라우저 +Name[ku]=Geroka torê Firefox +Name[lt]=Firefox interneto naršyklė +Name[nb]=Firefox Nettleser +Name[nl]=Firefox webbrowser +Name[nn]=Firefox Nettlesar +Name[no]=Firefox Nettleser +Name[pl]=Przeglądarka WWW Firefox +Name[pt]=Firefox Navegador Web +Name[pt_BR]=Navegador Web Firefox +Name[ro]=Firefox – Navigator Internet +Name[ru]=Веб-браузер Firefox +Name[sk]=Firefox - internetový prehliadač +Name[sl]=Firefox spletni brskalnik +Name[sv]=Firefox webbläsare +Name[tr]=Firefox Web Tarayıcısı +Name[ug]=Firefox توركۆرگۈ +Name[uk]=Веб-браузер Firefox +Name[vi]=Trình duyệt web Firefox +Name[zh_CN]=Firefox 网络浏览器 +Name[zh_TW]=Firefox 網路瀏覽器 +Comment=Browse the World Wide Web +Comment[ar]=تصفح الشبكة العنكبوتية العالمية +Comment[ast]=Restola pela Rede +Comment[bn]=ইন্টারনেট ব্রাউজ করুন +Comment[ca]=Navegueu per la web +Comment[cs]=Prohlížení stránek World Wide Webu +Comment[da]=Surf på internettet +Comment[de]=Im Internet surfen +Comment[el]=Μπορείτε να περιηγηθείτε στο διαδίκτυο (Web) +Comment[es]=Navegue por la web +Comment[et]=Lehitse veebi +Comment[fa]=صفحات شبکه جهانی اینترنت را مرور نمایید +Comment[fi]=Selaa Internetin WWW-sivuja +Comment[fr]=Naviguer sur le Web +Comment[gl]=Navegar pola rede +Comment[he]=גלישה ברחבי האינטרנט +Comment[hr]=Pretražite web +Comment[hu]=A világháló böngészése +Comment[it]=Esplora il web +Comment[ja]=ウェブを閲覧します +Comment[ko]=웹을 돌아 다닙니다 +Comment[ku]=Li torê bigere +Comment[lt]=Naršykite internete +Comment[nb]=Surf på nettet +Comment[nl]=Verken het internet +Comment[nn]=Surf på nettet +Comment[no]=Surf på nettet +Comment[pl]=Przeglądanie stron WWW +Comment[pt]=Navegue na Internet +Comment[pt_BR]=Navegue na Internet +Comment[ro]=Navigați pe Internet +Comment[ru]=Доступ в Интернет +Comment[sk]=Prehliadanie internetu +Comment[sl]=Brskajte po spletu +Comment[sv]=Surfa på webben +Comment[tr]=İnternet'te Gezinin +Comment[ug]=دۇنيادىكى توربەتلەرنى كۆرگىلى بولىدۇ +Comment[uk]=Перегляд сторінок Інтернету +Comment[vi]=Để duyệt các trang web +Comment[zh_CN]=浏览互联网 +Comment[zh_TW]=瀏覽網際網路 +GenericName=Web Browser +GenericName[ar]=متصفح ويب +GenericName[ast]=Restolador Web +GenericName[bn]=ওয়েব ব্রাউজার +GenericName[ca]=Navegador web +GenericName[cs]=Webový prohlížeč +GenericName[da]=Webbrowser +GenericName[el]=Περιηγητής διαδικτύου +GenericName[es]=Navegador web +GenericName[et]=Veebibrauser +GenericName[fa]=مرورگر اینترنتی +GenericName[fi]=WWW-selain +GenericName[fr]=Navigateur Web +GenericName[gl]=Navegador Web +GenericName[he]=דפדפן אינטרנט +GenericName[hr]=Web preglednik +GenericName[hu]=Webböngésző +GenericName[it]=Browser web +GenericName[ja]=ウェブ・ブラウザ +GenericName[ko]=웹 브라우저 +GenericName[ku]=Geroka torê +GenericName[lt]=Interneto naršyklė +GenericName[nb]=Nettleser +GenericName[nl]=Webbrowser +GenericName[nn]=Nettlesar +GenericName[no]=Nettleser +GenericName[pl]=Przeglądarka WWW +GenericName[pt]=Navegador Web +GenericName[pt_BR]=Navegador Web +GenericName[ro]=Navigator Internet +GenericName[ru]=Веб-браузер +GenericName[sk]=Internetový prehliadač +GenericName[sl]=Spletni brskalnik +GenericName[sv]=Webbläsare +GenericName[tr]=Web Tarayıcı +GenericName[ug]=توركۆرگۈ +GenericName[uk]=Веб-браузер +GenericName[vi]=Trình duyệt Web +GenericName[zh_CN]=网络浏览器 +GenericName[zh_TW]=網路瀏覽器 +Keywords=Internet;WWW;Browser;Web;Explorer +Keywords[ar]=انترنت;إنترنت;متصفح;ويب;وب +Keywords[ast]=Internet;WWW;Restolador;Web;Esplorador +Keywords[ca]=Internet;WWW;Navegador;Web;Explorador;Explorer +Keywords[cs]=Internet;WWW;Prohlížeč;Web;Explorer +Keywords[da]=Internet;Internettet;WWW;Browser;Browse;Web;Surf;Nettet +Keywords[de]=Internet;WWW;Browser;Web;Explorer;Webseite;Site;surfen;online;browsen +Keywords[el]=Internet;WWW;Browser;Web;Explorer;Διαδίκτυο;Περιηγητής;Firefox;Φιρεφοχ;Ιντερνετ +Keywords[es]=Explorador;Internet;WWW +Keywords[fi]=Internet;WWW;Browser;Web;Explorer;selain;Internet-selain;internetselain;verkkoselain;netti;surffaa +Keywords[fr]=Internet;WWW;Browser;Web;Explorer;Fureteur;Surfer;Navigateur +Keywords[he]=דפדפן;אינטרנט;רשת;אתרים;אתר;פיירפוקס;מוזילה; +Keywords[hr]=Internet;WWW;preglednik;Web +Keywords[hu]=Internet;WWW;Böngésző;Web;Háló;Net;Explorer +Keywords[it]=Internet;WWW;Browser;Web;Navigatore +Keywords[is]=Internet;WWW;Vafri;Vefur;Netvafri;Flakk +Keywords[ja]=Internet;WWW;Web;インターネット;ブラウザ;ウェブ;エクスプローラ +Keywords[nb]=Internett;WWW;Nettleser;Explorer;Web;Browser;Nettside +Keywords[nl]=Internet;WWW;Browser;Web;Explorer;Verkenner;Website;Surfen;Online +Keywords[pt]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[pt_BR]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[ru]=Internet;WWW;Browser;Web;Explorer;интернет;браузер;веб;файрфокс;огнелис +Keywords[sk]=Internet;WWW;Prehliadač;Web;Explorer +Keywords[sl]=Internet;WWW;Browser;Web;Explorer;Brskalnik;Splet +Keywords[tr]=İnternet;WWW;Tarayıcı;Web;Gezgin;Web sitesi;Site;sörf;çevrimiçi;tara +Keywords[uk]=Internet;WWW;Browser;Web;Explorer;Інтернет;мережа;переглядач;оглядач;браузер;веб;файрфокс;вогнелис;перегляд +Keywords[vi]=Internet;WWW;Browser;Web;Explorer;Trình duyệt;Trang web +Keywords[zh_CN]=Internet;WWW;Browser;Web;Explorer;网页;浏览;上网;火狐;Firefox;ff;互联网;网站; +Keywords[zh_TW]=Internet;WWW;Browser;Web;Explorer;網際網路;網路;瀏覽器;上網;網頁;火狐 +Exec=firefox %u +Icon=org.mozilla.firefox +Terminal=false +Type=Application +MimeType=text/html;text/xml;application/xhtml+xml;application/vnd.mozilla.xul+xml;text/mml;x-scheme-handler/http;x-scheme-handler/https; +StartupNotify=true +Categories=Network;WebBrowser; +Actions=new-window;new-private-window;profile-manager-window; +StartupWMClass=firefox + + +[Desktop Action new-window] +Name=Open a New Window +Name[ach]=Dirica manyen +Name[af]=Nuwe venster +Name[an]=Nueva finestra +Name[ar]=نافذة جديدة +Name[as]=নতুন উইন্ডো +Name[ast]=Ventana nueva +Name[az]=Yeni Pəncərə +Name[be]=Новае акно +Name[bg]=Нов прозорец +Name[bn_BD]=নতুন উইন্ডো (N) +Name[bn_IN]=নতুন উইন্ডো +Name[br]=Prenestr nevez +Name[brx]=गोदान उइन्ड'(N) +Name[bs]=Novi prozor +Name[ca]=Finestra nova +Name[cak]=K'ak'a' tzuwäch +Name[cs]=Nové okno +Name[cy]=Ffenestr Newydd +Name[da]=Nyt vindue +Name[de]=Neues Fenster +Name[dsb]=Nowe wokno +Name[el]=Νέο παράθυρο +Name[en_GB]=New Window +Name[en_US]=New Window +Name[en_ZA]=New Window +Name[eo]=Nova fenestro +Name[es_AR]=Nueva ventana +Name[es_CL]=Nueva ventana +Name[es_ES]=Nueva ventana +Name[es_MX]=Nueva ventana +Name[et]=Uus aken +Name[eu]=Leiho berria +Name[fa]=پنجره جدید‌ +Name[ff]=Henorde Hesere +Name[fi]=Uusi ikkuna +Name[fr]=Nouvelle fenêtre +Name[fy_NL]=Nij finster +Name[ga_IE]=Fuinneog Nua +Name[gd]=Uinneag ùr +Name[gl]=Nova xanela +Name[gn]=Ovetã pyahu +Name[gu_IN]=નવી વિન્ડો +Name[he]=חלון חדש +Name[hi_IN]=नया विंडो +Name[hr]=Novi prozor +Name[hsb]=Nowe wokno +Name[hu]=Új ablak +Name[hy_AM]=Նոր Պատուհան +Name[id]=Jendela Baru +Name[is]=Nýr gluggi +Name[it]=Nuova finestra +Name[ja]=新しいウィンドウ +Name[ja_JP]=新規ウインドウ +Name[ka]=ახალი ფანჯარა +Name[kk]=Жаңа терезе +Name[km]=បង្អួច​​​ថ្មី +Name[kn]=ಹೊಸ ಕಿಟಕಿ +Name[ko]=새 창 +Name[kok]=नवें जनेल +Name[ks]=نئئ وِنڈو +Name[lij]=Neuvo barcon +Name[lo]=ຫນ້າຕ່າງໃຫມ່ +Name[lt]=Naujas langas +Name[ltg]=Jauns lūgs +Name[lv]=Jauns logs +Name[mai]=नव विंडो +Name[mk]=Нов прозорец +Name[ml]=പുതിയ ജാലകം +Name[mr]=नवीन पटल +Name[ms]=Tetingkap Baru +Name[my]=ဝင်းဒိုးအသစ် +Name[nb_NO]=Nytt vindu +Name[ne_NP]=नयाँ सञ्झ्याल +Name[nl]=Nieuw venster +Name[nn_NO]=Nytt vindauge +Name[or]=ନୂତନ ୱିଣ୍ଡୋ +Name[pa_IN]=ਨਵੀਂ ਵਿੰਡੋ +Name[pl]=Nowe okno +Name[pt_BR]=Nova janela +Name[pt_PT]=Nova janela +Name[rm]=Nova fanestra +Name[ro]=Fereastră nouă +Name[ru]=Новое окно +Name[sat]=नावा विंडो (N) +Name[si]=නව කවුළුවක් +Name[sk]=Nové okno +Name[sl]=Novo okno +Name[son]=Zanfun taaga +Name[sq]=Dritare e Re +Name[sr]=Нови прозор +Name[sv_SE]=Nytt fönster +Name[ta]=புதிய சாளரம் +Name[te]=కొత్త విండో +Name[th]=หน้าต่างใหม่ +Name[tr]=Yeni pencere +Name[tsz]=Eraatarakua jimpani +Name[uk]=Нове вікно +Name[ur]=نیا دریچہ +Name[uz]=Yangi oyna +Name[vi]=Cửa sổ mới +Name[wo]=Palanteer bu bees +Name[xh]=Ifestile entsha +Name[zh_CN]=新建窗口 +Name[zh_TW]=開新視窗 +Exec=firefox --new-window %u + +[Desktop Action new-private-window] +Name=Open a New Private Window +Name[ach]=Dirica manyen me mung +Name[af]=Nuwe privaatvenster +Name[an]=Nueva finestra privada +Name[ar]=نافذة خاصة جديدة +Name[as]=নতুন ব্যক্তিগত উইন্ডো +Name[ast]=Ventana privada nueva +Name[az]=Yeni Məxfi Pəncərə +Name[be]=Новае акно адасаблення +Name[bg]=Нов прозорец за поверително сърфиране +Name[bn_BD]=নতুন ব্যক্তিগত উইন্ডো +Name[bn_IN]=নতুন ব্যক্তিগত উইন্ডো +Name[br]=Prenestr merdeiñ prevez nevez +Name[brx]=गोदान प्राइभेट उइन्ड' +Name[bs]=Novi privatni prozor +Name[ca]=Finestra privada nova +Name[cak]=K'ak'a' ichinan tzuwäch +Name[cs]=Nové anonymní okno +Name[cy]=Ffenestr Breifat Newydd +Name[da]=Nyt privat vindue +Name[de]=Neues privates Fenster +Name[dsb]=Nowe priwatne wokno +Name[el]=Νέο παράθυρο ιδιωτικής περιήγησης +Name[en_GB]=New Private Window +Name[en_US]=New Private Window +Name[en_ZA]=New Private Window +Name[eo]=Nova privata fenestro +Name[es_AR]=Nueva ventana privada +Name[es_CL]=Nueva ventana privada +Name[es_ES]=Nueva ventana privada +Name[es_MX]=Nueva ventana privada +Name[et]=Uus privaatne aken +Name[eu]=Leiho pribatu berria +Name[fa]=پنجره ناشناس جدید +Name[ff]=Henorde Suturo Hesere +Name[fi]=Uusi yksityinen ikkuna +Name[fr]=Nouvelle fenêtre de navigation privée +Name[fy_NL]=Nij priveefinster +Name[ga_IE]=Fuinneog Nua Phríobháideach +Name[gd]=Uinneag phrìobhaideach ùr +Name[gl]=Nova xanela privada +Name[gn]=Ovetã ñemi pyahu +Name[gu_IN]=નવી ખાનગી વિન્ડો +Name[he]=חלון פרטי חדש +Name[hi_IN]=नयी निजी विंडो +Name[hr]=Novi privatni prozor +Name[hsb]=Nowe priwatne wokno +Name[hu]=Új privát ablak +Name[hy_AM]=Սկսել Գաղտնի դիտարկում +Name[id]=Jendela Mode Pribadi Baru +Name[is]=Nýr huliðsgluggi +Name[it]=Nuova finestra anonima +Name[ja]=新しいプライベートウィンドウ +Name[ja_JP]=新規プライベートウインドウ +Name[ka]=ახალი პირადი ფანჯარა +Name[kk]=Жаңа жекелік терезе +Name[km]=បង្អួច​ឯកជន​ថ្មី +Name[kn]=ಹೊಸ ಖಾಸಗಿ ಕಿಟಕಿ +Name[ko]=새 사생활 보호 모드 +Name[kok]=नवो खाजगी विंडो +Name[ks]=نْو پرایوٹ وینڈو& +Name[lij]=Neuvo barcon privou +Name[lo]=ເປີດຫນ້າຕ່າງສວນຕົວຂື້ນມາໃຫມ່ +Name[lt]=Naujas privataus naršymo langas +Name[ltg]=Jauns privatais lūgs +Name[lv]=Jauns privātais logs +Name[mai]=नया निज विंडो (W) +Name[mk]=Нов приватен прозорец +Name[ml]=പുതിയ സ്വകാര്യ ജാലകം +Name[mr]=नवीन वैयक्तिक पटल +Name[ms]=Tetingkap Persendirian Baharu +Name[my]=New Private Window +Name[nb_NO]=Nytt privat vindu +Name[ne_NP]=नयाँ निजी सञ्झ्याल +Name[nl]=Nieuw privévenster +Name[nn_NO]=Nytt privat vindauge +Name[or]=ନୂତନ ବ୍ୟକ୍ତିଗତ ୱିଣ୍ଡୋ +Name[pa_IN]=ਨਵੀਂ ਪ੍ਰਾਈਵੇਟ ਵਿੰਡੋ +Name[pl]=Nowe okno prywatne +Name[pt_BR]=Nova janela privativa +Name[pt_PT]=Nova janela privada +Name[rm]=Nova fanestra privata +Name[ro]=Fereastră privată nouă +Name[ru]=Новое приватное окно +Name[sat]=नावा निजेराक् विंडो (W ) +Name[si]=නව පුද්ගලික කවුළුව (W) +Name[sk]=Nové okno v režime Súkromné prehliadanie +Name[sl]=Novo zasebno okno +Name[son]=Sutura zanfun taaga +Name[sq]=Dritare e Re Private +Name[sr]=Нови приватан прозор +Name[sv_SE]=Nytt privat fönster +Name[ta]=புதிய தனிப்பட்ட சாளரம் +Name[te]=కొత్త ఆంతరంగిక విండో +Name[th]=หน้าต่างส่วนตัวใหม่ +Name[tr]=Yeni gizli pencere +Name[tsz]=Juchiiti eraatarakua jimpani +Name[uk]=Приватне вікно +Name[ur]=نیا نجی دریچہ +Name[uz]=Yangi maxfiy oyna +Name[vi]=Cửa sổ riêng tư mới +Name[wo]=Panlanteeru biir bu bees +Name[xh]=Ifestile yangasese entsha +Name[zh_CN]=新建隐私浏览窗口 +Name[zh_TW]=新增隱私視窗 +Exec=firefox --private-window %u + +[Desktop Action profile-manager-window] +Name=Open the Profile Manager +Name[cs]=Správa profilů +Name[de]=Profilverwaltung öffnen +Name[es]=Abrir el Gestor de Perfiles +Name[fr]=Ouvrir le gestionnaire de profils +Exec=firefox --ProfileManager diff --git a/taskcluster/docker/firefox-flatpak/runme.sh b/taskcluster/docker/firefox-flatpak/runme.sh new file mode 100755 index 0000000000..d15aa0108c --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/runme.sh @@ -0,0 +1,186 @@ +#!/bin/bash +set -xe + +# Future products supporting Flatpaks will set this accordingly +: PRODUCT "${PRODUCT:=firefox}" + +# Required env variables + +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" +test "$FLATPAK_BRANCH" + +# Optional env variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" + +pwd + +# XXX: this is used to populate the datetime in org.mozilla.firefox.appdata.xml +DATE=$(date +%Y-%m-%d) +export DATE + +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +TARGET_TAR_XZ_FULL_PATH="$ARTIFACTS_DIR/target.flatpak.tar.xz" +SOURCE_DEST="${WORKSPACE}/source" + +# When updating this, please make sure to keep in sync the script for symbol +# scraping at +# https://github.com/mozilla/symbol-scrapers/blob/master/firefox-flatpak/script.sh +FREEDESKTOP_VERSION="23.08" +FIREFOX_BASEAPP_CHANNEL="23.08" + + +# XXX: these commands are temporarily, there's an upcoming fix in the upstream Docker image +# that we work on top of, from `freedesktopsdk`, that will make these two lines go away eventually +mkdir -p /root /tmp /var/tmp +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +# XXX ensure we have a clean slate in the local flatpak repo +rm -rf ~/.local/share/flatpak/ + + +CURL="curl --location --retry 10 --retry-delay 10" + +# Download en-US linux64 binary +$CURL -o "${WORKSPACE}/firefox.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/firefox-${VERSION}.tar.bz2" + +# Use list of locales to fetch L10N XPIs +$CURL -o "${WORKSPACE}/l10n_changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n_changesets.json") + +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +if [[ "$PRODUCT" == "firefox" ]]; then + # Get Flatpak configuration + PARTNER_CONFIG_DIR="$WORKSPACE/partner_config" + git clone https://github.com/mozilla-partners/flatpak.git "$PARTNER_CONFIG_DIR" + mv "$PARTNER_CONFIG_DIR/desktop/flatpak/distribution" "$DISTRIBUTION_DIR" +else + mkdir -p "$DISTRIBUTION_DIR" +fi + +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$DISTRIBUTION_DIR/extensions/langpack-${locale}@firefox.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +envsubst < "$SCRIPT_DIRECTORY/org.mozilla.firefox.appdata.xml.in" > "${WORKSPACE}/org.mozilla.firefox.appdata.xml" +cp -v "$SCRIPT_DIRECTORY/org.mozilla.firefox.desktop" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/launch-script.sh" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/firefox-symbolic.svg" "$WORKSPACE" +cd "${WORKSPACE}" + +flatpak remote-add --user --if-not-exists --from flathub https://dl.flathub.org/repo/flathub.flatpakrepo +# XXX: added --user to `flatpak install` to avoid ambiguity +flatpak install --user -y flathub org.mozilla.firefox.BaseApp//${FIREFOX_BASEAPP_CHANNEL} --no-deps + +# XXX: this command is temporarily, there's an upcoming fix in the upstream Docker image +# that we work on top of, from `freedesktopsdk`, that will make these two lines go away eventually +mkdir -p build +cp -r ~/.local/share/flatpak/app/org.mozilla.firefox.BaseApp/current/active/files build/files + +ARCH=$(flatpak --default-arch) +cat < build/metadata +[Application] +name=org.mozilla.firefox +runtime=org.freedesktop.Platform/${ARCH}/${FREEDESKTOP_VERSION} +sdk=org.freedesktop.Sdk/${ARCH}/${FREEDESKTOP_VERSION} +base=app/org.mozilla.firefox.BaseApp/${ARCH}/${FIREFOX_BASEAPP_CHANNEL} +[Extension org.mozilla.firefox.Locale] +directory=share/runtime/langpack +autodelete=true +locale-subset=true + +[Extension org.freedesktop.Platform.ffmpeg-full] +directory=lib/ffmpeg +add-ld-path=. +no-autodownload=true +version=${FREEDESKTOP_VERSION} + +[Extension org.mozilla.firefox.systemconfig] +directory=etc/firefox +no-autodownload=true +EOF + +cat < build/metadata.locale +[Runtime] +name=org.mozilla.firefox.Locale + +[ExtensionOf] +ref=app/org.mozilla.firefox/${ARCH}/${FLATPAK_BRANCH} +EOF + +appdir=build/files +install -d "${appdir}/lib/" +(cd "${appdir}/lib/" && tar jxf "${WORKSPACE}/firefox.tar.bz2") +install -D -m644 -t "${appdir}/share/appdata" org.mozilla.firefox.appdata.xml +install -D -m644 -t "${appdir}/share/applications" org.mozilla.firefox.desktop +for size in 16 32 48 64 128; do + install -D -m644 "${appdir}/lib/firefox/browser/chrome/icons/default/default${size}.png" "${appdir}/share/icons/hicolor/${size}x${size}/apps/org.mozilla.firefox.png" +done +install -D -m644 firefox-symbolic.svg "${appdir}/share/icons/hicolor/symbolic/apps/org.mozilla.firefox-symbolic.svg" +mkdir -p "${appdir}/lib/ffmpeg" +mkdir -p "${appdir}/etc/firefox" + +appstream-compose --prefix="${appdir}" --origin=flatpak --basename=org.mozilla.firefox org.mozilla.firefox +appstream-util mirror-screenshots "${appdir}"/share/app-info/xmls/org.mozilla.firefox.xml.gz "https://dl.flathub.org/repo/screenshots/org.mozilla.firefox-${FLATPAK_BRANCH}" build/screenshots "build/screenshots/org.mozilla.firefox-${FLATPAK_BRANCH}" + +# XXX: we used to `install -D` before which automatically created the components +# of target, now we need to manually do this since we're symlinking +mkdir -p "${appdir}/lib/firefox/distribution/extensions" +# XXX: we put the langpacks in /app/share/locale/$LANG_CODE and symlink that +# directory to where Firefox looks them up; this way only subset configured +# on user system is downloaded vs all locales +for locale in $locales; do + install -D -m644 -t "${appdir}/share/runtime/langpack/${locale%%-*}/" "${DISTRIBUTION_DIR}/extensions/langpack-${locale}@firefox.mozilla.org.xpi" + ln -sf "/app/share/runtime/langpack/${locale%%-*}/langpack-${locale}@firefox.mozilla.org.xpi" "${appdir}/lib/firefox/distribution/extensions/langpack-${locale}@firefox.mozilla.org.xpi" +done +install -D -m644 -t "${appdir}/lib/firefox/distribution" "$DISTRIBUTION_DIR/distribution.ini" +install -D -m755 launch-script.sh "${appdir}/bin/firefox" + +# We use features=devel to enable ptrace, which we need for the crash +# reporter. The application is still confined in a pid namespace, so +# that won't let us escape the flatpak sandbox. See bug 1653852. + +flatpak build-finish build \ + --allow=devel \ + --share=ipc \ + --share=network \ + --socket=pulseaudio \ + --socket=wayland \ + --socket=fallback-x11 \ + --socket=pcsc \ + --socket=cups \ + --require-version=0.11.1 \ + --persist=.mozilla \ + --env=DICPATH=/usr/share/hunspell \ + --filesystem=xdg-download:rw \ + --filesystem=/run/.heim_org.h5l.kcm-socket \ + --filesystem=xdg-run/speech-dispatcher:ro \ + --device=all \ + --talk-name=org.freedesktop.FileManager1 \ + --system-talk-name=org.freedesktop.NetworkManager \ + --talk-name=org.a11y.Bus \ + --talk-name="org.gtk.vfs.*" \ + --own-name="org.mpris.MediaPlayer2.firefox.*" \ + --own-name="org.mozilla.firefox.*" \ + --own-name="org.mozilla.firefox_beta.*" \ + --command=firefox + +flatpak build-export --disable-sandbox --no-update-summary --exclude='/share/runtime/langpack/*/*' repo build "$FLATPAK_BRANCH" +flatpak build-export --disable-sandbox --no-update-summary --metadata=metadata.locale --files=files/share/runtime/langpack repo build "$FLATPAK_BRANCH" +ostree commit --repo=repo --canonical-permissions --branch=screenshots/x86_64 build/screenshots +flatpak build-update-repo --generate-static-deltas repo +tar cvfJ flatpak.tar.xz repo + +mv -- flatpak.tar.xz "$TARGET_TAR_XZ_FULL_PATH" + +# XXX: if we ever wanted to go back to building flatpak bundles, we can revert this command; useful for testing individual artifacts, not publishable +# flatpak build-bundle "$WORKSPACE"/repo org.mozilla.firefox.flatpak org.mozilla.firefox +# TARGET_FULL_PATH="$ARTIFACTS_DIR/target.flatpak" +# mv -- *.flatpak "$TARGET_FULL_PATH" diff --git a/taskcluster/docker/firefox-snap/Dockerfile b/taskcluster/docker/firefox-snap/Dockerfile new file mode 100644 index 0000000000..dc002c0133 --- /dev/null +++ b/taskcluster/docker/firefox-snap/Dockerfile @@ -0,0 +1,71 @@ +## The below code is from snapcraft/docker/stable.Dockerfile +## The modifications done are part of the documentation for enabling core18 snaps. +## https://snapcraft.io/docs/t/creating-docker-images-for-snapcraft/11739 + +# /!\ The base image must follow the version of the `coreXX` package +# https://forum.snapcraft.io/t/issues-dockerizing-a-snapcraft-build-process/30294/5 +FROM ubuntu:focal + +ENV LANG='en_US.UTF-8' \ + LANGUAGE='en_US:en' \ + LC_ALL='en_US.UTF-8' \ + PATH="/snap/bin:$PATH" \ + SNAP='/snap/snapcraft/current' \ + SNAP_NAME='snapcraft' \ + SNAP_ARCH='amd64' \ + TERM='dumb' \ + TZ='Etc/UTC' + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Grab dependencies. snapd is now required per https://github.com/snapcore/snapcraft/pull/3210 +RUN apt-get update && \ + apt-get dist-upgrade --yes && \ + apt-get install --yes \ + curl \ + jq \ + squashfs-tools \ + locales \ + bzip2 \ + curl \ + gcc \ + git \ + python3 \ + locales \ + snapd \ + sudo \ + && \ +locale-gen "$LANG" + +COPY download_and_install_snap.sh . + +RUN bash download_and_install_snap.sh 'core20' +RUN bash download_and_install_snap.sh 'gnome-3-38-2004' +RUN bash download_and_install_snap.sh 'gnome-3-38-2004-sdk' +RUN bash download_and_install_snap.sh 'snapcraft' + +# Fix Python3 installation: Make sure we use the interpreter from +# the snapcraft snap: +RUN unlink /snap/snapcraft/current/usr/bin/python3 && \ + ln -s /snap/snapcraft/current/usr/bin/python3.* /snap/snapcraft/current/usr/bin/python3 && \ + echo /snap/snapcraft/current/lib/python3.*/site-packages >> /snap/snapcraft/current/usr/lib/python3/dist-packages/site-packages.pth + +# Create a snapcraft runner (TODO: move version detection to the core of snapcraft). +RUN mkdir -p /snap/bin +RUN echo "#!/bin/sh" > /snap/bin/snapcraft +RUN snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml)" && echo "export SNAP_VERSION=\"$snap_version\"" >> /snap/bin/snapcraft +RUN echo 'exec "$SNAP/usr/bin/python3" "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft +RUN chmod +x /snap/bin/snapcraft + +RUN mkdir /scripts/ +WORKDIR /scripts/ +# Copy everything in the docker/firefox-snap folder but the Dockerfile +# +# XXX The following pattern is neither a regex nor a glob one. It's +# documented at https://golang.org/pkg/path/filepath/#Match. There's no +# way of explicitly filtering out "Dockerfile". If one day, someone needs +# to add a file starting with "D", then we must revisit the pattern below. +COPY [^D]* /scripts/ + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/firefox-snap/Makefile b/taskcluster/docker/firefox-snap/Makefile new file mode 100644 index 0000000000..d71dc70881 --- /dev/null +++ b/taskcluster/docker/firefox-snap/Makefile @@ -0,0 +1,12 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = firefox-snapcraft +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest diff --git a/taskcluster/docker/firefox-snap/download_and_install_snap.sh b/taskcluster/docker/firefox-snap/download_and_install_snap.sh new file mode 100644 index 0000000000..7def88b2aa --- /dev/null +++ b/taskcluster/docker/firefox-snap/download_and_install_snap.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +SNAP_NAME="$1" +SNAP_CHANNEL="${2:-stable}" +SNAP_INSTALL_LOCATION="${3:-/snap}" + +SNAP_METADATA="$(curl --header 'X-Ubuntu-Series: 16' "https://api.snapcraft.io/api/v1/snaps/details/$SNAP_NAME?channel=$SNAP_CHANNEL")" + +set +x +SNAP_SHA512="$(echo "$SNAP_METADATA" | jq '.download_sha512' -r)" +SNAP_DOWNLOAD_URL="$(echo "$SNAP_METADATA" | jq '.download_url' -r)" +SNAP_LAST_UPDATED="$(echo "$SNAP_METADATA" | jq '.last_updated' -r)" +SNAP_REVISION="$(echo "$SNAP_METADATA" | jq '.revision' -r)" +SNAP_VERSION="$(echo "$SNAP_METADATA" | jq '.version' -r)" +set -x + +echo "Downloading $SNAP_NAME, version $SNAP_VERSION, revision $SNAP_REVISION (last updated: $SNAP_LAST_UPDATED)..." +curl --location "$SNAP_DOWNLOAD_URL" --output "$SNAP_NAME.snap" +sha512sum -c <(echo "$SNAP_SHA512 $SNAP_NAME.snap") + +mkdir -p "$SNAP_INSTALL_LOCATION/$SNAP_NAME" +unsquashfs -d "$SNAP_INSTALL_LOCATION/$SNAP_NAME/current" "$SNAP_NAME.snap" +rm "$SNAP_NAME.snap" + +echo "$SNAP_NAME version $SNAP_VERSION has correctly been uploaded and installed." \ No newline at end of file diff --git a/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py b/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..07bc007a31 --- /dev/null +++ b/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/taskcluster/docker/firefox-snap/firefox.desktop b/taskcluster/docker/firefox-snap/firefox.desktop new file mode 100644 index 0000000000..73e36e8d83 --- /dev/null +++ b/taskcluster/docker/firefox-snap/firefox.desktop @@ -0,0 +1,219 @@ +[Desktop Entry] +Version=1.0 +Name=Firefox Web Browser +Name[ar]=متصفح الويب فَيَرفُكْس +Name[ast]=Restolador web Firefox +Name[bn]=ফায়ারফক্স ওয়েব ব্রাউজার +Name[ca]=Navegador web Firefox +Name[cs]=Firefox Webový prohlížeč +Name[da]=Firefox - internetbrowser +Name[el]=Περιηγητής Firefox +Name[es]=Navegador web Firefox +Name[et]=Firefoxi veebibrauser +Name[fa]=مرورگر اینترنتی Firefox +Name[fi]=Firefox-selain +Name[fr]=Navigateur Web Firefox +Name[gl]=Navegador web Firefox +Name[he]=דפדפן האינטרנט Firefox +Name[hr]=Firefox web preglednik +Name[hu]=Firefox webböngésző +Name[it]=Firefox Browser Web +Name[ja]=Firefox ウェブ・ブラウザ +Name[ko]=Firefox 웹 브라우저 +Name[ku]=Geroka torê Firefox +Name[lt]=Firefox interneto naršyklė +Name[nb]=Firefox Nettleser +Name[nl]=Firefox webbrowser +Name[nn]=Firefox Nettlesar +Name[no]=Firefox Nettleser +Name[pl]=Przeglądarka WWW Firefox +Name[pt]=Firefox Navegador Web +Name[pt_BR]=Navegador Web Firefox +Name[ro]=Firefox – Navigator Internet +Name[ru]=Веб-браузер Firefox +Name[sk]=Firefox - internetový prehliadač +Name[sl]=Firefox spletni brskalnik +Name[sv]=Firefox webbläsare +Name[tr]=Firefox Web Tarayıcısı +Name[ug]=Firefox توركۆرگۈ +Name[uk]=Веб-браузер Firefox +Name[vi]=Trình duyệt web Firefox +Name[zh_CN]=Firefox 网络浏览器 +Name[zh_TW]=Firefox 網路瀏覽器 +Comment=Browse the World Wide Web +Comment[ar]=تصفح الشبكة العنكبوتية العالمية +Comment[ast]=Restola pela Rede +Comment[bn]=ইন্টারনেট ব্রাউজ করুন +Comment[ca]=Navegueu per la web +Comment[cs]=Prohlížení stránek World Wide Webu +Comment[da]=Surf på internettet +Comment[de]=Im Internet surfen +Comment[el]=Μπορείτε να περιηγηθείτε στο διαδίκτυο (Web) +Comment[es]=Navegue por la web +Comment[et]=Lehitse veebi +Comment[fa]=صفحات شبکه جهانی اینترنت را مرور نمایید +Comment[fi]=Selaa Internetin WWW-sivuja +Comment[fr]=Naviguer sur le Web +Comment[gl]=Navegar pola rede +Comment[he]=גלישה ברחבי האינטרנט +Comment[hr]=Pretražite web +Comment[hu]=A világháló böngészése +Comment[it]=Esplora il web +Comment[ja]=ウェブを閲覧します +Comment[ko]=웹을 돌아 다닙니다 +Comment[ku]=Li torê bigere +Comment[lt]=Naršykite internete +Comment[nb]=Surf på nettet +Comment[nl]=Verken het internet +Comment[nn]=Surf på nettet +Comment[no]=Surf på nettet +Comment[pl]=Przeglądanie stron WWW +Comment[pt]=Navegue na Internet +Comment[pt_BR]=Navegue na Internet +Comment[ro]=Navigați pe Internet +Comment[ru]=Доступ в Интернет +Comment[sk]=Prehliadanie internetu +Comment[sl]=Brskajte po spletu +Comment[sv]=Surfa på webben +Comment[tr]=İnternet'te Gezinin +Comment[ug]=دۇنيادىكى توربەتلەرنى كۆرگىلى بولىدۇ +Comment[uk]=Перегляд сторінок Інтернету +Comment[vi]=Để duyệt các trang web +Comment[zh_CN]=浏览互联网 +Comment[zh_TW]=瀏覽網際網路 +GenericName=Web Browser +GenericName[ar]=متصفح ويب +GenericName[ast]=Restolador Web +GenericName[bn]=ওয়েব ব্রাউজার +GenericName[ca]=Navegador web +GenericName[cs]=Webový prohlížeč +GenericName[da]=Webbrowser +GenericName[el]=Περιηγητής διαδικτύου +GenericName[es]=Navegador web +GenericName[et]=Veebibrauser +GenericName[fa]=مرورگر اینترنتی +GenericName[fi]=WWW-selain +GenericName[fr]=Navigateur Web +GenericName[gl]=Navegador Web +GenericName[he]=דפדפן אינטרנט +GenericName[hr]=Web preglednik +GenericName[hu]=Webböngésző +GenericName[it]=Browser web +GenericName[ja]=ウェブ・ブラウザ +GenericName[ko]=웹 브라우저 +GenericName[ku]=Geroka torê +GenericName[lt]=Interneto naršyklė +GenericName[nb]=Nettleser +GenericName[nl]=Webbrowser +GenericName[nn]=Nettlesar +GenericName[no]=Nettleser +GenericName[pl]=Przeglądarka WWW +GenericName[pt]=Navegador Web +GenericName[pt_BR]=Navegador Web +GenericName[ro]=Navigator Internet +GenericName[ru]=Веб-браузер +GenericName[sk]=Internetový prehliadač +GenericName[sl]=Spletni brskalnik +GenericName[sv]=Webbläsare +GenericName[tr]=Web Tarayıcı +GenericName[ug]=توركۆرگۈ +GenericName[uk]=Веб-браузер +GenericName[vi]=Trình duyệt Web +GenericName[zh_CN]=网络浏览器 +GenericName[zh_TW]=網路瀏覽器 +Keywords=Internet;WWW;Browser;Web;Explorer +Keywords[ar]=انترنت;إنترنت;متصفح;ويب;وب +Keywords[ast]=Internet;WWW;Restolador;Web;Esplorador +Keywords[ca]=Internet;WWW;Navegador;Web;Explorador;Explorer +Keywords[cs]=Internet;WWW;Prohlížeč;Web;Explorer +Keywords[da]=Internet;Internettet;WWW;Browser;Browse;Web;Surf;Nettet +Keywords[de]=Internet;WWW;Browser;Web;Explorer;Webseite;Site;surfen;online;browsen +Keywords[el]=Internet;WWW;Browser;Web;Explorer;Διαδίκτυο;Περιηγητής;Firefox;Φιρεφοχ;Ιντερνετ +Keywords[es]=Explorador;Internet;WWW +Keywords[fi]=Internet;WWW;Browser;Web;Explorer;selain;Internet-selain;internetselain;verkkoselain;netti;surffaa +Keywords[fr]=Internet;WWW;Browser;Web;Explorer;Fureteur;Surfer;Navigateur +Keywords[he]=דפדפן;אינטרנט;רשת;אתרים;אתר;פיירפוקס;מוזילה; +Keywords[hr]=Internet;WWW;preglednik;Web +Keywords[hu]=Internet;WWW;Böngésző;Web;Háló;Net;Explorer +Keywords[it]=Internet;WWW;Browser;Web;Navigatore +Keywords[is]=Internet;WWW;Vafri;Vefur;Netvafri;Flakk +Keywords[ja]=Internet;WWW;Web;インターネット;ブラウザ;ウェブ;エクスプローラ +Keywords[nb]=Internett;WWW;Nettleser;Explorer;Web;Browser;Nettside +Keywords[nl]=Internet;WWW;Browser;Web;Explorer;Verkenner;Website;Surfen;Online +Keywords[pt]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[pt_BR]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[ru]=Internet;WWW;Browser;Web;Explorer;интернет;браузер;веб;файрфокс;огнелис +Keywords[sk]=Internet;WWW;Prehliadač;Web;Explorer +Keywords[sl]=Internet;WWW;Browser;Web;Explorer;Brskalnik;Splet +Keywords[tr]=İnternet;WWW;Tarayıcı;Web;Gezgin;Web sitesi;Site;sörf;çevrimiçi;tara +Keywords[uk]=Internet;WWW;Browser;Web;Explorer;Інтернет;мережа;переглядач;оглядач;браузер;веб;файрфокс;вогнелис;перегляд +Keywords[vi]=Internet;WWW;Browser;Web;Explorer;Trình duyệt;Trang web +Keywords[zh_CN]=Internet;WWW;Browser;Web;Explorer;网页;浏览;上网;火狐;Firefox;ff;互联网;网站; +Keywords[zh_TW]=Internet;WWW;Browser;Web;Explorer;網際網路;網路;瀏覽器;上網;網頁;火狐 +Exec=firefox %u +Terminal=false +X-MultipleArgs=false +Type=Application +Icon=/browser/chrome/icons/default/default128.png +Categories=GNOME;GTK;Network;WebBrowser; +MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https;x-scheme-handler/ftp;x-scheme-handler/chrome;video/webm;application/x-xpinstall; +StartupNotify=true +Actions=NewWindow;NewPrivateWindow; + +[Desktop Action NewWindow] +Name=Open a New Window +Name[ar]=افتح نافذة جديدة +Name[ast]=Abrir una ventana nueva +Name[bn]=Abrir una ventana nueva +Name[ca]=Obre una finestra nova +Name[cs]=Otevřít nové okno +Name[da]=Åbn et nyt vindue +Name[de]=Ein neues Fenster öffnen +Name[el]=Άνοιγμα νέου παραθύρου +Name[es]=Abrir una ventana nueva +Name[fi]=Avaa uusi ikkuna +Name[fr]=Ouvrir une nouvelle fenêtre +Name[gl]=Abrir unha nova xanela +Name[he]=פתיחת חלון חדש +Name[hr]=Otvori novi prozor +Name[hu]=Új ablak nyitása +Name[it]=Apri una nuova finestra +Name[ja]=新しいウィンドウを開く +Name[ko]=새 창 열기 +Name[ku]=Paceyeke nû veke +Name[lt]=Atverti naują langą +Name[nb]=Åpne et nytt vindu +Name[nl]=Nieuw venster openen +Name[pt]=Abrir nova janela +Name[pt_BR]=Abrir nova janela +Name[ro]=Deschide o fereastră nouă +Name[ru]=Новое окно +Name[sk]=Otvoriť nové okno +Name[sl]=Odpri novo okno +Name[sv]=Öppna ett nytt fönster +Name[tr]=Yeni pencere aç +Name[ug]=يېڭى كۆزنەك ئېچىش +Name[uk]=Відкрити нове вікно +Name[vi]=Mở cửa sổ mới +Name[zh_CN]=新建窗口 +Name[zh_TW]=開啟新視窗 +Exec=firefox -new-window + +[Desktop Action NewPrivateWindow] +Name=Open a New Private Window +Name[ar]=افتح نافذة جديدة للتصفح الخاص +Name[ca]=Obre una finestra nova en mode d'incògnit +Name[de]=Ein neues privates Fenster öffnen +Name[es]=Abrir una ventana privada nueva +Name[fi]=Avaa uusi yksityinen ikkuna +Name[fr]=Ouvrir une nouvelle fenêtre de navigation privée +Name[he]=פתיחת חלון גלישה פרטית חדש +Name[hu]=Új privát ablak nyitása +Name[it]=Apri una nuova finestra anonima +Name[nb]=Åpne et nytt privat vindu +Name[ru]=Новое приватное окно +Name[sl]=Odpri novo okno zasebnega brskanja +Name[tr]=Yeni bir pencere aç +Name[uk]=Відкрити нове вікно у потайливому режимі +Name[zh_TW]=開啟新隱私瀏覽視窗 +Exec=firefox -private-window diff --git a/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in b/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in new file mode 100644 index 0000000000..82a65af2a8 --- /dev/null +++ b/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in @@ -0,0 +1,91 @@ +name: firefox +version: @VERSION@-@BUILD_NUMBER@ +summary: Mozilla Firefox web browser +description: Firefox is a powerful, extensible web browser with support for modern web application technologies. +confinement: strict +grade: stable +base: core20 +compression: lzo + +apps: + firefox: + command: firefox + command-chain: [tmpdir] + desktop: distribution/firefox.desktop + extensions: [gnome-3-38] + environment: + HOME: "$SNAP_USER_COMMON" + GTK_USE_PORTAL: 1 + slots: + - dbus-daemon + - mpris + plugs: + - avahi-observe + - browser-sandbox + - camera + - cups-control + - gsettings + - hardware-observe + - home + - joystick + - network + - network-observe + - opengl + - pulseaudio + - removable-media + - screen-inhibit-control + - system-packages-doc + - u2f-devices + - unity7 + - upower-observe + +plugs: + browser-sandbox: + interface: browser-support + allow-sandbox: true + etc-firefox-policies: + interface: system-files + read: [/etc/firefox/policies] + +layout: + /usr/share/libdrm: + bind: $SNAP/gnome-platform/usr/share/libdrm + +parts: + firefox: + plugin: dump + source: source + stage-packages: + - libxt6 + - libasound2 + - libpci3 + - libpulse0 + - libgl1-mesa-dri + - libgl1-mesa-glx + - libmirclient9 + - desktop-file-utils + - ffmpeg + - libc-bin + - locales-all + - libcurl3-gnutls + prime: + - -usr/lib/*/libharfbuzz* + - -usr/lib/*/*pango* + + # Find files provided by the base and platform snap and ensure they aren't + # duplicated in this snap + cleanup: + after: [firefox] + plugin: nil + build-snaps: [core20, gnome-3-38-2004] + override-prime: | + set -eux + for snap in "core20" "gnome-3-38-2004"; do + cd "/snap/$snap/current" && find . -type f,l -exec rm -f "$SNAPCRAFT_PRIME/{}" \; + done + +slots: + dbus-daemon: + interface: dbus + bus: session + name: org.mozilla.firefox diff --git a/taskcluster/docker/firefox-snap/policies.json b/taskcluster/docker/firefox-snap/policies.json new file mode 100644 index 0000000000..f36622021f --- /dev/null +++ b/taskcluster/docker/firefox-snap/policies.json @@ -0,0 +1,5 @@ +{ + "policies": { + "DisableAppUpdate": true + } +} diff --git a/taskcluster/docker/firefox-snap/runme.sh b/taskcluster/docker/firefox-snap/runme.sh new file mode 100755 index 0000000000..da084109cc --- /dev/null +++ b/taskcluster/docker/firefox-snap/runme.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -xe + +# Thunderbird Snap builds will set this to "thunderbird" +: PRODUCT "${PRODUCT:=firefox}" + +# Required env variables +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" + +# Optional env variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" +: PUSH_TO_CHANNEL "" + +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +TARGET="target.snap" +TARGET_FULL_PATH="$ARTIFACTS_DIR/$TARGET" +SOURCE_DEST="${WORKSPACE}/source" + +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +CURL="curl --location --retry 10 --retry-delay 10" + +# Download and extract en-US linux64 binary +$CURL -o "${WORKSPACE}/${PRODUCT}.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/${PRODUCT}-${VERSION}.tar.bz2" +tar -C "$SOURCE_DEST" -xf "${WORKSPACE}/${PRODUCT}.tar.bz2" --strip-components=1 + +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +if [[ "$PRODUCT" == "firefox" ]]; then + # Get Ubuntu configuration + PARTNER_CONFIG_DIR="$WORKSPACE/partner_config" + git clone https://github.com/mozilla-partners/canonical.git "$PARTNER_CONFIG_DIR" + mv "$PARTNER_CONFIG_DIR/desktop/ubuntu/distribution" "$DISTRIBUTION_DIR" +else + mkdir -p "$DISTRIBUTION_DIR" +fi + +cp -v "$SCRIPT_DIRECTORY/${PRODUCT}.desktop" "$DISTRIBUTION_DIR" + +# Add a group policy file to disable app updates, as those are handled by snapd +cp -v "$SCRIPT_DIRECTORY/policies.json" "$DISTRIBUTION_DIR" + +# Use list of locales to fetch L10N XPIs +$CURL -o "${WORKSPACE}/l10n_changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n_changesets.json") + +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$SOURCE_DEST/distribution/extensions/langpack-${locale}@${PRODUCT}.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +# In addition to the packages downloaded below, snapcraft fetches deb packages from ubuntu.com, +# when a snap is built,. They may bump packages there and remove the old ones. Updating the +# database allows snapcraft to find the latest packages. +# For more context, see 1448239 +apt-get update + +# Extract gtk30.mo from Ubuntu language packs +apt download language-pack-gnome-*-base +for i in *.deb; do + # shellcheck disable=SC2086 + dpkg-deb --fsys-tarfile $i | tar xv -C "$SOURCE_DEST" --wildcards "./usr/share/locale-langpack/*/LC_MESSAGES/gtk30.mo" || true +done + +# Add wrapper script to set TMPDIR appropriate for the snap +cp -v "$SCRIPT_DIRECTORY/tmpdir" "$SOURCE_DEST" + +# Generate snapcraft manifest +sed -e "s/@VERSION@/${VERSION}/g" -e "s/@BUILD_NUMBER@/${BUILD_NUMBER}/g" "${PRODUCT}.snapcraft.yaml.in" > "${WORKSPACE}/snapcraft.yaml" +cd "${WORKSPACE}" + +# Make sure snapcraft knows we're building amd64, even though we may not be on this arch. +export SNAP_ARCH='amd64' + +snapcraft + +mv -- *.snap "$TARGET_FULL_PATH" + +cd "$ARTIFACTS_DIR" + +# Generate checksums file +size=$(stat --printf="%s" "$TARGET_FULL_PATH") +sha=$(sha512sum "$TARGET_FULL_PATH" | awk '{print $1}') +echo "$sha sha512 $size $TARGET" > "$TARGET.checksums" + +echo "Generating signing manifest" +hash=$(sha512sum "$TARGET.checksums" | awk '{print $1}') + +cat << EOF > signing_manifest.json +[{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}] +EOF + +# For posterity +find . -ls +cat "$TARGET.checksums" +cat signing_manifest.json diff --git a/taskcluster/docker/firefox-snap/tmpdir b/taskcluster/docker/firefox-snap/tmpdir new file mode 100755 index 0000000000..e7e60801fa --- /dev/null +++ b/taskcluster/docker/firefox-snap/tmpdir @@ -0,0 +1,6 @@ +#!/bin/bash + +# Set TMPDIR to be under the user's default Downloads dir +export TMPDIR=$(xdg-user-dir DOWNLOAD)/firefox.tmp + +exec "$@" diff --git a/taskcluster/docker/funsize-update-generator/Dockerfile b/taskcluster/docker/funsize-update-generator/Dockerfile new file mode 100644 index 0000000000..bbd4f518df --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Dockerfile @@ -0,0 +1,48 @@ +FROM debian:11 + +# Required software +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + bzip2 \ + ca-certificates \ + curl \ + gcc \ + jq \ + libdpkg-perl \ + libgetopt-simple-perl \ + liblzma-dev \ + locales \ + python3 \ + libpython3-dev \ + python3-dev \ + python3-pip \ + xz-utils +RUN useradd -d /home/worker -s /bin/bash -m worker +COPY requirements.txt / + +RUN locale-gen en_CA.UTF-8 +ENV LANG en_CA.UTF-8 +ENV LANGUAGE en_CA.UTF-8 +ENV LANG_ALL en_CA.UTF-8 +ENV LC_ALL en_CA.UTF-8 + +RUN ["pip", "install", "-r", "/requirements.txt"] + +# scripts +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ + +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /*.sh +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker + +USER worker + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/funsize-update-generator/Makefile b/taskcluster/docker/funsize-update-generator/Makefile new file mode 100644 index 0000000000..6b67f0ed90 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Makefile @@ -0,0 +1,9 @@ +IMAGE_NAME = funsize-update-generator + +build: + docker build -t $(IMAGE_NAME) --no-cache --rm . + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff --git a/taskcluster/docker/funsize-update-generator/README b/taskcluster/docker/funsize-update-generator/README new file mode 100644 index 0000000000..5e9507be71 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/README @@ -0,0 +1,7 @@ + +To run this locally for testing/development purposes: + +1. Find a funsize generating task ID +2. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com" -e SIGNING_CERT='nightly' -e MAR_CHANNEL_ID='firefox-mozilla-central' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64" funsize-update-generator /runme.sh + +The TASK_ID should be a recent "partials" Task. diff --git a/taskcluster/docker/funsize-update-generator/dep1.pubkey b/taskcluster/docker/funsize-update-generator/dep1.pubkey new file mode 100644 index 0000000000..927b2cc947 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/dep1.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA8Y6AS+xwKoXZl0X5qOKr +0I00xC4UN+IMjA1LIQoZ2GBkiqQF3q8v2nWTFE0+47+3NtP0l8tvsQY+LSYR4Fek +v2Vx4m/CAMKmWzW6Vtlj80y6rQ04V19l41bZXvCIBW5fm9sAvPgc7CngkcLySNqk +8vf57cUEpOmbsjSOCmK0j8hh03I1eWogpbAVEchSm1xN2sUJaVTvz5j8BfE6Vm0i +nN7V0zF+AOxzvntZIpfUqMZbHRiMkGn4l9rjia1Rz0qUc9RNCJkNocyKtQ2N2wnN +FjHpmK9x2V71cS1JQGhgLegrswPCAWY1lTmiLk9LweqGoVL0rqR4LCkb0VCaeSRe +6bUEYcU1ZQedE80zGKB3AfoC5br1shYY0xjmyRSCQ8m8WE60HzXhL8wczKrn5yoJ +iF6BxFwcYsvrWBPgIYVZLcqjODfR/M62o8yIfTC7yBcIdycJ0sWhB47dHAFxv1kc +wv8Ik9ftvDyupE8kwcl58fNOXz93j7IxMry/ey27NyYpESPOUNcjT8TP26FdGebg +4iJx0/LaYmaNUdchfBBlaYqGdH6ZGK0OeVxzHstGuG0gebm/igYcpaFxiQzvWijX +MIAU56s4g+yj7pSzT5/s9r8Gv+YhsNHKm4hnwLZaITV0lLMT5h/OZGseQTPMBnAR +hK3CIfcqG0I23hdwI29ZuUMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/nightly.pubkey b/taskcluster/docker/funsize-update-generator/nightly.pubkey new file mode 100644 index 0000000000..e51049844c --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/nightly.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc +AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc +ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA +nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg +yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C +H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc +NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ +pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j +H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0 +lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d +2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV +Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/release.pubkey b/taskcluster/docker/funsize-update-generator/release.pubkey new file mode 100644 index 0000000000..ec1103d828 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/release.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6 +i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI +ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h +1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0 +GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi +LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6 +fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE +CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8 +m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF +K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U +zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK +r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/requirements.in b/taskcluster/docker/funsize-update-generator/requirements.in new file mode 100644 index 0000000000..3360c71690 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.in @@ -0,0 +1,7 @@ +aiohttp +awscli +mar +redo +requests +scriptworker +sh diff --git a/taskcluster/docker/funsize-update-generator/requirements.txt b/taskcluster/docker/funsize-update-generator/requirements.txt new file mode 100644 index 0000000000..c4ef0cfd0d --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.txt @@ -0,0 +1,663 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements.txt requirements.in +# +aiohttp==3.8.3 \ + --hash=sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8 \ + --hash=sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142 \ + --hash=sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18 \ + --hash=sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34 \ + --hash=sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a \ + --hash=sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033 \ + --hash=sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06 \ + --hash=sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4 \ + --hash=sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d \ + --hash=sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b \ + --hash=sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc \ + --hash=sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091 \ + --hash=sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d \ + --hash=sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85 \ + --hash=sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb \ + --hash=sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937 \ + --hash=sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf \ + --hash=sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1 \ + --hash=sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b \ + --hash=sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d \ + --hash=sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269 \ + --hash=sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da \ + --hash=sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346 \ + --hash=sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494 \ + --hash=sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697 \ + --hash=sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4 \ + --hash=sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585 \ + --hash=sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c \ + --hash=sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da \ + --hash=sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad \ + --hash=sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2 \ + --hash=sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6 \ + --hash=sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c \ + --hash=sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849 \ + --hash=sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa \ + --hash=sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b \ + --hash=sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb \ + --hash=sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7 \ + --hash=sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715 \ + --hash=sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76 \ + --hash=sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d \ + --hash=sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276 \ + --hash=sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6 \ + --hash=sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37 \ + --hash=sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb \ + --hash=sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d \ + --hash=sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c \ + --hash=sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446 \ + --hash=sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008 \ + --hash=sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342 \ + --hash=sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d \ + --hash=sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7 \ + --hash=sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061 \ + --hash=sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba \ + --hash=sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7 \ + --hash=sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290 \ + --hash=sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0 \ + --hash=sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d \ + --hash=sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8 \ + --hash=sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f \ + --hash=sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48 \ + --hash=sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502 \ + --hash=sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62 \ + --hash=sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9 \ + --hash=sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403 \ + --hash=sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77 \ + --hash=sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476 \ + --hash=sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e \ + --hash=sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96 \ + --hash=sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5 \ + --hash=sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784 \ + --hash=sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091 \ + --hash=sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b \ + --hash=sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97 \ + --hash=sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a \ + --hash=sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2 \ + --hash=sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9 \ + --hash=sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d \ + --hash=sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73 \ + --hash=sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017 \ + --hash=sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363 \ + --hash=sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c \ + --hash=sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d \ + --hash=sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618 \ + --hash=sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491 \ + --hash=sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b \ + --hash=sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca + # via + # -r requirements.in + # scriptworker + # taskcluster +aiomemoizettl==0.0.3 \ + --hash=sha256:07a6becac60f6cd2604b9f2b73bcd9a50079a0b7b55e2a4e45b1eec5a3ea9659 \ + --hash=sha256:0a80d2dc765e545263f515363b6700ec8cf86fa3968b529f56390b28e34f743d + # via scriptworker +aiosignal==1.3.1 \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 + # via aiohttp +arrow==1.2.3 \ + --hash=sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1 \ + --hash=sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2 + # via scriptworker +asn1crypto==1.5.1 \ + --hash=sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c \ + --hash=sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67 + # via mar +async-timeout==4.0.2 \ + --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ + --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c + # via + # aiohttp + # taskcluster +attrs==22.2.0 \ + --hash=sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836 \ + --hash=sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99 + # via + # aiohttp + # jsonschema +awscli==1.27.46 \ + --hash=sha256:276cef3cc89e6c1888eb327d4a2d7702105a49271b1e3e93472bcd9f02beb24c \ + --hash=sha256:a9b64b091db2edd13a611b4c4ec47fc2e7c67b3efe5ea840466f2138a9634c7b + # via -r requirements.in +botocore==1.29.46 \ + --hash=sha256:78bf25933e35eb6354a9e80fe156f86dce4d346a92afe364dfce25c17ab0639f \ + --hash=sha256:dbac2fde265f13beb9191ec3ff63b90b515e9ed63875edc3afbd72c5f585e48b + # via + # awscli + # s3transfer +certifi==2022.12.7 \ + --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ + --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 + # via requests +cffi==1.15.1 \ + --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ + --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ + --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ + --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ + --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ + --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ + --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ + --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ + --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ + --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ + --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ + --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ + --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ + --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ + --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ + --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ + --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ + --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ + --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ + --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ + --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ + --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ + --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ + --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ + --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ + --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ + --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ + --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ + --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ + --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ + --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ + --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ + --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ + --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ + --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ + --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ + --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ + --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ + --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ + --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ + --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ + --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ + --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ + --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ + --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ + --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ + --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ + --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ + --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ + --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ + --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ + --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ + --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ + --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ + --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ + --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ + --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ + --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ + --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ + --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ + --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ + --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ + --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ + --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via cryptography +charset-normalizer==2.1.1 \ + --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ + --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f + # via + # aiohttp + # requests +click==8.1.3 \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 + # via mar +colorama==0.4.4 \ + --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ + --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 + # via awscli +construct==2.10.68 \ + --hash=sha256:7b2a3fd8e5f597a5aa1d614c3bd516fa065db01704c72a1efaaeec6ef23d8b45 + # via mar +cryptography==39.0.0 \ + --hash=sha256:1a6915075c6d3a5e1215eab5d99bcec0da26036ff2102a1038401d6ef5bef25b \ + --hash=sha256:1ee1fd0de9851ff32dbbb9362a4d833b579b4a6cc96883e8e6d2ff2a6bc7104f \ + --hash=sha256:407cec680e811b4fc829de966f88a7c62a596faa250fc1a4b520a0355b9bc190 \ + --hash=sha256:50386acb40fbabbceeb2986332f0287f50f29ccf1497bae31cf5c3e7b4f4b34f \ + --hash=sha256:6f97109336df5c178ee7c9c711b264c502b905c2d2a29ace99ed761533a3460f \ + --hash=sha256:754978da4d0457e7ca176f58c57b1f9de6556591c19b25b8bcce3c77d314f5eb \ + --hash=sha256:76c24dd4fd196a80f9f2f5405a778a8ca132f16b10af113474005635fe7e066c \ + --hash=sha256:7dacfdeee048814563eaaec7c4743c8aea529fe3dd53127313a792f0dadc1773 \ + --hash=sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72 \ + --hash=sha256:844ad4d7c3850081dffba91cdd91950038ee4ac525c575509a42d3fc806b83c8 \ + --hash=sha256:875aea1039d78557c7c6b4db2fe0e9d2413439f4676310a5f269dd342ca7a717 \ + --hash=sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9 \ + --hash=sha256:ad04f413436b0781f20c52a661660f1e23bcd89a0e9bb1d6d20822d048cf2856 \ + --hash=sha256:bae6c7f4a36a25291b619ad064a30a07110a805d08dc89984f4f441f6c1f3f96 \ + --hash=sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288 \ + --hash=sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39 \ + --hash=sha256:e324de6972b151f99dc078defe8fb1b0a82c6498e37bff335f5bc6b1e3ab5a1e \ + --hash=sha256:e5d71c5d5bd5b5c3eebcf7c5c2bb332d62ec68921a8c593bea8c394911a005ce \ + --hash=sha256:f3ed2d864a2fa1666e749fe52fb8e23d8e06b8012e8bd8147c73797c506e86f1 \ + --hash=sha256:f671c1bb0d6088e94d61d80c606d65baacc0d374e67bf895148883461cd848de \ + --hash=sha256:f6c0db08d81ead9576c4d94bbb27aed8d7a430fa27890f39084c2d0e2ec6b0df \ + --hash=sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf \ + --hash=sha256:fec8b932f51ae245121c4671b4bbc030880f363354b2f0e0bd1366017d891458 + # via + # mar + # pyjwt + # scriptworker +dictdiffer==0.9.0 \ + --hash=sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578 \ + --hash=sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595 + # via scriptworker +docutils==0.16 \ + --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \ + --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc + # via awscli +frozenlist==1.3.3 \ + --hash=sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c \ + --hash=sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f \ + --hash=sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a \ + --hash=sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784 \ + --hash=sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27 \ + --hash=sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d \ + --hash=sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3 \ + --hash=sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678 \ + --hash=sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a \ + --hash=sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483 \ + --hash=sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8 \ + --hash=sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf \ + --hash=sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99 \ + --hash=sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c \ + --hash=sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48 \ + --hash=sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5 \ + --hash=sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56 \ + --hash=sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e \ + --hash=sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1 \ + --hash=sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401 \ + --hash=sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4 \ + --hash=sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e \ + --hash=sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649 \ + --hash=sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a \ + --hash=sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d \ + --hash=sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0 \ + --hash=sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6 \ + --hash=sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d \ + --hash=sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b \ + --hash=sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6 \ + --hash=sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf \ + --hash=sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef \ + --hash=sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7 \ + --hash=sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842 \ + --hash=sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba \ + --hash=sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420 \ + --hash=sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b \ + --hash=sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d \ + --hash=sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332 \ + --hash=sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936 \ + --hash=sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816 \ + --hash=sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91 \ + --hash=sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420 \ + --hash=sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448 \ + --hash=sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411 \ + --hash=sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4 \ + --hash=sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32 \ + --hash=sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b \ + --hash=sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0 \ + --hash=sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530 \ + --hash=sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669 \ + --hash=sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7 \ + --hash=sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1 \ + --hash=sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5 \ + --hash=sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce \ + --hash=sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4 \ + --hash=sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e \ + --hash=sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2 \ + --hash=sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d \ + --hash=sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9 \ + --hash=sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642 \ + --hash=sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0 \ + --hash=sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703 \ + --hash=sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb \ + --hash=sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1 \ + --hash=sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13 \ + --hash=sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab \ + --hash=sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38 \ + --hash=sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb \ + --hash=sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb \ + --hash=sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81 \ + --hash=sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8 \ + --hash=sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd \ + --hash=sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4 + # via + # aiohttp + # aiosignal +github3-py==3.2.0 \ + --hash=sha256:09b72be1497d346b0968cde8360a0d6af79dc206d0149a63cd3ec86c65c377cc \ + --hash=sha256:a9016e40609c6f5cb9954dd188d08257dafd09c4da8c0e830a033fca00054b0d + # via scriptworker +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 + # via + # requests + # yarl +immutabledict==2.2.3 \ + --hash=sha256:0e1e8a3f2b3ff062daa19795f947e9ec7a58add269d44e34d3ab4319e1343853 \ + --hash=sha256:a7b078ebcc4a58ddc73b55f808b26e7c8c2d5183fad325615112689e1a63e714 + # via scriptworker +jmespath==1.0.1 \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe + # via botocore +json-e==4.5.0 \ + --hash=sha256:618a94aecc8b8bc7733d6cd0ee7b676e45675566625a38958aa8b30379d9758f \ + --hash=sha256:e733ce77b4acbbc2c48211057f8cb5af45999e6be4ce0f07585c5580df45826e + # via scriptworker +jsonschema==4.17.3 \ + --hash=sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d \ + --hash=sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6 + # via scriptworker +mar==3.2.0 \ + --hash=sha256:602f64a99fb2db578a3d76c9f1ac3efd29bd5f8f0ae8568759fb205162b017d0 \ + --hash=sha256:dd032cf72b65c62fb4e58e04a347076fda41f84265b41dd248098cc11e396c6e + # via -r requirements.in +mohawk==1.1.0 \ + --hash=sha256:3ed296a30453d0b724679e0fd41e4e940497f8e461a9a9c3b7f36e43bab0fa09 \ + --hash=sha256:d2a0e3ab10a209cc79e95e28f2dd54bd4a73fd1998ffe27b7ba0f962b6be9723 + # via taskcluster +multidict==6.0.4 \ + --hash=sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9 \ + --hash=sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8 \ + --hash=sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03 \ + --hash=sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710 \ + --hash=sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161 \ + --hash=sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664 \ + --hash=sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569 \ + --hash=sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067 \ + --hash=sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313 \ + --hash=sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706 \ + --hash=sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2 \ + --hash=sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636 \ + --hash=sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49 \ + --hash=sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93 \ + --hash=sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603 \ + --hash=sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0 \ + --hash=sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60 \ + --hash=sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4 \ + --hash=sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e \ + --hash=sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1 \ + --hash=sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60 \ + --hash=sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951 \ + --hash=sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc \ + --hash=sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe \ + --hash=sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95 \ + --hash=sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d \ + --hash=sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8 \ + --hash=sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed \ + --hash=sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2 \ + --hash=sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775 \ + --hash=sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87 \ + --hash=sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c \ + --hash=sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2 \ + --hash=sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98 \ + --hash=sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3 \ + --hash=sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe \ + --hash=sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78 \ + --hash=sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660 \ + --hash=sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176 \ + --hash=sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e \ + --hash=sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988 \ + --hash=sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c \ + --hash=sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c \ + --hash=sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0 \ + --hash=sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449 \ + --hash=sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f \ + --hash=sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde \ + --hash=sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5 \ + --hash=sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d \ + --hash=sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac \ + --hash=sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a \ + --hash=sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9 \ + --hash=sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca \ + --hash=sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11 \ + --hash=sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35 \ + --hash=sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063 \ + --hash=sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b \ + --hash=sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982 \ + --hash=sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258 \ + --hash=sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1 \ + --hash=sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52 \ + --hash=sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480 \ + --hash=sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7 \ + --hash=sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461 \ + --hash=sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d \ + --hash=sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc \ + --hash=sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779 \ + --hash=sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a \ + --hash=sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547 \ + --hash=sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0 \ + --hash=sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171 \ + --hash=sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf \ + --hash=sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d \ + --hash=sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba + # via + # aiohttp + # yarl +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via rsa +pycparser==2.21 \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via cffi +pyjwt[crypto]==2.6.0 \ + --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \ + --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14 + # via github3-py +pyrsistent==0.19.3 \ + --hash=sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8 \ + --hash=sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440 \ + --hash=sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a \ + --hash=sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c \ + --hash=sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3 \ + --hash=sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393 \ + --hash=sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9 \ + --hash=sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da \ + --hash=sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf \ + --hash=sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64 \ + --hash=sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a \ + --hash=sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3 \ + --hash=sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98 \ + --hash=sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2 \ + --hash=sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8 \ + --hash=sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf \ + --hash=sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc \ + --hash=sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7 \ + --hash=sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28 \ + --hash=sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2 \ + --hash=sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b \ + --hash=sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a \ + --hash=sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64 \ + --hash=sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19 \ + --hash=sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1 \ + --hash=sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9 \ + --hash=sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c + # via jsonschema +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # arrow + # botocore + # github3-py + # taskcluster +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via + # awscli + # scriptworker +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 \ + --hash=sha256:c76e4c23ab2f8840261736a851323cd98493710e7a9d36a1058535dca501f293 + # via -r requirements.in +requests==2.28.1 \ + --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ + --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 + # via + # -r requirements.in + # github3-py + # taskcluster +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via awscli +s3transfer==0.6.0 \ + --hash=sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd \ + --hash=sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947 + # via awscli +scriptworker==46.0.0 \ + --hash=sha256:94efa759e539f36079abad40db074e9b3c1945602716f7facdd82fe5b53aa6a9 \ + --hash=sha256:e0b0b88c9d722c06d7a5bf19eab4f6f282613b36d171ba950b5501d8ad198207 + # via -r requirements.in +sh==1.14.3 \ + --hash=sha256:e4045b6c732d9ce75d571c79f5ac2234edd9ae4f5fa9d59b09705082bdca18c7 + # via -r requirements.in +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # mar + # mohawk + # python-dateutil +slugid==2.0.0 \ + --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \ + --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c + # via taskcluster +taskcluster==46.1.1 \ + --hash=sha256:1f49339b725a465d2ef4c9fb8bbe5ecaaf449707e5353fd6a5a453c776d057b6 \ + --hash=sha256:222c2511c9dfca4cc70ee64082c2b940e2e8513ae9df3adb7afedacac4ebc97d + # via scriptworker +taskcluster-urls==13.0.1 \ + --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \ + --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \ + --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b + # via taskcluster +uritemplate==4.1.1 \ + --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ + --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e + # via github3-py +urllib3==1.26.13 \ + --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ + --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 + # via + # botocore + # requests +yarl==1.8.2 \ + --hash=sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87 \ + --hash=sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89 \ + --hash=sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a \ + --hash=sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08 \ + --hash=sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996 \ + --hash=sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077 \ + --hash=sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901 \ + --hash=sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e \ + --hash=sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee \ + --hash=sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574 \ + --hash=sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165 \ + --hash=sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634 \ + --hash=sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229 \ + --hash=sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b \ + --hash=sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f \ + --hash=sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7 \ + --hash=sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf \ + --hash=sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89 \ + --hash=sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0 \ + --hash=sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1 \ + --hash=sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe \ + --hash=sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf \ + --hash=sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76 \ + --hash=sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951 \ + --hash=sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863 \ + --hash=sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06 \ + --hash=sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562 \ + --hash=sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6 \ + --hash=sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c \ + --hash=sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e \ + --hash=sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1 \ + --hash=sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3 \ + --hash=sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3 \ + --hash=sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778 \ + --hash=sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8 \ + --hash=sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2 \ + --hash=sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b \ + --hash=sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d \ + --hash=sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f \ + --hash=sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c \ + --hash=sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581 \ + --hash=sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918 \ + --hash=sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c \ + --hash=sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e \ + --hash=sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220 \ + --hash=sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37 \ + --hash=sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739 \ + --hash=sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77 \ + --hash=sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6 \ + --hash=sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42 \ + --hash=sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946 \ + --hash=sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5 \ + --hash=sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d \ + --hash=sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146 \ + --hash=sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a \ + --hash=sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83 \ + --hash=sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef \ + --hash=sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80 \ + --hash=sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588 \ + --hash=sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5 \ + --hash=sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2 \ + --hash=sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef \ + --hash=sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826 \ + --hash=sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05 \ + --hash=sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516 \ + --hash=sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0 \ + --hash=sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4 \ + --hash=sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2 \ + --hash=sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0 \ + --hash=sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd \ + --hash=sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8 \ + --hash=sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b \ + --hash=sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1 \ + --hash=sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c + # via aiohttp diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh new file mode 100644 index 0000000000..71ea04ea92 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +set -xe + +test "$TASK_ID" +test "$SIGNING_CERT" + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# Strip trailing / if present +TASKCLUSTER_ROOT_URL="${TASKCLUSTER_ROOT_URL%/}" +export TASKCLUSTER_ROOT_URL + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="${TASKCLUSTER_ROOT_URL%/}/api/queue/v1" + +curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + +# auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/ +# -> bucket of tc-gp-private-1d-us-east-1, path of releng/mbsdiff-cache/ +# Trailing slash is important, due to prefix permissions in S3. +S3_BUCKET_AND_PATH=$(jq -r '.scopes[] | select(contains ("auth:aws-s3"))' /home/worker/task.json | awk -F: '{print $4}') + +# Will be empty if there's no scope for AWS S3. +if [ -n "${S3_BUCKET_AND_PATH}" ] && getent hosts taskcluster +then + # Does this parse as we expect? + S3_PATH=${S3_BUCKET_AND_PATH#*/} + AWS_BUCKET_NAME=${S3_BUCKET_AND_PATH%/"${S3_PATH}"*} + test "${S3_PATH}" + test "${AWS_BUCKET_NAME}" + + set +x # Don't echo these. + secret_url="${TASKCLUSTER_PROXY_URL}/api/auth/v1/aws/s3/read-write/${AWS_BUCKET_NAME}/${S3_PATH}" + AUTH=$(curl "${secret_url}") + AWS_ACCESS_KEY_ID=$(echo "${AUTH}" | jq -r '.credentials.accessKeyId') + AWS_SECRET_ACCESS_KEY=$(echo "${AUTH}" | jq -r '.credentials.secretAccessKey') + AWS_SESSION_TOKEN=$(echo "${AUTH}" | jq -r '.credentials.sessionToken') + export AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY + export AWS_SESSION_TOKEN + AUTH= + + if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ]; then + # Pass the full bucket/path prefix, as the script just appends local files. + export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -S ${S3_BUCKET_AND_PATH}" + fi + set -x +else + # disable caching + export MBSDIFF_HOOK= +fi + +# EXTRA_PARAMS is optional +# shellcheck disable=SC2086 +python3 /home/worker/bin/funsize.py \ + --artifacts-dir "$ARTIFACTS_DIR" \ + --task-definition /home/worker/task.json \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + $EXTRA_PARAMS diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py new file mode 100644 index 0000000000..84fd2fbd0b --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import asyncio +import configparser +import json +import logging +import os +import shutil +import tempfile +import time +from contextlib import AsyncExitStack +from distutils.util import strtobool +from pathlib import Path + +import aiohttp +from mardor.reader import MarReader +from mardor.signing import get_keysize +from scriptworker.utils import get_hash, retry_async + +log = logging.getLogger(__name__) + + +ROOT_URL = os.environ.get( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" +) +QUEUE_PREFIX = f"{ROOT_URL}/api/queue/" +ALLOWED_URL_PREFIXES = ( + "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", + "http://download.cdn.mozilla.net/pub/firefox/nightly/", + "http://ftp.mozilla.org/", + "http://download.mozilla.org/", + "https://archive.mozilla.org/", + "http://archive.mozilla.org/", + QUEUE_PREFIX, +) +STAGING_URL_PREFIXES = ( + "http://ftp.stage.mozaws.net/", + "https://ftp.stage.mozaws.net/", +) + +BCJ_OPTIONS = { + "x86": ["--x86"], + "x86_64": ["--x86"], + "aarch64": [], + # macOS Universal Builds + "macos-x86_64-aarch64": [], +} + + +def verify_signature(mar, cert): + log.info("Checking %s signature", mar) + with open(mar, "rb") as mar_fh: + m = MarReader(mar_fh) + if not m.verify(verify_key=cert): + raise ValueError( + "MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert + ) + + +def process_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-dir", required=True) + parser.add_argument("--signing-cert", type=argparse.FileType("rb"), required=True) + parser.add_argument("--task-definition", required=True, type=argparse.FileType("r")) + parser.add_argument( + "--allow-staging-prefixes", + action="store_true", + default=strtobool(os.environ.get("FUNSIZE_ALLOW_STAGING_PREFIXES", "false")), + help="Allow files from staging buckets.", + ) + parser.add_argument( + "-q", + "--quiet", + dest="log_level", + action="store_const", + const=logging.INFO, + default=logging.DEBUG, + ) + parser.add_argument( + "--arch", + type=str, + required=True, + choices=BCJ_OPTIONS.keys(), + help="The archtecture you are building.", + ) + return parser.parse_args() + + +def validate_mar_channel_id(mar, channel_ids): + log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids) + # We may get a string with a list representation, or a single entry string. + channel_ids = set(channel_ids.split(",")) + + product_info = MarReader(open(mar, "rb")).productinfo + if not isinstance(product_info, tuple): + raise ValueError( + "Malformed product information in mar: {}".format(product_info) + ) + + found_channel_ids = set(product_info[1].split(",")) + + if not found_channel_ids.issubset(channel_ids): + raise ValueError( + "MAR_CHANNEL_ID mismatch, {} not in {}".format(product_info[1], channel_ids) + ) + + log.info("%s channel %s in %s", mar, product_info[1], channel_ids) + + +async def retry_download(*args, semaphore=None, **kwargs): # noqa: E999 + """Retry download() calls.""" + async with AsyncExitStack() as stack: + if semaphore: + await stack.enter_async_context(semaphore) + await retry_async( + download, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + args=args, + kwargs=kwargs, + ) + + +def verify_allowed_url(mar, allowed_url_prefixes): + if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes): + raise ValueError( + "{mar} is not in allowed URL prefixes: {p}".format( + mar=mar, p=allowed_url_prefixes + ) + ) + + +async def download(url, dest, mode=None): # noqa: E999 + log.info("Downloading %s to %s", url, dest) + chunk_size = 4096 + bytes_downloaded = 0 + async with aiohttp.ClientSession(raise_for_status=True) as session: + start = time.time() + async with session.get(url, timeout=120) as resp: + # Additional early logging for download timeouts. + log.debug("Fetching from url %s", resp.url) + for history in resp.history: + log.debug("Redirection history: %s", history.url) + log.debug("Headers for %s: %s", resp.url, resp.headers) + if "Content-Length" in resp.headers: + log.debug( + "Content-Length expected for %s: %s", + url, + resp.headers["Content-Length"], + ) + log_interval = chunk_size * 1024 + with open(dest, "wb") as fd: + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + fd.write(chunk) + bytes_downloaded += len(chunk) + log_interval -= len(chunk) + if log_interval <= 0: + log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded) + log_interval = chunk_size * 1024 + end = time.time() + log.info( + "Downloaded %s, %s bytes in %s seconds: sha256:%s", + url, + bytes_downloaded, + int(end - start), + get_hash(dest, hash_alg="sha256"), + ) + if mode: + log.info("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +async def download_buildsystem_bits(partials_config, downloads, tools_dir): + """Download external tools needed to make partials.""" + + # We're making the assumption that the "to" mar is the same for all, + # as that's the way this task is currently used. + to_url = extract_download_urls(partials_config, mar_type="to").pop() + + repo = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceRepository", + ) + revision = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceStamp", + ) + + urls = { + "make_incremental_update.sh": f"{repo}/raw-file/{revision}/tools/" + "update-packaging/make_incremental_update.sh", + "common.sh": f"{repo}/raw-file/{revision}/tools/update-packaging/common.sh", + "mar": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mar", + "mbsdiff": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mbsdiff", + } + for filename, url in urls.items(): + filename = tools_dir / filename + await retry_download(url, dest=filename, mode=0o755) + + +def find_file(directory, filename): + log.debug("Searching for %s in %s", filename, directory) + return next(Path(directory).rglob(filename)) + + +def get_option(directory, filename, section, option): + log.info("Extracting [%s]: %s from %s/**/%s", section, option, directory, filename) + f = find_file(directory, filename) + config = configparser.ConfigParser() + config.read(f) + rv = config.get(section, option) + log.info("Found %s", rv) + return rv + + +def extract_download_urls(partials_config, mar_type): + """Extract a set of urls to download from the task configuration. + + mar_type should be one of "from", "to" + """ + return {definition[f"{mar_type}_mar"] for definition in partials_config} + + +async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert): + """Download, check signature, channel ID and unpack MAR files.""" + # Separate these categories so we can opt to perform checks on only 'to' downloads. + from_urls = extract_download_urls(partials_config, mar_type="from") + to_urls = extract_download_urls(partials_config, mar_type="to") + tasks = list() + downloads = dict() + + semaphore = asyncio.Semaphore(2) # Magic 2 to reduce network timeout errors. + for url in from_urls.union(to_urls): + verify_allowed_url(url, allowed_url_prefixes) + downloads[url] = { + "download_path": Path(tempfile.mkdtemp()) / Path(url).name, + } + tasks.append( + retry_download(url, downloads[url]["download_path"], semaphore=semaphore) + ) + + await asyncio.gather(*tasks) + + for url in downloads: + # Verify signature, but not from an artifact as we don't + # depend on the signing task + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION") and not url.startswith( + QUEUE_PREFIX + ): + verify_signature(downloads[url]["download_path"], signing_cert) + + # Only validate the target channel ID, as we update from beta->release + if url in to_urls: + validate_mar_channel_id( + downloads[url]["download_path"], os.environ["MAR_CHANNEL_ID"] + ) + + downloads[url]["extracted_path"] = tempfile.mkdtemp() + with open(downloads[url]["download_path"], "rb") as mar_fh: + log.info( + "Unpacking %s into %s", + downloads[url]["download_path"], + downloads[url]["extracted_path"], + ) + m = MarReader(mar_fh) + m.extract(downloads[url]["extracted_path"]) + + return downloads + + +async def run_command(cmd, cwd="/", env=None, label=None, silent=False): + log.info("Running: %s", cmd) + if not env: + env = dict() + process = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + env=env, + ) + if label: + label = "{}: ".format(label) + else: + label = "" + + async def read_output(stream, label, printcmd): + while True: + line = await stream.readline() + if line == b"": + break + printcmd("%s%s", label, line.decode("utf-8").rstrip()) + + if silent: + await process.wait() + else: + await asyncio.gather( + read_output(process.stdout, label, log.info), + read_output(process.stderr, label, log.warning), + ) + await process.wait() + + +async def generate_partial(from_dir, to_dir, dest_mar, mar_data, tools_dir, arch): + log.info("Generating partial %s", dest_mar) + env = os.environ.copy() + env["LC_ALL"] = "C" + env["MAR"] = tools_dir / "mar" + env["MBSDIFF"] = tools_dir / "mbsdiff" + if arch: + env["BCJ_OPTIONS"] = " ".join(BCJ_OPTIONS[arch]) + env["MOZ_PRODUCT_VERSION"] = mar_data["version"] + env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"] + env["BRANCH"] = mar_data["branch"] + + make_incremental_update = tools_dir / "make_incremental_update.sh" + cmd = f"{make_incremental_update} {dest_mar} {from_dir} {to_dir}" + + await run_command(cmd, cwd=dest_mar.parent, env=env, label=dest_mar.name) + validate_mar_channel_id(dest_mar, mar_data["MAR_CHANNEL_ID"]) + + +async def manage_partial( + partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None +): + from_url = partial_def["from_mar"] + to_url = partial_def["to_mar"] + from_path = downloads[from_url]["extracted_path"] + to_path = downloads[to_url]["extracted_path"] + + mar_data = { + "MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"], + "version": get_option( + to_path, filename="application.ini", section="App", option="Version" + ), + "appName": get_option( + from_path, filename="application.ini", section="App", option="Name" + ), + # Use Gecko repo and rev from platform.ini, not application.ini + "repo": get_option( + to_path, filename="platform.ini", section="Build", option="SourceRepository" + ), + "revision": get_option( + to_path, filename="platform.ini", section="Build", option="SourceStamp" + ), + "locale": partial_def["locale"], + "from_mar": partial_def["from_mar"], + "from_size": os.path.getsize(downloads[from_url]["download_path"]), + "from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"), + "from_buildid": get_option( + from_path, filename="application.ini", section="App", option="BuildID" + ), + "to_mar": partial_def["to_mar"], + "to_size": os.path.getsize(downloads[to_url]["download_path"]), + "to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"), + "to_buildid": get_option( + to_path, filename="application.ini", section="App", option="BuildID" + ), + "mar": partial_def["dest_mar"], + } + # if branch not set explicitly use repo-name + mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name) + + for field in ( + "update_number", + "previousVersion", + "previousBuildNumber", + "toVersion", + "toBuildNumber", + ): + if field in partial_def: + mar_data[field] = partial_def[field] + + dest_mar = Path(artifacts_dir) / mar_data["mar"] + + async with semaphore: + await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch) + + mar_data["size"] = os.path.getsize(dest_mar) + mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512") + return mar_data + + +async def async_main(args, signing_cert): + tasks = [] + + allowed_url_prefixes = list(ALLOWED_URL_PREFIXES) + if args.allow_staging_prefixes: + allowed_url_prefixes += STAGING_URL_PREFIXES + + task = json.load(args.task_definition) + + downloads = await download_and_verify_mars( + task["extra"]["funsize"]["partials"], allowed_url_prefixes, signing_cert + ) + + tools_dir = Path(tempfile.mkdtemp()) + await download_buildsystem_bits( + partials_config=task["extra"]["funsize"]["partials"], + downloads=downloads, + tools_dir=tools_dir, + ) + + # May want to consider os.cpu_count() if we ever run on osx/win. + # sched_getaffinity is the list of cores we can run on, not the total. + semaphore = asyncio.Semaphore(len(os.sched_getaffinity(0))) + for definition in task["extra"]["funsize"]["partials"]: + tasks.append( + asyncio.ensure_future( + retry_async( + manage_partial, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + kwargs=dict( + partial_def=definition, + artifacts_dir=args.artifacts_dir, + tools_dir=tools_dir, + arch=args.arch, + downloads=downloads, + semaphore=semaphore, + ), + ) + ) + ) + manifest = await asyncio.gather(*tasks) + + for url in downloads: + downloads[url]["download_path"].unlink() + shutil.rmtree(downloads[url]["extracted_path"]) + shutil.rmtree(tools_dir) + + return manifest + + +def main(): + args = process_arguments() + + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") + log.setLevel(args.log_level) + + signing_cert = args.signing_cert.read() + assert get_keysize(signing_cert) == 4096 + + artifacts_dir = Path(args.artifacts_dir) + if not artifacts_dir.exists(): + artifacts_dir.mkdir() + + loop = asyncio.get_event_loop() + manifest = loop.run_until_complete(async_main(args, signing_cert)) + loop.close() + + manifest_file = artifacts_dir / "manifest.json" + with open(manifest_file, "w") as fp: + json.dump(manifest, fp, indent=2, sort_keys=True) + + log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True))) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh new file mode 100755 index 0000000000..965d938247 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool contains functions that are to be used to handle/enable funsize +# Author: Mihai Tabara +# + +HOOK= +AWS_BUCKET_NAME= +LOCAL_CACHE_DIR= + +# Don't cache files smaller than this, as it's slower with S3 +# Bug 1437473 +CACHE_THRESHOLD=500000 + +S3_CACHE_HITS=0 +S3_CACHE_MISSES=0 + +getsha512(){ + openssl sha512 "${1}" | awk '{print $2}' +} + +print_usage(){ + echo "$(basename "$0") [-S S3-BUCKET-NAME] [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" + echo "Script that saves/retrieves from cache presumptive patches as args" + echo "" + echo "-A SERVER-URL - host where to send the files" + echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" + echo "-g pre hook - tests whether patch already in cache" + echo "-u post hook - upload patch to cache for future use" + echo "" + echo "PATH-FROM-URL : path on disk for source file" + echo "PATH-TO-URL : path on disk for destination file" + echo "PATH-PATCH : path on disk for patch between source and destination" +} + +upload_patch(){ + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 0 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + patch_path="$3" + patch_filename="$(basename "$3")" + + # save to local cache first + if [ -n "$LOCAL_CACHE_DIR" ]; then + local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" + if $local_cmd >&2; then + cp -avf "${patch_path}" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" + echo "${patch_path} saved on local cache." + fi + fi + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${patch_filename}" + if aws s3 cp "${patch_path}" "${BUCKET_PATH}"; then + echo "${patch_path} saved on s://${AWS_BUCKET_NAME}" + return 0 + fi + echo "${patch_path} failed to be uploaded to s3://${AWS_BUCKET_NAME}" + return 1 + fi + return 0 +} + +get_patch(){ + # $1 and $2 are the /path/to/filename + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 1 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + destination_file="$3" + s3_filename="$(basename "$3")" + + # Try to retrieve from local cache first. + if [ -n "$LOCAL_CACHE_DIR" ]; then + if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then + cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" + echo "Successful retrieved ${destination_file} from local cache." + return 0 + fi + fi + # If not in the local cache, we might find it remotely. + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${s3_filename}" + if aws s3 ls "${BUCKET_PATH}"; then + ((S3_CACHE_HITS++)) + echo "s3 cache hit for ${s3_filename} (${S3_CACHE_HITS} total hits)" + if aws s3 cp "${BUCKET_PATH}" "${destination_file}"; then + echo "Successful retrieved ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 0 + else + echo "Failed to retrieve ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 1 + fi + # Not found, fall through to default error + else + ((S3_CACHE_MISSES++)) + echo "s3 cache miss for ${s3_filename} (${S3_CACHE_MISSES} total misses)" + fi + fi + return 1 +} + +OPTIND=1 + +while getopts ":S:c:gu" option; do + case $option in + S) + # This will probably be bucketname/path/prefix but we can use it either way + AWS_BUCKET_NAME="$OPTARG" + # Ensure trailing slash is there. + if [[ ! $AWS_BUCKET_NAME =~ .*/$ ]]; then + AWS_BUCKET_NAME="${AWS_BUCKET_NAME}/" + fi + ;; + c) + LOCAL_CACHE_DIR="$OPTARG" + ;; + g) + HOOK="PRE" + ;; + u) + HOOK="POST" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_usage + exit 1 + ;; + *) + echo "Unimplemented option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$HOOK" == "PRE" ]; then + get_patch "$1" "$2" "$3" +elif [ "$HOOK" == "POST" ]; then + upload_patch "$1" "$2" "$3" +fi diff --git a/taskcluster/docker/gdb-test/Dockerfile b/taskcluster/docker/gdb-test/Dockerfile new file mode 100644 index 0000000000..071f9554af --- /dev/null +++ b/taskcluster/docker/gdb-test/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install gdb diff --git a/taskcluster/docker/github-sync/Dockerfile b/taskcluster/docker/github-sync/Dockerfile new file mode 100644 index 0000000000..bff131ad7b --- /dev/null +++ b/taskcluster/docker/github-sync/Dockerfile @@ -0,0 +1,14 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Kartikaya Gupta + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/.ghsync + +ADD prepare.sh /setup/prepare-docker.sh +ADD requirements.txt /setup/requirements.txt +RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/github-sync/prepare.sh b/taskcluster/docker/github-sync/prepare.sh new file mode 100755 index 0000000000..6c0be27168 --- /dev/null +++ b/taskcluster/docker/github-sync/prepare.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +test "$(whoami)" == 'root' + +# Install stuff we need +apt-get -y update +apt-get install -y \ + cmake \ + curl \ + gcc \ + git \ + g++ \ + libffi-dev \ + libgit2-dev \ + libssl-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools + +# Python packages +pip3 install --break-system-packages -r "$1" diff --git a/taskcluster/docker/github-sync/requirements.txt b/taskcluster/docker/github-sync/requirements.txt new file mode 100644 index 0000000000..13fd42681d --- /dev/null +++ b/taskcluster/docker/github-sync/requirements.txt @@ -0,0 +1,3 @@ +requests == 2.21.0 +pygit2 == 1.11.1 +python-hglib == 2.6.1 diff --git a/taskcluster/docker/image_builder/Dockerfile b/taskcluster/docker/image_builder/Dockerfile new file mode 100644 index 0000000000..ad38a48c49 --- /dev/null +++ b/taskcluster/docker/image_builder/Dockerfile @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM golang:1.14 as skopeo + +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] +RUN ["git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] +ENV GO111MODULE=on CGO_ENABLED=0 +RUN ["go", "build", \ + "-mod=vendor", "-o", "out/skopeo", \ + "-tags", "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp", \ + # Set unixTempDirForBigFiles so skopeo will extract in a directory hidden by kaniko + # We create the directory below. + "-ldflags", " -X github.com/containers/image/v5/internal/tmpdir.unixTempDirForBigFiles=/workspace/tmp -X github.com/containers/image/v5/signature.systemDefaultPolicyPath=/kaniko/containers/policy.json -extldflags \"-static\" -w -s", \ + "./cmd/skopeo"] + +FROM golang:1.14 as kaniko +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] +RUN ["make"] + +# Build the `build-image` command as a static binary using musl +# The setup is loosely based on a stripped down version of +# https://github.com/emk/rust-musl-builder/blob/master/Dockerfile +FROM debian:buster as build-image + +COPY apt.conf /etc/apt/apt.conf.d/99taskcluster + +RUN apt-get update && \ + apt-get install \ + build-essential \ + ca-certificates \ + curl \ + musl-dev \ + musl-tools \ + && \ + useradd rust --user-group --create-home --shell /bin/bash + +# Run all further code as user `rust`, and create our working directories +# as the appropriate user. +USER rust + +# Set up our path with all our binary directories, including those for the +# musl-gcc toolchain and for our Rust toolchain. +ENV PATH=/home/rust/.cargo/bin:$PATH + +# The Rust toolchain to use when building our image. Set by `hooks/build`. +ENV TOOLCHAIN=1.42.0 \ + TARGET=x86_64-unknown-linux-musl + +# Install our Rust toolchain and the `musl` target. We patch the +# command-line we pass to the installer so that it won't attempt to +# interact with the user or fool around with TTYs. We also set the default +# `--target` to musl so that our users don't need to keep overriding it +# manually. +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- -y \ + --profile minimal \ + --default-toolchain $TOOLCHAIN \ + --target $TARGET + +# Expect our source code to live in /home/rust/src. We'll run the build as +# user `rust`, which will be uid 1000, gid 1000 outside the container. +RUN mkdir -p /home/rust/src +WORKDIR /home/rust/src +# Add our source code. +ADD --chown=rust:rust build-image/ ./ + +# --out-dir is not yet stable +ENV RUSTC_BOOTSTRAP=1 +# Build our application. +RUN ["cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] + +FROM scratch as empty + +FROM scratch + +COPY --from=skopeo /go/src/out/skopeo /kaniko-bootstrap/skopeo +COPY --from=kaniko /go/src/out/executor /kaniko-bootstrap/executor +COPY --from=build-image \ + /home/rust/src/bin/build-image \ + /kaniko-bootstrap/build-image + +ADD https://mkcert.org/generate/ /kaniko-bootstrap/ssl/certs/ca-certificats.crt +ENV SSL_CERT_DIR=/kaniko/ssl/certs + +ADD policy.json /kaniko-bootstrap/containers/policy.json + +ENV HOME /root +ENV USER /root +WORKDIR /workspace + +ENV PATH /usr/local/bin:/kaniko + +VOLUME /workspace +ENTRYPOINT ["/kaniko-bootstrap/build-image"] diff --git a/taskcluster/docker/image_builder/VERSION b/taskcluster/docker/image_builder/VERSION new file mode 100644 index 0000000000..fcdb2e109f --- /dev/null +++ b/taskcluster/docker/image_builder/VERSION @@ -0,0 +1 @@ +4.0.0 diff --git a/taskcluster/docker/image_builder/apt.conf b/taskcluster/docker/image_builder/apt.conf new file mode 100644 index 0000000000..84c0cf10ef --- /dev/null +++ b/taskcluster/docker/image_builder/apt.conf @@ -0,0 +1,5 @@ +quiet "true"; +APT::Get::Assume-Yes "true"; +APT::Install-Recommends "false"; +Acquire::Check-Valid-Until "false"; +Acquire::Retries "5"; diff --git a/taskcluster/docker/image_builder/build-image/Cargo.lock b/taskcluster/docker/image_builder/build-image/Cargo.lock new file mode 100644 index 0000000000..7e07920591 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/Cargo.lock @@ -0,0 +1,1085 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "build-image" +version = "0.1.0" +dependencies = [ + "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd 0.5.3+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cc" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "encoding_rs" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "envy" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-channel" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-util" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "generator" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "h2" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hermit-abi" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-channel 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-rustls" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "indexmap" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "js-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "loom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "once_cell" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-project" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pin-project-internal 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-hack" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "reqwest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-futures 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki-roots 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ring" +version = "0.16.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustls" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tokio" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-rustls" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "try-lock" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tinyvec 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro-support 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "web-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "webpki" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "webpki-roots" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd" +version = "0.5.3+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "zstd-safe 2.0.5+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-safe" +version = "2.0.5+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd-sys 1.4.17+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-sys" +version = "1.4.17+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" +"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +"checksum base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +"checksum bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "118cf036fbb97d0816e3c34b2d7a1e8cfc60f68fcf63d550ddbe9bd5f59c213b" +"checksum cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "77c1f1d60091c1b73e2b1f4560ab419204b178e625fa945ded7b660becd2bd46" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +"checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +"checksum envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f938a4abd5b75fe3737902dbc2e79ca142cc1526827a9e40b829a086758531a9" +"checksum fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures-channel 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +"checksum futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +"checksum futures-io 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" +"checksum futures-macro 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +"checksum futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" +"checksum futures-task 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +"checksum futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +"checksum generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +"checksum h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" +"checksum hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +"checksum http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +"checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +"checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +"checksum hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" +"checksum hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" +"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +"checksum indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +"checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +"checksum itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +"checksum jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +"checksum js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)" = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)" = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" +"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +"checksum mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +"checksum mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +"checksum mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)" = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +"checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +"checksum pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +"checksum pin-project-internal 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +"checksum pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" +"checksum pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +"checksum proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +"checksum proc-macro-nested 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +"checksum proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +"checksum quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" +"checksum ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)" = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" +"checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +"checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" +"checksum sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +"checksum serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +"checksum serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +"checksum syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)" = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +"checksum tinyvec 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +"checksum tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" +"checksum tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" +"checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +"checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +"checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +"checksum untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +"checksum url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +"checksum version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +"checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +"checksum wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" +"checksum wasm-bindgen-backend 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" +"checksum wasm-bindgen-futures 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dba48d66049d2a6cc8488702e7259ab7afc9043ad0dc5448444f46f2a453b362" +"checksum wasm-bindgen-macro 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" +"checksum wasm-bindgen-macro-support 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" +"checksum wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" +"checksum web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)" = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" +"checksum webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +"checksum webpki-roots 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum zstd 0.5.3+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01b32eaf771efa709e8308605bbf9319bf485dc1503179ec0469b611937c0cd8" +"checksum zstd-safe 2.0.5+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cfb642e0d27f64729a639c52db457e0ae906e7bc6f5fe8f5c453230400f1055" +"checksum zstd-sys 1.4.17+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b89249644df056b522696b1bb9e7c18c87e8ffa3e2f0dc3b0155875d6498f01b" diff --git a/taskcluster/docker/image_builder/build-image/Cargo.toml b/taskcluster/docker/image_builder/build-image/Cargo.toml new file mode 100644 index 0000000000..be42fe1e34 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/Cargo.toml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[package] +name = "build-image" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version= "0.10.4", features = ["rustls-tls", "blocking"], default-features = false} +zstd = "0.5.1" +url = "2.1.1" +anyhow = "1.0.27" +serde = { version = "1.0.105", features = ["derive"]} +serde_json = "1.0.50" +envy = "0.4.1" +fs_extra = "1.1" + +[workspace] diff --git a/taskcluster/docker/image_builder/build-image/src/config.rs b/taskcluster/docker/image_builder/build-image/src/config.rs new file mode 100644 index 0000000000..94c1d55a10 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/config.rs @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use serde::de::Error; +use serde::Deserialize; +use std::collections::HashMap; + +fn default_image_name() -> String { + "mozilla.org/taskgraph/default-image:latest".into() +} +fn default_zstd_level() -> i32 { + 3 +} + +fn from_json<'de, D, T>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, + T: serde::de::DeserializeOwned, +{ + let value: String = serde::Deserialize::deserialize(deserializer)?; + serde_json::from_str(&value).map_err(|err| { + D::Error::invalid_value(serde::de::Unexpected::Str(&value), &&*err.to_string()) + }) +} + +#[derive(Deserialize, Debug, PartialEq, Eq)] +pub struct Config { + pub context_task_id: String, + pub context_path: String, + pub parent_task_id: Option, + #[serde(default = "default_image_name")] + pub image_name: String, + #[serde(default = "default_zstd_level")] + pub docker_image_zstd_level: i32, + #[serde(default)] + pub debug: bool, + #[serde(default, deserialize_with = "from_json")] + pub docker_build_args: HashMap, +} + +impl Config { + pub fn from_env() -> Result { + Ok(envy::from_env()?) + } +} + +#[cfg(test)] +mod test { + use anyhow::Result; + + #[test] + fn test() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: Default::default() + } + ); + Ok(()) + } + + #[test] + fn test_docker_build_args() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ( + "DOCKER_BUILD_ARGS".into(), + serde_json::json! ({ + "test": "Value", + }) + .to_string(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: [("test".to_string(), "Value".to_string())] + .iter() + .cloned() + .collect(), + } + ); + Ok(()) + } +} diff --git a/taskcluster/docker/image_builder/build-image/src/main.rs b/taskcluster/docker/image_builder/build-image/src/main.rs new file mode 100644 index 0000000000..997617c84e --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/main.rs @@ -0,0 +1,182 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#![forbid(unsafe_code)] + +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use anyhow::{ensure, Context, Result}; +use fs_extra::dir::{move_dir, CopyOptions}; +use serde::Deserialize; + +mod config; +mod taskcluster; + +use config::Config; + +fn log_step(msg: &str) { + println!("[build-image] {}", msg); +} + +fn read_image_digest(path: &str) -> Result { + let output = Command::new("/kaniko/skopeo") + .arg("inspect") + .arg(format!("docker-archive:{}", path)) + .stdout(std::process::Stdio::piped()) + .spawn()? + .wait_with_output()?; + ensure!(output.status.success(), "Could not inspect parent image."); + + #[derive(Deserialize, Debug)] + #[serde(rename_all = "PascalCase")] + struct ImageInfo { + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tag: Option, + digest: String, + // ... + } + + let image_info: ImageInfo = serde_json::from_slice(&output.stdout) + .with_context(|| format!("Could parse image info from {:?}", path))?; + Ok(image_info.digest) +} + +fn download_parent_image( + cluster: &taskcluster::TaskCluster, + task_id: &str, + dest: &str, +) -> Result { + zstd::stream::copy_decode( + cluster.stream_artifact(&task_id, "public/image.tar.zst")?, + std::fs::File::create(dest)?, + ) + .context("Could not download parent image.")?; + + read_image_digest(dest) +} + +fn build_image( + context_path: &str, + dest: &str, + debug: bool, + build_args: HashMap, +) -> Result<()> { + let mut command = Command::new("/kaniko/executor"); + command + .stderr(std::process::Stdio::inherit()) + .args(&["--context", &format!("tar://{}", context_path)]) + .args(&["--destination", "image"]) + .args(&["--dockerfile", "Dockerfile"]) + .arg("--no-push") + .args(&["--cache-dir", "/workspace/cache"]) + .arg("--single-snapshot") + // FIXME: Generating reproducible layers currently causes OOM. + // .arg("--reproducible") + .arg("--whitelist-var-run=false") + .args(&["--tarPath", dest]); + if debug { + command.args(&["-v", "debug"]); + } + for (key, value) in build_args { + command.args(&["--build-arg", &format!("{}={}", key, value)]); + } + let status = command.status()?; + ensure!(status.success(), "Could not build image."); + Ok(()) +} + +fn repack_image(source: &str, dest: &str, image_name: &str) -> Result<()> { + let status = Command::new("/kaniko/skopeo") + .arg("copy") + .arg(format!("docker-archive:{}", source)) + .arg(format!("docker-archive:{}:{}", dest, image_name)) + .stderr(std::process::Stdio::inherit()) + .status()?; + ensure!(status.success(), "Could repack image."); + Ok(()) +} + +fn main() -> Result<()> { + // Kaniko expects everything to be in /kaniko, so if not running from there, move + // everything there. + if let Some(path) = std::env::current_exe()?.parent() { + if path != Path::new("/kaniko") { + let mut options = CopyOptions::new(); + options.copy_inside = true; + move_dir(path, "/kaniko", &options)?; + } + } + + let config = Config::from_env().context("Could not parse environment variables.")?; + + let cluster = taskcluster::TaskCluster::from_env()?; + + let mut build_args = config.docker_build_args; + + build_args.insert("TASKCLUSTER_ROOT_URL".into(), cluster.root_url()); + + log_step("Downloading context."); + + std::io::copy( + &mut cluster.stream_artifact(&config.context_task_id, &config.context_path)?, + &mut std::fs::File::create("/workspace/context.tar.gz")?, + ) + .context("Could not download image context.")?; + + if let Some(parent_task_id) = config.parent_task_id { + log_step("Downloading image."); + let digest = download_parent_image(&cluster, &parent_task_id, "/workspace/parent.tar")?; + + log_step(&format!("Parent image digest {}", &digest)); + std::fs::create_dir_all("/workspace/cache")?; + std::fs::rename( + "/workspace/parent.tar", + format!("/workspace/cache/{}", digest), + )?; + + build_args.insert( + "DOCKER_IMAGE_PARENT".into(), + format!("parent:latest@{}", digest), + ); + } + + log_step("Building image."); + build_image( + "/workspace/context.tar.gz", + "/workspace/image-pre.tar", + config.debug, + build_args, + )?; + log_step("Repacking image."); + repack_image( + "/workspace/image-pre.tar", + "/workspace/image.tar", + &config.image_name, + )?; + + log_step("Compressing image."); + compress_file( + "/workspace/image.tar", + "/workspace/image.tar.zst", + config.docker_image_zstd_level, + )?; + + Ok(()) +} + +fn compress_file( + source: impl AsRef, + dest: impl AsRef, + zstd_level: i32, +) -> Result<()> { + Ok(zstd::stream::copy_encode( + std::fs::File::open(source)?, + std::fs::File::create(dest)?, + zstd_level, + )?) +} diff --git a/taskcluster/docker/image_builder/build-image/src/taskcluster.rs b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs new file mode 100644 index 0000000000..3b39d669f0 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::{Context, Result}; + +pub struct TaskCluster { + root_url: url::Url, + client: reqwest::blocking::Client, +} + +impl TaskCluster { + pub fn from_env() -> Result { + std::env::var("TASKCLUSTER_ROOT_URL") + .context("TASKCLUSTER_ROOT_URL not set.") + .and_then(|var| var.parse().context("Couldn't parse TASKCLUSTER_ROOT_URL.")) + .map(|root_url| TaskCluster { + root_url, + client: reqwest::blocking::Client::new(), + }) + } + + /// Return the root URL as suitable for passing to other processes. + /// + /// In particular, any trailing slashes are removed. + pub fn root_url(&self) -> String { + self.root_url.as_str().trim_end_matches("/").to_string() + } + + pub fn task_artifact_url(&self, task_id: &str, path: &str) -> url::Url { + let mut url = self.root_url.clone(); + url.set_path(&format!("api/queue/v1/task/{}/artifacts/{}", task_id, path)); + url + } + + pub fn stream_artifact(&self, task_id: &str, path: &str) -> Result { + let url = self.task_artifact_url(task_id, path); + Ok(self.client.get(url).send()?.error_for_status()?) + } +} + +#[cfg(test)] +mod test { + #[test] + fn test_url() { + let cluster = super::TaskCluster { + root_url: url::Url::parse("http://taskcluster.example").unwrap(), + client: reqwest::blocking::Client::new(), + }; + assert_eq!( + cluster.task_artifact_url("QzDLgP4YRwanIvgPt6ClfA","public/docker-contexts/decision.tar.gz"), + url::Url::parse("http://taskcluster.example/api/queue/v1/task/QzDLgP4YRwanIvgPt6ClfA/artifacts/public/docker-contexts/decision.tar.gz").unwrap(), + ); + } +} diff --git a/taskcluster/docker/image_builder/policy.json b/taskcluster/docker/image_builder/policy.json new file mode 100644 index 0000000000..c9a9b225cf --- /dev/null +++ b/taskcluster/docker/image_builder/policy.json @@ -0,0 +1,11 @@ +{ + "default": [{ "type": "reject" }], + "transports": { + "docker-archive": { + "": [{ "type": "insecureAcceptAnything" }] + }, + "dir": { + "": [{ "type": "insecureAcceptAnything" }] + } + } +} diff --git a/taskcluster/docker/index-task/.eslintrc.js b/taskcluster/docker/index-task/.eslintrc.js new file mode 100644 index 0000000000..ff84d9cb69 --- /dev/null +++ b/taskcluster/docker/index-task/.eslintrc.js @@ -0,0 +1,11 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + node: true, + }, +}; diff --git a/taskcluster/docker/index-task/Dockerfile b/taskcluster/docker/index-task/Dockerfile new file mode 100644 index 0000000000..493eda4e20 --- /dev/null +++ b/taskcluster/docker/index-task/Dockerfile @@ -0,0 +1,11 @@ +FROM node:10-alpine + +ENV NODE_ENV production +RUN mkdir /app +ADD insert-indexes.js /app/ +ADD package.json /app/ +ADD yarn.lock /app/ +WORKDIR /app +RUN yarn --frozen-lockfile && yarn cache clean + +ENTRYPOINT ["node"] diff --git a/taskcluster/docker/index-task/README b/taskcluster/docker/index-task/README new file mode 100644 index 0000000000..9ec00e7897 --- /dev/null +++ b/taskcluster/docker/index-task/README @@ -0,0 +1,36 @@ +Index-Image +=========== + +This image is designed to be used for indexing other tasks. It takes a task +definition as follows: +```js +{ + ..., + scopes: [ + 'index:insert-task:my-index.namespace', + 'index:insert-task:...', + ], + payload: { + image: '...', + env: { + TARGET_TASKID: '', + }, + command: [ + 'insert-indexes.js', + 'my-index.namespace.one', + 'my-index.namespace.two', + '....', + ], + features: { + taskclusterProxy: true, + }, + maxRunTime: 600, + }, +} +``` + +As can be seen the `taskId` to be indexed is given by the environment variable +`TARGET_TASKID` and the `command` arguments specifies namespaces that it must +be index under. It is **important** to also include scopes on the form +`index:insert-task:<...>` for all namespaces `<...>` given as `command` +arguments. diff --git a/taskcluster/docker/index-task/insert-indexes.js b/taskcluster/docker/index-task/insert-indexes.js new file mode 100644 index 0000000000..39ff45fd9c --- /dev/null +++ b/taskcluster/docker/index-task/insert-indexes.js @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +let taskcluster = require("taskcluster-client"); + +// Create instance of index client +let index = new taskcluster.Index({ + delayFactor: 750, // Good solid delay for background process + retries: 8, // A few extra retries for robustness + rootUrl: + process.env.TASKCLUSTER_PROXY_URL || process.env.TASKCLUSTER_ROOT_URL, +}); + +// Create queue instance for fetching taskId +let queue = new taskcluster.Queue({ + delayFactor: 750, // Good solid delay for background process + retries: 8, // A few extra retries for robustness + rootUrl: + process.env.TASKCLUSTER_PROXY_URL || process.env.TASKCLUSTER_ROOT_URL, +}); + +// Load input +let taskId = process.env.TARGET_TASKID; +let rank = parseInt(process.env.INDEX_RANK, 10); +let namespaces = process.argv.slice(2); + +// Validate input +if (!taskId) { + console.log("Expected target task as environment variable: TARGET_TASKID"); + process.exit(1); +} + +if (isNaN(rank)) { + console.log("Expected index rank as environment variable: INDEX_RANK"); + process.exit(1); +} + +// Fetch task definition to get expiration and then insert into index +queue + .task(taskId) + .then(task => task.expires) + .then(expires => { + return Promise.all( + namespaces.map(namespace => { + console.log( + "Inserting %s into index (rank %d) under: %s", + taskId, + rank, + namespace + ); + return index.insertTask(namespace, { + taskId, + rank, + data: {}, + expires, + }); + }) + ); + }) + .then(() => { + console.log("indexing successfully completed."); + process.exit(0); + }) + .catch(err => { + console.log("Error:\n%s", err); + if (err.stack) { + console.log("Stack:\n%s", err.stack); + } + console.log("Properties:\n%j", err); + throw err; + }) + .catch(() => process.exit(1)); diff --git a/taskcluster/docker/index-task/package.json b/taskcluster/docker/index-task/package.json new file mode 100644 index 0000000000..14c035123c --- /dev/null +++ b/taskcluster/docker/index-task/package.json @@ -0,0 +1,12 @@ +{ + "private": true, + "scripts": { + "start": "node index.js" + }, + "dependencies": { + "taskcluster-client": "^12.2.0" + }, + "engines": { + "node": "10" + } +} diff --git a/taskcluster/docker/index-task/yarn.lock b/taskcluster/docker/index-task/yarn.lock new file mode 100644 index 0000000000..326936d8a6 --- /dev/null +++ b/taskcluster/docker/index-task/yarn.lock @@ -0,0 +1,326 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +amqplib@^0.5.1: + version "0.5.3" + resolved "https://registry.yarnpkg.com/amqplib/-/amqplib-0.5.3.tgz#7ccfc85d12ee7cd3c6dc861bb07f0648ec3d7193" + integrity sha512-ZOdUhMxcF+u62rPI+hMtU1NBXSDFQ3eCJJrenamtdQ7YYwh7RZJHOIM1gonVbZ5PyVdYH4xqBPje9OYqk7fnqw== + dependencies: + bitsyntax "~0.1.0" + bluebird "^3.5.2" + buffer-more-ints "~1.0.0" + readable-stream "1.x >=1.1.9" + safe-buffer "~5.1.2" + url-parse "~1.4.3" + +asap@~2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" + integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +bitsyntax@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/bitsyntax/-/bitsyntax-0.1.0.tgz#b0c59acef03505de5a2ed62a2f763c56ae1d6205" + integrity sha512-ikAdCnrloKmFOugAfxWws89/fPc+nw0OOG1IzIE72uSOg/A3cYptKCjSUhDTuj7fhsJtzkzlv7l3b8PzRHLN0Q== + dependencies: + buffer-more-ints "~1.0.0" + debug "~2.6.9" + safe-buffer "~5.1.2" + +bluebird@^3.5.2: + version "3.5.3" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.3.tgz#7d01c6f9616c9a51ab0f8c549a79dfe6ec33efa7" + integrity sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw== + +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + integrity sha1-T4owBctKfjiJ90kDD9JbluAdLjE= + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + integrity sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw== + dependencies: + hoek "4.x.x" + +buffer-more-ints@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz#ef4f8e2dddbad429ed3828a9c55d44f05c611422" + integrity sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg== + +combined-stream@^1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828" + integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w== + dependencies: + delayed-stream "~1.0.0" + +component-emitter@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + integrity sha1-E3kY1teCg/ffemt8WmPhQOaUJeY= + +cookiejar@^2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c" + integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA== + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +cryptiles@3.x.x: + version "3.1.4" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.4.tgz#769a68c95612b56faadfcebf57ac86479cbe8322" + integrity sha512-8I1sgZHfVwcSOY6mSGpVU3lw/GSIZvusg8dD2+OGehCJpOhQRLNcH0qb9upQnOH4XhgxxFJSg6E2kx95deb1Tw== + dependencies: + boom "5.x.x" + +debug@^3.1.0: + version "3.2.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== + dependencies: + ms "^2.1.1" + +debug@~2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +extend@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +form-data@^2.3.1: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +formidable@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.1.tgz#70fb7ca0290ee6ff961090415f4b3df3d2082659" + integrity sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg== + +hawk@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + integrity sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ== + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + +hoek@4.x.x: + version "4.2.1" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" + integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== + +inherits@~2.0.1, inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= + +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +lodash@^4.17.4: + version "4.17.11" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" + integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== + +methods@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + +mime-db@~1.37.0: + version "1.37.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" + integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== + +mime-types@^2.1.12: + version "2.1.21" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" + integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== + dependencies: + mime-db "~1.37.0" + +mime@^1.4.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== + +promise@^8.0.1: + version "8.0.2" + resolved "https://registry.yarnpkg.com/promise/-/promise-8.0.2.tgz#9dcd0672192c589477d56891271bdc27547ae9f0" + integrity sha512-EIyzM39FpVOMbqgzEHhxdrEhtOSDOtjMZQ0M6iVfCE+kWNgCkAyOdnuCWqfmflylftfadU6FkiMgHZA2kUzwRw== + dependencies: + asap "~2.0.6" + +qs@^6.5.1: + version "6.6.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.6.0.tgz#a99c0f69a8d26bf7ef012f871cdabb0aee4424c2" + integrity sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA== + +querystringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.0.tgz#7ded8dfbf7879dcc60d0a644ac6754b283ad17ef" + integrity sha512-sluvZZ1YiTLD5jsqZcDmFyV2EwToyXZBfpoVOmktMmW+VEnhgakFHnasVph65fOjGPTWN0Nw3+XQaSeMayr0kg== + +"readable-stream@1.x >=1.1.9": + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= + +safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +slugid@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/slugid/-/slugid-1.1.0.tgz#e09f00899c09f5a7058edc36dd49f046fd50a82a" + integrity sha1-4J8AiZwJ9acFjtw23UnwRv1QqCo= + dependencies: + uuid "^2.0.1" + +sntp@2.x.x: + version "2.1.0" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.1.0.tgz#2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8" + integrity sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg== + dependencies: + hoek "4.x.x" + +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +superagent@~3.8.1: + version "3.8.3" + resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128" + integrity sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA== + dependencies: + component-emitter "^1.2.0" + cookiejar "^2.1.0" + debug "^3.1.0" + extend "^3.0.0" + form-data "^2.3.1" + formidable "^1.2.0" + methods "^1.1.1" + mime "^1.4.1" + qs "^6.5.1" + readable-stream "^2.3.5" + +taskcluster-client@^12.2.0: + version "12.2.0" + resolved "https://registry.yarnpkg.com/taskcluster-client/-/taskcluster-client-12.2.0.tgz#423aee3b17566d14f8ad23e4e47532265a74fb89" + integrity sha512-2Fu5ICS2663kC2t8ymJYzRDnipj3DsCK//b+H/83RjJvC6cWZ0akKzq0ySvPlNA6ic2UcL4I03bJTCJYBX1dqg== + dependencies: + amqplib "^0.5.1" + debug "^3.1.0" + hawk "^6.0.2" + lodash "^4.17.4" + promise "^8.0.1" + slugid "^1.1.0" + superagent "~3.8.1" + taskcluster-lib-urls "^10.0.0" + +taskcluster-lib-urls@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/taskcluster-lib-urls/-/taskcluster-lib-urls-10.1.1.tgz#67d5b9449b947e5234eafdd15c46267dde29bf74" + integrity sha512-tdrK++rCX73FMXk/cXwS6RLTjA3pX8hJlxg1ECLs3L3llCOPMNhQ4wi6lb6yMgHc/s5on/Edj6AlAH7gkxzgPg== + +url-parse@~1.4.3: + version "1.4.4" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.4.tgz#cac1556e95faa0303691fec5cf9d5a1bc34648f8" + integrity sha512-/92DTTorg4JjktLNLe6GPS2/RvAd/RGr6LuktmWSMLEOa6rjnlrFXNgSbSmkNvCoL2T028A0a1JaJLzRMlFoHg== + dependencies: + querystringify "^2.0.0" + requires-port "^1.0.0" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +uuid@^2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a" + integrity sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho= diff --git a/taskcluster/docker/lint/Dockerfile b/taskcluster/docker/lint/Dockerfile new file mode 100644 index 0000000000..e34d9730d7 --- /dev/null +++ b/taskcluster/docker/lint/Dockerfile @@ -0,0 +1,36 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Andrew Halberstadt + +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts + +# We do want to install recommended packages. +RUN sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +RUN mkdir /build +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py + +# %include taskcluster/docker/recipes/install-node.sh +ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh + +ADD system-setup.sh /tmp/system-setup.sh +# %include tools/lint/eslint/manifest.tt +ADD topsrcdir/tools/lint/eslint/manifest.tt /tmp/eslint.tt +# %include tools/lint/eslint/eslint-plugin-mozilla/manifest.tt +ADD topsrcdir/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt /tmp/eslint-plugin-mozilla.tt +# %include tools/lint/spell/codespell_requirements.txt +ADD topsrcdir/tools/lint/spell/codespell_requirements.txt /tmp/codespell_requirements.txt +# %include tools/lint/tox/tox_requirements.txt +ADD topsrcdir/tools/lint/tox/tox_requirements.txt /tmp/tox_requirements.txt +RUN bash /tmp/system-setup.sh + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/lint/system-setup.sh b/taskcluster/docker/lint/system-setup.sh new file mode 100644 index 0000000000..33516d890a --- /dev/null +++ b/taskcluster/docker/lint/system-setup.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() +apt_packages+=('curl') +apt_packages+=('iproute2') +apt_packages+=('locales') +apt_packages+=('graphviz') +apt_packages+=('python3-pip') +apt_packages+=('python-is-python3') +apt_packages+=('shellcheck') +apt_packages+=('sudo') +apt_packages+=('wget') +apt_packages+=('unzip') +apt_packages+=('tar') +apt_packages+=('zstd') + +apt-get update +apt-get install "${apt_packages[@]}" + +# Without this we get spurious "LC_ALL: cannot change locale (en_US.UTF-8)" errors, +# and python scripts raise UnicodeEncodeError when trying to print unicode characters. +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +su -c 'git config --global user.email "worker@mozilla.test"' worker +su -c 'git config --global user.name "worker"' worker + +tooltool_fetch() { + cat >manifest.tt + /build/tooltool.py fetch + rm manifest.tt +} + +cd /build + +### +# ESLint Setup +### + +# install node +# shellcheck disable=SC1091 +. install-node.sh + +npm install -g yarn@1.22.18 + +/build/tooltool.py fetch -m /tmp/eslint.tt +mv /build/node_modules /build/node_modules_eslint +/build/tooltool.py fetch -m /tmp/eslint-plugin-mozilla.tt +mv /build/node_modules /build/node_modules_eslint-plugin-mozilla + +### +# fzf setup +### + +tooltool_fetch < + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace + +RUN dpkg --add-architecture amd64 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + git \ + gzip \ + openssh-client \ + unzip \ + zip + +COPY known_hosts /etc/ssh/ssh_known_hosts diff --git a/taskcluster/docker/partner-repack/known_hosts b/taskcluster/docker/partner-repack/known_hosts new file mode 100644 index 0000000000..f4c560e0e7 --- /dev/null +++ b/taskcluster/docker/partner-repack/known_hosts @@ -0,0 +1,3 @@ +github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl +github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= +github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= diff --git a/taskcluster/docker/periodic-updates/.eslintrc.js b/taskcluster/docker/periodic-updates/.eslintrc.js new file mode 100644 index 0000000000..ecaf80d033 --- /dev/null +++ b/taskcluster/docker/periodic-updates/.eslintrc.js @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + globals: { + // JS files in this folder are commonly xpcshell scripts where |arguments| + // is defined in the global scope. + arguments: false, + }, + rules: { + // Enforce return statements in callbacks of array methods. + "array-callback-return": "error", + + // Verify calls of super() in constructors. + "constructor-super": "error", + + // Require default case in switch statements. + "default-case": "error", + + // Disallow use of alert(), confirm(), and prompt(). + "no-alert": "error", + + // Disallow likely erroneous `switch` scoped lexical declarations in + // case/default clauses. + "no-case-declarations": "error", + + // Disallow use of the console API. + "no-console": "error", + + // Disallow constant expressions in conditions (except for loops). + "no-constant-condition": ["error", { checkLoops: false }], + + // Disallow extending of native objects. + "no-extend-native": "error", + + // Disallow use of assignment in return statement. + "no-return-assign": ["error", "always"], + + // Disallow template literal placeholder syntax in regular strings. + "no-template-curly-in-string": "error", + + // Disallow unmodified loop conditions. + "no-unmodified-loop-condition": "error", + + // No expressions where a statement is expected + "no-unused-expressions": "error", + + // Disallow unnecessary escape usage in strings and regular expressions. + "no-useless-escape": "error", + + // Require "use strict" to be defined globally in the script. + strict: ["error", "global"], + + // Disallow Yoda conditions. + yoda: ["error", "never"], + }, +}; diff --git a/taskcluster/docker/periodic-updates/Dockerfile b/taskcluster/docker/periodic-updates/Dockerfile new file mode 100644 index 0000000000..24cabe02b5 --- /dev/null +++ b/taskcluster/docker/periodic-updates/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Julien Cristau + +ADD setup.sh /setup/setup.sh + +RUN cd /setup && ./setup.sh + +COPY runme.sh / +COPY scripts/* /home/worker/scripts/ + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/periodic-updates/README.md b/taskcluster/docker/periodic-updates/README.md new file mode 100644 index 0000000000..d21c0c3656 --- /dev/null +++ b/taskcluster/docker/periodic-updates/README.md @@ -0,0 +1,96 @@ + +==Periodic File Updates== + +This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklisting, and +will produce a diff for each necessary to update the in-tree files. + +If given a conduit API token, it will also use the arcanist client to submit the commits for review. + + +==Quick Start== + +```sh +docker build -t hsts-local --no-cache --rm . + +docker run -e DO_HSTS=1 -e DO_HPKP=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local +``` + +HSTS checks will only be run if the `DO_HSTS` environment variable is set. +Likewise for `DO_HPKP` and the HPKP checks. Environment variables are used +rather than command line arguments to make constructing taskcluster tasks +easier. + +To prevent a full build when landing with Phabricator, set the `DONTBUILD` +environment variable. + +==Background== + +These scripts have been moved from +`https://hg.mozilla.org/build/tools/scripts/periodic_file_updates/` and +`security/manager/tools/` in the main repos. + +==HSTS Checks== + +`scripts/getHSTSPreloadList.js` will examine the current contents of +nsSTSPreloadList.inc from whichever `BRANCH` is specified, add in the mandatory +hosts, and those from the Chromium source, and check them all to see if their +SSL configuration is valid, and whether or not they have the +Strict-Transport-Security header set with an appropriate `max-age`. + +This javascript has been modified to use async calls to improve performance. + +==HPKP Checks== + +`scripts/genHPKPStaticPins.js` will ensure the list of pinned public keys are +up to date. + +==Example Taskcluster Task== + +https://firefox-ci-tc.services.mozilla.com/tasks/create/ + +```yaml +provisionerId: aws-provisioner-v1 +workerType: gecko-1-b-linux +retries: 0 +created: '2018-02-07T14:45:57.347Z' +deadline: '2018-02-07T17:45:57.348Z' +expires: '2019-02-07T17:45:57.348Z' +scopes: [] +payload: + image: srfraser/hsts1 + maxRunTime: 1800 + artifacts: + public/build/nsSTSPreloadList.diff: + path: /home/worker/artifacts/nsSTSPreloadList.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + public/build/StaticHPKPins.h.diff: + path: /home/worker/artifacts/StaticHPKPins.h.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + public/build/blocklist.diff: + path: /home/worker/artifacts/blocklist.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + env: + DO_HSTS: 1 + DO_HPKP: 1 + PRODUCT: firefox + BRANCH: mozilla-central + USE_MOZILLA_CENTRAL: 1 + REVIEWERS: catlee +metadata: + name: Periodic updates testing + description: Produce diffs for HSTS and HPKP in-tree files. + owner: sfraser@mozilla.com + source: 'https://firefox-ci-tc.services.mozilla.com/tasks/create' +tags: {} +extra: + treeherder: + jobKind: test + machine: + platform: linux64 + tier: 1 + symbol: 'hsts' + +``` diff --git a/taskcluster/docker/periodic-updates/runme.sh b/taskcluster/docker/periodic-updates/runme.sh new file mode 100755 index 0000000000..b9f6eb31c3 --- /dev/null +++ b/taskcluster/docker/periodic-updates/runme.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set -xe + +# Things to be set by task definition. +# --pinset --hsts --hpkp +# -b branch +# --use-mozilla-central +# -p firefox +# Artifact directory +# Artifact names. + + +test "${BRANCH}" +test "${PRODUCT}" + +PARAMS="" + +if [ -n "${USE_MOZILLA_CENTRAL}" ] +then + PARAMS="${PARAMS} --use-mozilla-central" +fi + +# TODO change these, so that they're run if the artifact location is specified? +if [ -n "${DO_HSTS}" ] +then + PARAMS="${PARAMS} --hsts" +fi + +if [ -n "${DO_HPKP}" ] +then + PARAMS="${PARAMS} --hpkp" +fi + +if [ -n "${DO_REMOTE_SETTINGS}" ] +then + PARAMS="${PARAMS} --remote-settings" +fi + +if [ -n "${DO_SUFFIX_LIST}" ] +then + PARAMS="${PARAMS} --suffix-list" +fi + +if [ -n "${DO_MOBILE_EXPERIMENTS}" ] +then + PARAMS="${PARAMS} --mobile-experiments" +fi + +if [ -n "${DONTBUILD}" ] +then + PARAMS="${PARAMS} -d" +fi + + +export ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + +# Get Arcanist API token + +if [ -n "${TASK_ID}" ] +then + curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}') +fi +if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster +then + set +x # Don't echo these + secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. +then + TOKEN="${ARC_TOKEN}" +fi + +if [ -n "${TOKEN}" ] +then + cat >"${HOME}/.arcrc" < " + + "" + ); +} + +var { NetUtil } = ChromeUtils.importESModule( + "resource://gre/modules/NetUtil.sys.mjs" +); +var { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +var gCertDB = Cc["@mozilla.org/security/x509certdb;1"].getService( + Ci.nsIX509CertDB +); + +const SHA256_PREFIX = "sha256/"; +const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_"; + +// Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable) +const PINNING_MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 14; + +const FILE_HEADER = + "/* This Source Code Form is subject to the terms of the Mozilla Public\n" + + " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" + + " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" + + "\n" + + "/*****************************************************************************/\n" + + "/* This is an automatically generated file. If you're not */\n" + + "/* PublicKeyPinningService.cpp, you shouldn't be #including it. */\n" + + "/*****************************************************************************/\n" + + "#include " + + "\n"; + +const DOMAINHEADER = + "/* Domainlist */\n" + + "struct TransportSecurityPreload {\n" + + " // See bug 1338873 about making these fields const.\n" + + " const char* mHost;\n" + + " bool mIncludeSubdomains;\n" + + " bool mTestMode;\n" + + " bool mIsMoz;\n" + + " int32_t mId;\n" + + " const StaticFingerprints* pinset;\n" + + "};\n\n"; + +const PINSETDEF = + "/* Pinsets are each an ordered list by the actual value of the fingerprint */\n" + + "struct StaticFingerprints {\n" + + " // See bug 1338873 about making these fields const.\n" + + " size_t size;\n" + + " const char* const* data;\n" + + "};\n\n"; + +// Command-line arguments +var gStaticPins = parseJson(arguments[0]); + +// Open the output file. +var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); +file.initWithPath(arguments[1]); +var gFileOutputStream = FileUtils.openSafeFileOutputStream(file); + +function writeString(string) { + gFileOutputStream.write(string, string.length); +} + +function readFileToString(filename) { + let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(filename); + let stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + stream.init(file, -1, 0, 0); + let buf = NetUtil.readInputStreamToString(stream, stream.available()); + return buf; +} + +function stripComments(buf) { + let lines = buf.split("\n"); + let entryRegex = /^\s*\/\//; + let data = ""; + for (let i = 0; i < lines.length; ++i) { + let match = entryRegex.exec(lines[i]); + if (!match) { + data = data + lines[i]; + } + } + return data; +} + +function download(filename) { + let req = new XMLHttpRequest(); + req.open("GET", filename, false); // doing the request synchronously + try { + req.send(); + } catch (e) { + throw new Error(`ERROR: problem downloading '${filename}': ${e}`); + } + + if (req.status != 200) { + throw new Error( + "ERROR: problem downloading '" + filename + "': status " + req.status + ); + } + + let resultDecoded; + try { + resultDecoded = atob(req.responseText); + } catch (e) { + throw new Error( + "ERROR: could not decode data as base64 from '" + filename + "': " + e + ); + } + return resultDecoded; +} + +function downloadAsJson(filename) { + // we have to filter out '//' comments, while not mangling the json + let result = download(filename).replace(/^(\s*)?\/\/[^\n]*\n/gm, ""); + let data = null; + try { + data = JSON.parse(result); + } catch (e) { + throw new Error( + "ERROR: could not parse data from '" + filename + "': " + e + ); + } + return data; +} + +// Returns a Subject Public Key Digest from the given pem, if it exists. +function getSKDFromPem(pem) { + let cert = gCertDB.constructX509FromBase64(pem, pem.length); + return cert.sha256SubjectPublicKeyInfoDigest; +} + +/** + * Hashes |input| using the SHA-256 algorithm in the following manner: + * btoa(sha256(atob(input))) + * + * @param {string} input Base64 string to decode and return the hash of. + * @returns {string} Base64 encoded SHA-256 hash. + */ +function sha256Base64(input) { + let decodedValue; + try { + decodedValue = atob(input); + } catch (e) { + throw new Error(`ERROR: could not decode as base64: '${input}': ${e}`); + } + + // Convert |decodedValue| to an array so that it can be hashed by the + // nsICryptoHash instance below. + // In most cases across the code base, convertToByteArray() of + // nsIScriptableUnicodeConverter is used to do this, but the method doesn't + // seem to work here. + let data = []; + for (let i = 0; i < decodedValue.length; i++) { + data[i] = decodedValue.charCodeAt(i); + } + + let hasher = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + // true is passed so that the hasher returns a Base64 encoded string. + return hasher.finish(true); +} + +// Downloads the static certs file and tries to map Google Chrome nicknames +// to Mozilla nicknames, as well as storing any hashes for pins for which we +// don't have root PEMs. Each entry consists of a line containing the name of +// the pin followed either by a hash in the format "sha256/" + base64(hash), +// a PEM encoded public key, or a PEM encoded certificate. +// For certificates that we have in our database, +// return a map of Google's nickname to ours. For ones that aren't return a +// map of Google's nickname to SHA-256 values. This code is modeled after agl's +// https://github.com/agl/transport-security-state-generate, which doesn't +// live in the Chromium repo because go is not an official language in +// Chromium. +// For all of the entries in this file: +// - If the entry has a hash format, find the Mozilla pin name (cert nickname) +// and stick the hash into certSKDToName +// - If the entry has a PEM format, parse the PEM, find the Mozilla pin name +// and stick the hash in certSKDToName +// We MUST be able to find a corresponding cert nickname for the Chrome names, +// otherwise we skip all pinsets referring to that Chrome name. +function downloadAndParseChromeCerts(filename, certNameToSKD, certSKDToName) { + // Prefixes that we care about. + const BEGIN_CERT = "-----BEGIN CERTIFICATE-----"; + const END_CERT = "-----END CERTIFICATE-----"; + const BEGIN_PUB_KEY = "-----BEGIN PUBLIC KEY-----"; + const END_PUB_KEY = "-----END PUBLIC KEY-----"; + + // Parsing states. + const PRE_NAME = 0; + const POST_NAME = 1; + const IN_CERT = 2; + const IN_PUB_KEY = 3; + let state = PRE_NAME; + + let lines = download(filename).split("\n"); + let pemCert = ""; + let pemPubKey = ""; + let hash = ""; + let chromeNameToHash = {}; + let chromeNameToMozName = {}; + let chromeName; + for (let line of lines) { + // Skip comments and newlines. + if (!line.length || line[0] == "#") { + continue; + } + switch (state) { + case PRE_NAME: + chromeName = line; + state = POST_NAME; + break; + case POST_NAME: + if (line.startsWith(SHA256_PREFIX)) { + hash = line.substring(SHA256_PREFIX.length); + chromeNameToHash[chromeName] = hash; + certNameToSKD[chromeName] = hash; + certSKDToName[hash] = chromeName; + state = PRE_NAME; + } else if (line.startsWith(BEGIN_CERT)) { + state = IN_CERT; + } else if (line.startsWith(BEGIN_PUB_KEY)) { + state = IN_PUB_KEY; + } else if ( + chromeName == "PinsListTimestamp" && + line.match(/^[0-9]+$/) + ) { + // If the name of this entry is "PinsListTimestamp", this line should + // be the pins list timestamp. It should consist solely of digits. + // Ignore it and expect other entries to come. + state = PRE_NAME; + } else { + throw new Error( + "ERROR: couldn't parse Chrome certificate file line: " + line + ); + } + break; + case IN_CERT: + if (line.startsWith(END_CERT)) { + state = PRE_NAME; + hash = getSKDFromPem(pemCert); + pemCert = ""; + let mozName; + if (hash in certSKDToName) { + mozName = certSKDToName[hash]; + } else { + // Not one of our built-in certs. Prefix the name with + // GOOGLE_PIN_. + mozName = GOOGLE_PIN_PREFIX + chromeName; + dump( + "Can't find hash in builtin certs for Chrome nickname " + + chromeName + + ", inserting " + + mozName + + "\n" + ); + certSKDToName[hash] = mozName; + certNameToSKD[mozName] = hash; + } + chromeNameToMozName[chromeName] = mozName; + } else { + pemCert += line; + } + break; + case IN_PUB_KEY: + if (line.startsWith(END_PUB_KEY)) { + state = PRE_NAME; + hash = sha256Base64(pemPubKey); + pemPubKey = ""; + chromeNameToHash[chromeName] = hash; + certNameToSKD[chromeName] = hash; + certSKDToName[hash] = chromeName; + } else { + pemPubKey += line; + } + break; + default: + throw new Error( + "ERROR: couldn't parse Chrome certificate file " + line + ); + } + } + return [chromeNameToHash, chromeNameToMozName]; +} + +// We can only import pinsets from chrome if for every name in the pinset: +// - We have a hash from Chrome's static certificate file +// - We have a builtin cert +// If the pinset meets these requirements, we store a map array of pinset +// objects: +// { +// pinset_name : { +// // Array of names with entries in certNameToSKD +// sha256_hashes: [] +// } +// } +// and an array of imported pinset entries: +// { name: string, include_subdomains: boolean, test_mode: boolean, +// pins: pinset_name } +function downloadAndParseChromePins( + filename, + chromeNameToHash, + chromeNameToMozName, + certNameToSKD, + certSKDToName +) { + let chromePreloads = downloadAsJson(filename); + let chromePins = chromePreloads.pinsets; + let chromeImportedPinsets = {}; + let chromeImportedEntries = []; + + chromePins.forEach(function (pin) { + let valid = true; + let pinset = { name: pin.name, sha256_hashes: [] }; + // Translate the Chrome pinset format to ours + pin.static_spki_hashes.forEach(function (name) { + if (name in chromeNameToHash) { + let hash = chromeNameToHash[name]; + pinset.sha256_hashes.push(certSKDToName[hash]); + + // We should have already added hashes for all of these when we + // imported the certificate file. + if (!certNameToSKD[name]) { + throw new Error("ERROR: No hash for name: " + name); + } + } else if (name in chromeNameToMozName) { + pinset.sha256_hashes.push(chromeNameToMozName[name]); + } else { + dump( + "Skipping Chrome pinset " + + pinset.name + + ", couldn't find " + + "builtin " + + name + + " from cert file\n" + ); + valid = false; + } + }); + if (valid) { + chromeImportedPinsets[pinset.name] = pinset; + } + }); + + // Grab the domain entry lists. Chrome's entry format is similar to + // ours, except theirs includes a HSTS mode. + const cData = gStaticPins.chromium_data; + let entries = chromePreloads.entries; + entries.forEach(function (entry) { + // HSTS entry only + if (!entry.pins) { + return; + } + let pinsetName = cData.substitute_pinsets[entry.pins]; + if (!pinsetName) { + pinsetName = entry.pins; + } + + // We trim the entry name here to avoid breaking hostname comparisons in the + // HPKP implementation. + entry.name = entry.name.trim(); + + let isProductionDomain = cData.production_domains.includes(entry.name); + let isProductionPinset = cData.production_pinsets.includes(pinsetName); + let excludeDomain = cData.exclude_domains.includes(entry.name); + let isTestMode = !isProductionPinset && !isProductionDomain; + if (entry.pins && !excludeDomain && chromeImportedPinsets[entry.pins]) { + chromeImportedEntries.push({ + name: entry.name, + include_subdomains: entry.include_subdomains, + test_mode: isTestMode, + is_moz: false, + pins: pinsetName, + }); + } + }); + return [chromeImportedPinsets, chromeImportedEntries]; +} + +// Returns a pair of maps [certNameToSKD, certSKDToName] between cert +// nicknames and digests of the SPKInfo for the mozilla trust store +function loadNSSCertinfo(extraCertificates) { + let allCerts = gCertDB.getCerts(); + let certNameToSKD = {}; + let certSKDToName = {}; + for (let cert of allCerts) { + let name = cert.displayName; + let SKD = cert.sha256SubjectPublicKeyInfoDigest; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + + for (let cert of extraCertificates) { + let name = cert.commonName; + let SKD = cert.sha256SubjectPublicKeyInfoDigest; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + + { + // This is the pinning test certificate. The key hash identifies the + // default RSA key from pykey. + let name = "End Entity Test Cert"; + let SKD = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8="; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + return [certNameToSKD, certSKDToName]; +} + +function parseJson(filename) { + let json = stripComments(readFileToString(filename)); + return JSON.parse(json); +} + +function nameToAlias(certName) { + // change the name to a string valid as a c identifier + // remove non-ascii characters + certName = certName.replace(/[^[:ascii:]]/g, "_"); + // replace non word characters + certName = certName.replace(/[^A-Za-z0-9]/g, "_"); + + return "k" + certName + "Fingerprint"; +} + +function compareByName(a, b) { + return a.name.localeCompare(b.name); +} + +function genExpirationTime() { + let now = new Date(); + let nowMillis = now.getTime(); + let expirationMillis = nowMillis + PINNING_MINIMUM_REQUIRED_MAX_AGE * 1000; + let expirationMicros = expirationMillis * 1000; + return ( + "static const PRTime kPreloadPKPinsExpirationTime = INT64_C(" + + expirationMicros + + ");\n" + ); +} + +function writeFullPinset(certNameToSKD, certSKDToName, pinset) { + if (!pinset.sha256_hashes || !pinset.sha256_hashes.length) { + throw new Error(`ERROR: Pinset ${pinset.name} does not contain any hashes`); + } + writeFingerprints( + certNameToSKD, + certSKDToName, + pinset.name, + pinset.sha256_hashes + ); +} + +function writeFingerprints(certNameToSKD, certSKDToName, name, hashes) { + let varPrefix = "kPinset_" + name; + writeString("static const char* const " + varPrefix + "_Data[] = {\n"); + let SKDList = []; + for (let certName of hashes) { + if (!(certName in certNameToSKD)) { + throw new Error(`ERROR: Can't find '${certName}' in certNameToSKD`); + } + SKDList.push(certNameToSKD[certName]); + } + for (let skd of SKDList.sort()) { + writeString(" " + nameToAlias(certSKDToName[skd]) + ",\n"); + } + if (!hashes.length) { + // ANSI C requires that an initialiser list be non-empty. + writeString(" 0\n"); + } + writeString("};\n"); + writeString( + "static const StaticFingerprints " + + varPrefix + + " = {\n " + + "sizeof(" + + varPrefix + + "_Data) / sizeof(const char*),\n " + + varPrefix + + "_Data\n};\n\n" + ); +} + +function writeEntry(entry) { + let printVal = ` { "${entry.name}", `; + if (entry.include_subdomains) { + printVal += "true, "; + } else { + printVal += "false, "; + } + // Default to test mode if not specified. + let testMode = true; + if (entry.hasOwnProperty("test_mode")) { + testMode = entry.test_mode; + } + if (testMode) { + printVal += "true, "; + } else { + printVal += "false, "; + } + if ( + entry.is_moz || + (entry.pins.includes("mozilla") && entry.pins != "mozilla_test") + ) { + printVal += "true, "; + } else { + printVal += "false, "; + } + if ("id" in entry) { + if (entry.id >= 256) { + throw new Error("ERROR: Not enough buckets in histogram"); + } + if (entry.id >= 0) { + printVal += entry.id + ", "; + } + } else { + printVal += "-1, "; + } + printVal += "&kPinset_" + entry.pins; + printVal += " },\n"; + writeString(printVal); +} + +function writeDomainList(chromeImportedEntries) { + writeString("/* Sort hostnames for binary search. */\n"); + writeString( + "static const TransportSecurityPreload " + + "kPublicKeyPinningPreloadList[] = {\n" + ); + let count = 0; + let mozillaDomains = {}; + gStaticPins.entries.forEach(function (entry) { + mozillaDomains[entry.name] = true; + }); + // For any domain for which we have set pins, exclude them from + // chromeImportedEntries. + for (let i = chromeImportedEntries.length - 1; i >= 0; i--) { + if (mozillaDomains[chromeImportedEntries[i].name]) { + dump( + "Skipping duplicate pinset for domain " + + JSON.stringify(chromeImportedEntries[i], undefined, 2) + + "\n" + ); + chromeImportedEntries.splice(i, 1); + } + } + let sortedEntries = gStaticPins.entries; + sortedEntries.push.apply(sortedEntries, chromeImportedEntries); + for (let entry of sortedEntries.sort(compareByName)) { + count++; + writeEntry(entry); + } + writeString("};\n"); + + writeString("\n// Pinning Preload List Length = " + count + ";\n"); + writeString("\nstatic const int32_t kUnknownId = -1;\n"); +} + +function writeFile( + certNameToSKD, + certSKDToName, + chromeImportedPinsets, + chromeImportedEntries +) { + // Compute used pins from both Chrome's and our pinsets, so we can output + // them later. + let usedFingerprints = {}; + let mozillaPins = {}; + gStaticPins.pinsets.forEach(function (pinset) { + mozillaPins[pinset.name] = true; + pinset.sha256_hashes.forEach(function (name) { + usedFingerprints[name] = true; + }); + }); + for (let key in chromeImportedPinsets) { + let pinset = chromeImportedPinsets[key]; + pinset.sha256_hashes.forEach(function (name) { + usedFingerprints[name] = true; + }); + } + + writeString(FILE_HEADER); + + // Write actual fingerprints. + Object.keys(usedFingerprints) + .sort() + .forEach(function (certName) { + if (certName) { + writeString("/* " + certName + " */\n"); + writeString("static const char " + nameToAlias(certName) + "[] =\n"); + writeString(' "' + certNameToSKD[certName] + '";\n'); + writeString("\n"); + } + }); + + // Write the pinsets + writeString(PINSETDEF); + writeString("/* PreloadedHPKPins.json pinsets */\n"); + gStaticPins.pinsets.sort(compareByName).forEach(function (pinset) { + writeFullPinset(certNameToSKD, certSKDToName, pinset); + }); + writeString("/* Chrome static pinsets */\n"); + for (let key in chromeImportedPinsets) { + if (mozillaPins[key]) { + dump("Skipping duplicate pinset " + key + "\n"); + } else { + dump("Writing pinset " + key + "\n"); + writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]); + } + } + + // Write the domainlist entries. + writeString(DOMAINHEADER); + writeDomainList(chromeImportedEntries); + writeString("\n"); + writeString(genExpirationTime()); +} + +function loadExtraCertificates(certStringList) { + let constructedCerts = []; + for (let certString of certStringList) { + constructedCerts.push(gCertDB.constructX509FromBase64(certString)); + } + return constructedCerts; +} + +var extraCertificates = loadExtraCertificates(gStaticPins.extra_certificates); +var [certNameToSKD, certSKDToName] = loadNSSCertinfo(extraCertificates); +var [chromeNameToHash, chromeNameToMozName] = downloadAndParseChromeCerts( + gStaticPins.chromium_data.cert_file_url, + certNameToSKD, + certSKDToName +); +var [chromeImportedPinsets, chromeImportedEntries] = downloadAndParseChromePins( + gStaticPins.chromium_data.json_file_url, + chromeNameToHash, + chromeNameToMozName, + certNameToSKD, + certSKDToName +); + +writeFile( + certNameToSKD, + certSKDToName, + chromeImportedPinsets, + chromeImportedEntries +); + +FileUtils.closeSafeFileOutputStream(gFileOutputStream); diff --git a/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js new file mode 100644 index 0000000000..aeaa29bc2d --- /dev/null +++ b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js @@ -0,0 +1,557 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +// How to run this file: +// 1. [obtain firefox source code] +// 2. [build/obtain firefox binaries] +// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell [path to]/getHSTSPreloadlist.js [absolute path to]/nsSTSPreloadlist.inc' +// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current +// working directory. + +var gSSService = Cc["@mozilla.org/ssservice;1"].getService( + Ci.nsISiteSecurityService +); + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const SOURCE = + "https://chromium.googlesource.com/chromium/src/+/refs/heads/main/net/http/transport_security_state_static.json?format=TEXT"; +const TOOL_SOURCE = + "https://hg.mozilla.org/mozilla-central/file/default/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js"; +const OUTPUT = "nsSTSPreloadList.inc"; +const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18; +const MAX_CONCURRENT_REQUESTS = 500; +const MAX_RETRIES = 1; +const REQUEST_TIMEOUT = 30 * 1000; +const ERROR_NONE = "no error"; +const ERROR_CONNECTING_TO_HOST = "could not connect to host"; +const ERROR_NO_HSTS_HEADER = "did not receive HSTS header"; +const ERROR_MAX_AGE_TOO_LOW = "max-age too low: "; +const HEADER = `/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/*****************************************************************************/ +/* This is an automatically generated file. If you're not */ +/* nsSiteSecurityService.cpp, you shouldn't be #including it. */ +/*****************************************************************************/ + +#include +`; + +const GPERF_DELIM = "%%\n"; + +function download() { + let req = new XMLHttpRequest(); + req.open("GET", SOURCE, false); // doing the request synchronously + try { + req.send(); + } catch (e) { + throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`); + } + + if (req.status != 200) { + throw new Error( + "ERROR: problem downloading '" + SOURCE + "': status " + req.status + ); + } + + let resultDecoded; + try { + resultDecoded = atob(req.responseText); + } catch (e) { + throw new Error( + "ERROR: could not decode data as base64 from '" + SOURCE + "': " + e + ); + } + + // we have to filter out '//' comments, while not mangling the json + let result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/gm, ""); + let data = null; + try { + data = JSON.parse(result); + } catch (e) { + throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`); + } + return data; +} + +function getHosts(rawdata) { + let hosts = []; + + if (!rawdata || !rawdata.entries) { + throw new Error( + "ERROR: source data not formatted correctly: 'entries' not found" + ); + } + + for (let entry of rawdata.entries) { + if (entry.mode && entry.mode == "force-https") { + if (entry.name) { + // We trim the entry name here to avoid malformed URI exceptions when we + // later try to connect to the domain. + entry.name = entry.name.trim(); + entry.retries = MAX_RETRIES; + // We prefer the camelCase variable to the JSON's snake case version + entry.includeSubdomains = entry.include_subdomains; + hosts.push(entry); + } else { + throw new Error("ERROR: entry not formatted correctly: no name found"); + } + } + } + + return hosts; +} + +function processStsHeader(host, header, status, securityInfo) { + let maxAge = { + value: 0, + }; + let includeSubdomains = { + value: false, + }; + let error = ERROR_NONE; + if ( + header != null && + securityInfo != null && + securityInfo.overridableErrorCategory == + Ci.nsITransportSecurityInfo.ERROR_UNSET + ) { + try { + let uri = Services.io.newURI("https://" + host.name); + gSSService.processHeader(uri, header, {}, maxAge, includeSubdomains); + } catch (e) { + dump( + "ERROR: could not process header '" + + header + + "' from " + + host.name + + ": " + + e + + "\n" + ); + error = e; + } + } else if (status == 0) { + error = ERROR_CONNECTING_TO_HOST; + } else { + error = ERROR_NO_HSTS_HEADER; + } + + if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) { + error = ERROR_MAX_AGE_TOO_LOW; + } + + return { + name: host.name, + maxAge: maxAge.value, + includeSubdomains: includeSubdomains.value, + error, + retries: host.retries - 1, + forceInclude: host.forceInclude, + }; +} + +// RedirectAndAuthStopper prevents redirects and HTTP authentication +function RedirectAndAuthStopper() {} + +RedirectAndAuthStopper.prototype = { + // nsIChannelEventSink + asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) { + throw Components.Exception("", Cr.NS_ERROR_ENTITY_CHANGED); + }, + + // nsIAuthPrompt2 + promptAuth(channel, level, authInfo) { + return false; + }, + + asyncPromptAuth(channel, callback, context, level, authInfo) { + throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED); + }, + + getInterface(iid) { + return this.QueryInterface(iid); + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIChannelEventSink", + "nsIAuthPrompt2", + ]), +}; + +function fetchstatus(host) { + return new Promise((resolve, reject) => { + let xhr = new XMLHttpRequest(); + let uri = "https://" + host.name + "/"; + + xhr.open("head", uri, true); + xhr.setRequestHeader("X-Automated-Tool", TOOL_SOURCE); + xhr.timeout = REQUEST_TIMEOUT; + + let errorHandler = () => { + dump("ERROR: exception making request to " + host.name + "\n"); + resolve( + processStsHeader( + host, + null, + xhr.status, + xhr.channel && xhr.channel.securityInfo + ) + ); + }; + + xhr.onerror = errorHandler; + xhr.ontimeout = errorHandler; + xhr.onabort = errorHandler; + + xhr.onload = () => { + let header = xhr.getResponseHeader("strict-transport-security"); + resolve( + processStsHeader(host, header, xhr.status, xhr.channel.securityInfo) + ); + }; + + xhr.channel.notificationCallbacks = new RedirectAndAuthStopper(); + xhr.send(); + }); +} + +async function getHSTSStatus(host) { + do { + host = await fetchstatus(host); + } while (shouldRetry(host)); + return host; +} + +function compareHSTSStatus(a, b) { + if (a.name > b.name) { + return 1; + } + if (a.name < b.name) { + return -1; + } + return 0; +} + +function writeTo(string, fos) { + fos.write(string, string.length); +} + +// Determines and returns a string representing a declaration of when this +// preload list should no longer be used. +// This is the current time plus MINIMUM_REQUIRED_MAX_AGE. +function getExpirationTimeString() { + let now = new Date(); + let nowMillis = now.getTime(); + // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds + let expirationMillis = nowMillis + MINIMUM_REQUIRED_MAX_AGE * 1000; + let expirationMicros = expirationMillis * 1000; + return ( + "const PRTime gPreloadListExpirationTime = INT64_C(" + + expirationMicros + + ");\n" + ); +} + +function shouldRetry(response) { + return ( + response.error != ERROR_NO_HSTS_HEADER && + response.error != ERROR_MAX_AGE_TOO_LOW && + response.error != ERROR_NONE && + response.retries > 0 + ); +} + +// Copied from browser/components/migration/MigrationUtils.sys.mjs +function spinResolve(promise) { + if (!(promise instanceof Promise)) { + return promise; + } + let done = false; + let result = null; + let error = null; + promise + .catch(e => { + error = e; + }) + .then(r => { + result = r; + done = true; + }); + + Services.tm.spinEventLoopUntil( + "getHSTSPreloadList.js:spinResolve", + () => done + ); + if (error) { + throw error; + } else { + return result; + } +} + +async function probeHSTSStatuses(inHosts) { + let totalLength = inHosts.length; + dump("Examining " + totalLength + " hosts.\n"); + + // Make requests in batches of MAX_CONCURRENT_REQUESTS. Otherwise, we have + // too many in-flight requests and the time it takes to process them causes + // them all to time out. + let allResults = []; + while (inHosts.length) { + let promises = []; + for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length; i++) { + let host = inHosts.shift(); + promises.push(getHSTSStatus(host)); + } + let results = await Promise.all(promises); + let progress = ( + (100 * (totalLength - inHosts.length)) / + totalLength + ).toFixed(2); + dump(progress + "% done\n"); + allResults = allResults.concat(results); + } + + dump("HSTS Probe received " + allResults.length + " statuses.\n"); + return allResults; +} + +function readCurrentList(filename) { + var currentHosts = {}; + var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(filename); + var fis = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsILineInputStream + ); + fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF); + var line = {}; + + // While we generate entries matching the latest version format, + // we still need to be able to read entries in the previous version formats + // for bootstrapping a latest version preload list from a previous version + // preload list. Hence these regexes. + const entryRegexes = [ + /([^,]+), (0|1)/, // v3 + / {2}\/\* "([^"]*)", (true|false) \*\//, // v2 + / {2}{ "([^"]*)", (true|false) },/, // v1 + ]; + + while (fis.readLine(line)) { + let match; + entryRegexes.find(r => { + match = r.exec(line.value); + return match; + }); + if (match) { + currentHosts[match[1]] = match[2] == "1" || match[2] == "true"; + } + } + return currentHosts; +} + +function combineLists(newHosts, currentHosts) { + let newHostsSet = new Set(); + + for (let newHost of newHosts) { + newHostsSet.add(newHost.name); + } + + for (let currentHost in currentHosts) { + if (!newHostsSet.has(currentHost)) { + newHosts.push({ name: currentHost, retries: MAX_RETRIES }); + } + } +} + +const TEST_ENTRIES = [ + { + name: "includesubdomains.preloaded.test", + includeSubdomains: true, + }, + { + name: "includesubdomains2.preloaded.test", + includeSubdomains: true, + }, + { + name: "noincludesubdomains.preloaded.test", + includeSubdomains: false, + }, +]; + +function deleteTestHosts(currentHosts) { + for (let testEntry of TEST_ENTRIES) { + delete currentHosts[testEntry.name]; + } +} + +function getTestHosts() { + let hosts = []; + for (let testEntry of TEST_ENTRIES) { + hosts.push({ + name: testEntry.name, + maxAge: MINIMUM_REQUIRED_MAX_AGE, + includeSubdomains: testEntry.includeSubdomains, + error: ERROR_NONE, + // This deliberately doesn't have a value for `retries` (because we should + // never attempt to connect to this host). + forceInclude: true, + }); + } + return hosts; +} + +async function insertHosts(inoutHostList, inAddedHosts) { + for (let host of inAddedHosts) { + inoutHostList.push(host); + } +} + +function filterForcedInclusions(inHosts, outNotForced, outForced) { + // Apply our filters (based on policy today) to determine which entries + // will be included without being checked (forced); the others will be + // checked using active probing. + for (let host of inHosts) { + if ( + host.policy == "google" || + host.policy == "public-suffix" || + host.policy == "public-suffix-requested" + ) { + host.forceInclude = true; + host.error = ERROR_NONE; + outForced.push(host); + } else { + outNotForced.push(host); + } + } +} + +function output(statuses) { + dump("INFO: Writing output to " + OUTPUT + "\n"); + try { + let file = new FileUtils.File( + PathUtils.join(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, OUTPUT) + ); + let fos = FileUtils.openSafeFileOutputStream(file); + writeTo(HEADER, fos); + writeTo(getExpirationTimeString(), fos); + + writeTo(GPERF_DELIM, fos); + + for (let status of statuses) { + let includeSubdomains = status.includeSubdomains ? 1 : 0; + writeTo(status.name + ", " + includeSubdomains + "\n", fos); + } + + writeTo(GPERF_DELIM, fos); + FileUtils.closeSafeFileOutputStream(fos); + dump("finished writing output file\n"); + } catch (e) { + dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n"); + throw e; + } +} + +function errorToString(status) { + return status.error == ERROR_MAX_AGE_TOO_LOW + ? status.error + status.maxAge + : status.error; +} + +async function main(args) { + if (args.length != 1) { + throw new Error( + "Usage: getHSTSPreloadList.js " + ); + } + + // get the current preload list + let currentHosts = readCurrentList(args[0]); + // delete any hosts we use in tests so we don't actually connect to them + deleteTestHosts(currentHosts); + // disable the current preload list so it won't interfere with requests we make + Services.prefs.setBoolPref( + "network.stricttransportsecurity.preloadlist", + false + ); + // download and parse the raw json file from the Chromium source + let rawdata = download(); + // get just the hosts with mode: "force-https" + let hosts = getHosts(rawdata); + // add hosts in the current list to the new list (avoiding duplicates) + combineLists(hosts, currentHosts); + + // Don't contact hosts that are forced to be included anyway + let hostsToContact = []; + let forcedHosts = []; + filterForcedInclusions(hosts, hostsToContact, forcedHosts); + + // Initialize the final status list + let hstsStatuses = []; + // Add the hosts we use in tests + dump("Adding test hosts\n"); + insertHosts(hstsStatuses, getTestHosts()); + // Add in the hosts that are forced + dump("Adding forced hosts\n"); + insertHosts(hstsStatuses, forcedHosts); + + let total = await probeHSTSStatuses(hostsToContact) + .then(function (probedStatuses) { + return hstsStatuses.concat(probedStatuses); + }) + .then(function (statuses) { + return statuses.sort(compareHSTSStatus); + }) + .then(function (statuses) { + for (let status of statuses) { + // If we've encountered an error for this entry (other than the site not + // sending an HSTS header), be safe and don't remove it from the list + // (given that it was already on the list). + if ( + !status.forceInclude && + status.error != ERROR_NONE && + status.error != ERROR_NO_HSTS_HEADER && + status.error != ERROR_MAX_AGE_TOO_LOW && + status.name in currentHosts + ) { + // dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n"); + status.maxAge = MINIMUM_REQUIRED_MAX_AGE; + status.includeSubdomains = currentHosts[status.name]; + } + } + return statuses; + }) + .then(function (statuses) { + // Filter out entries we aren't including. + var includedStatuses = statuses.filter(function (status) { + if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) { + // dump("INFO: " + status.name + " NOT ON the preload list\n"); + return false; + } + + // dump("INFO: " + status.name + " ON the preload list (includeSubdomains: " + status.includeSubdomains + ")\n"); + if (status.forceInclude && status.error != ERROR_NONE) { + dump( + status.name + + ": " + + errorToString(status) + + " (error ignored - included regardless)\n" + ); + } + return true; + }); + return includedStatuses; + }); + + // Write the output file + output(total); + + dump("HSTS probing all done\n"); +} + +// arguments is a global within xpcshell +spinResolve(main(arguments)); diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh new file mode 100755 index 0000000000..c5b9c78f6f --- /dev/null +++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh @@ -0,0 +1,672 @@ +#!/bin/bash + +set -ex + +function usage { + cat <&2 + exit 31 + fi + if [ ! -f "${TESTS_ARCHIVE}" ]; then + echo "Downloaded file '${TESTS_ARCHIVE}' not found in directory '$(pwd)'." >&2 + exit 32 + fi + # Unpack the browser and move xpcshell in place for updating the preload list. + echo "INFO: Unpacking resources..." + ${UNPACK_CMD} "${BROWSER_ARCHIVE}" + mkdir -p tests + cd tests + ${UNTAR} "../${TESTS_ARCHIVE}" + cd "${BASEDIR}" + cp tests/bin/xpcshell "${PRODUCT}" +} + +# Downloads the current in-tree HSTS (HTTP Strict Transport Security) files. +# Runs a simple xpcshell script to generate up-to-date HSTS information. +# Compares the new HSTS output with the old to determine whether we need to update. +function compare_hsts_files { + cd "${BASEDIR}" + + HSTS_PRELOAD_INC_HG="${HGREPO}/raw-file/default/security/manager/ssl/$(basename "${HSTS_PRELOAD_INC_OLD}")" + + echo "INFO: Downloading existing include file..." + rm -rf "${HSTS_PRELOAD_ERRORS}" "${HSTS_PRELOAD_INC_OLD}" + echo "INFO: ${WGET} ${HSTS_PRELOAD_INC_HG}" + ${WGET} -O "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_HG}" + + if [ ! -f "${HSTS_PRELOAD_INC_OLD}" ]; then + echo "Downloaded file '${HSTS_PRELOAD_INC_OLD}' not found in directory '$(pwd)' - this should have been downloaded above from ${HSTS_PRELOAD_INC_HG}." >&2 + exit 41 + fi + + # Run the script to get an updated preload list. + echo "INFO: Generating new HSTS preload list..." + cd "${BASEDIR}/${PRODUCT}" + if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HSTS_PRELOAD_SCRIPT}" "${HSTS_PRELOAD_INC_OLD}"; then + echo "HSTS preload list generation failed" >&2 + exit 43 + fi + + # The created files should be non-empty. + echo "INFO: Checking whether new HSTS preload list is valid..." + if [ ! -s "${HSTS_PRELOAD_INC_NEW}" ]; then + echo "New HSTS preload list ${HSTS_PRELOAD_INC_NEW} is empty. That's less good." >&2 + exit 42 + fi + cd "${BASEDIR}" + + # Check for differences + echo "INFO: diffing old/new HSTS preload lists into ${HSTS_DIFF_ARTIFACT}" + ${DIFF} "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_NEW}" | tee "${HSTS_DIFF_ARTIFACT}" + if [ -s "${HSTS_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +# Downloads the current in-tree HPKP (HTTP public key pinning) files. +# Runs a simple xpcshell script to generate up-to-date HPKP information. +# Compares the new HPKP output with the old to determine whether we need to update. +function compare_hpkp_files { + cd "${BASEDIR}" + HPKP_PRELOAD_JSON_HG="${HGREPO}/raw-file/default/security/manager/tools/$(basename "${HPKP_PRELOAD_JSON}")" + + HPKP_PRELOAD_OUTPUT_HG="${HGREPO}/raw-file/default/security/manager/ssl/${HPKP_PRELOAD_INC}" + + rm -f "${HPKP_PRELOAD_OUTPUT}" + ${WGET} -O "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT_HG}" + ${WGET} -O "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_JSON_HG}" + + # Run the script to get an updated preload list. + echo "INFO: Generating new HPKP preload list..." + cd "${BASEDIR}/${PRODUCT}" + if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HPKP_PRELOAD_SCRIPT}" "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_OUTPUT}" > "${HPKP_PRELOAD_ERRORS}"; then + echo "HPKP preload list generation failed" >&2 + exit 54 + fi + + # The created files should be non-empty. + echo "INFO: Checking whether new HPKP preload list is valid..." + + if [ ! -s "${HPKP_PRELOAD_OUTPUT}" ]; then + echo "${HPKP_PRELOAD_OUTPUT} is empty. That's less good." >&2 + exit 52 + fi + if ! grep kPreloadPKPinsExpirationTime "${HPKP_PRELOAD_OUTPUT}"; then + echo "${HPKP_PRELOAD_OUTPUT} is missing an expiration time. Truncated?" >&2 + exit 53 + fi + cd "${BASEDIR}" + + echo "INFO: diffing old/new HPKP preload lists..." + ${DIFF} "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT}" | tee "${HPKP_DIFF_ARTIFACT}" + if [ -s "${HPKP_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +function is_valid_xml { + xmlfile=$1 + XMLLINT=$(command -v xmllint 2>/dev/null | head -n1) + + if [ ! -x "${XMLLINT}" ]; then + echo "ERROR: xmllint not found in PATH" + exit 60 + fi + ${XMLLINT} --nonet --noout "${xmlfile}" +} + +# Downloads the public suffix list +function compare_suffix_lists { + HG_SUFFIX_URL="${HGREPO}/raw-file/default/${HG_SUFFIX_PATH}" + cd "${BASEDIR}" + + echo "INFO: ${WGET} -O ${GITHUB_SUFFIX_LOCAL} ${GITHUB_SUFFIX_URL}" + rm -f "${GITHUB_SUFFIX_LOCAL}" + ${WGET} -O "${GITHUB_SUFFIX_LOCAL}" "${GITHUB_SUFFIX_URL}" + + echo "INFO: ${WGET} -O ${HG_SUFFIX_LOCAL} ${HG_SUFFIX_URL}" + rm -f "${HG_SUFFIX_LOCAL}" + ${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}" + + echo "INFO: diffing in-tree suffix list against the suffix list from AMO..." + ${DIFF} ${GITHUB_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}" + if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +function compare_remote_settings_files { + REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1" + + # 1. List remote settings collections from server. + echo "INFO: fetch remote settings list from server" + ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\ + ${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\ + # 2. For each entry ${bucket, collection, last_modified} + while IFS="/" read -r bucket collection last_modified; do + + # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder + hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json" + local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json" + mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}" + ${WGET} -qO "$local_location_input" "$hg_dump_url" + if [ $? -eq 8 ]; then + # We don't keep any dump for this collection, skip it. + # Try to clean up in case no collection in this bucket has dump. + rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty + continue + fi + + # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder + remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}" + local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json" + mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}" + ${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}" + + # 5. Download attachments if needed. + if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then + # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm. + # The server should return one "bloomfilter-base" record, but in case it returns multiple, + # return the most recent one. The server may send multiple entries if we ever decide to use + # the "filter_expression" feature of Remote Settings to send different records to specific + # channels. In that case this code should be updated to recognize the filter expression, + # but until we do, simply select the most recent record - can't go wrong with that. + # Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters. + update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \ + 'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last' + fi + # TODO: Bug 1873448. This cannot handle new/removed files currently, due to the + # build system making it difficult. + if [ "${bucket}" = "main" ] && [ "${collection}" = "search-config-icons" ]; then + ${JQ} -r '.data[] | .id' < "${local_location_output}" |\ + while read -r id; do + # We do not want quotes around ${id} + # shellcheck disable=SC2086 + update_remote_settings_attachment "${bucket}" "${collection}" ${id} ".[] | select(.id == \"${id}\")" + done + fi + # NOTE: The downloaded data is not validated. xpcshell should be used for that. + done + + echo "INFO: diffing old/new remote settings dumps..." + ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}" + if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +# Helper for compare_remote_settings_files to download attachments from remote settings. +# The format and location is documented at: +# https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#packaging-attachments +function update_remote_settings_attachment() { + local bucket=$1 + local collection=$2 + local attachment_id=$3 + # $4 is a jq filter on the arrays that should return one record with the attachment + local jq_attachment_selector=".data | map(select(.attachment)) | $4" + + # These paths match _readAttachmentDump in services/settings/Attachments.jsm. + local path_to_attachment="${bucket}/${collection}/${attachment_id}" + local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json" + local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}" + local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}" + + # Those files should have been created by compare_remote_settings_files before the function call. + local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json" + local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json" + + # Compute the metadata based on already-downloaded records. + mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}" + ${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}" + mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}" + ${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}" + + if cmp --silent "${old_meta}" "${new_meta}" ; then + # Metadata not changed, don't bother downloading the attachments themselves. + return + fi + # Metadata changed. Download attachments. + + echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}" + + # Overwrited old_meta with the actual file from the repo. The content should be equivalent, + # but can have minor differences (e.g. different line endings) if the checked in file was not + # generated by this script (e.g. manually checked in). + ${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}" + + ${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}" + + if [ -z "${ATTACHMENT_BASE_URL}" ] ; then + ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url) + fi + attachment_path_from_meta=$(${JQ} -r < "${new_meta}" .attachment.location) + ${WGET} -qO "${REMOTE_SETTINGS_OUTPUT}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}" +} + +function compare_mobile_experiments() { + echo "INFO ${WGET} ${EXPERIMENTER_URL}" + ${WGET} -O experiments.json "${EXPERIMENTER_URL}" + ${WGET} -O fenix-experiments-old.json "${HGREPO}/raw-file/default/${FENIX_INITIAL_EXPERIMENTS}" + ${WGET} -O focus-experiments-old.json "${HGREPO}/raw-file/default/${FOCUS_INITIAL_EXPERIMENTS}" + + # shellcheck disable=SC2016 + ${JQ} --arg APP_NAME fenix '{"data":map(select(.appName == $APP_NAME))}' < experiments.json > fenix-experiments-new.json + # shellcheck disable=SC2016 + ${JQ} --arg APP_NAME focus_android '{"data":map(select(.appName == $APP_NAME))}' < experiments.json > focus-experiments-new.json + + ( ${DIFF} fenix-experiments-old.json fenix-experiments-new.json; ${DIFF} focus-experiments-old.json focus-experiments-new.json ) > "${EXPERIMENTER_DIFF_ARTIFACT}" + if [ -s "${EXPERIMENTER_DIFF_ARTIFACT}" ]; then + # no change + return 1 + else + return 0 + fi +} + +# Clones an hg repo +function clone_repo { + cd "${BASEDIR}" + if [ ! -d "${REPODIR}" ]; then + ${HG} robustcheckout --sharebase /tmp/hg-store -b default "${HGREPO}" "${REPODIR}" + fi + + ${HG} -R "${REPODIR}" pull + ${HG} -R "${REPODIR}" update -C default +} + +# Copies new HSTS files in place, and commits them. +function stage_hsts_files { + cd "${BASEDIR}" + cp -f "${HSTS_PRELOAD_INC_NEW}" "${REPODIR}/security/manager/ssl/" +} + +function stage_hpkp_files { + cd "${BASEDIR}" + cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}" +} + +function stage_remote_settings_files { + cd "${BASEDIR}" + cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}" +} + +function stage_tld_suffix_files { + cd "${BASEDIR}" + cp -a "${GITHUB_SUFFIX_LOCAL}" "${REPODIR}/${HG_SUFFIX_PATH}" +} + +function stage_mobile_experiments_files { + cd "${BASEDIR}" + + cp fenix-experiments-new.json "${REPODIR}/${FENIX_INITIAL_EXPERIMENTS}" + cp focus-experiments-new.json "${REPODIR}/${FOCUS_INITIAL_EXPERIMENTS}" +} + +# Push all pending commits to Phabricator +function push_repo { + cd "${REPODIR}" + if [ ! -r "${HOME}/.arcrc" ] + then + return 1 + fi + if ! ARC=$(command -v arc) && ! ARC=$(command -v arcanist) + then + return 1 + fi + if [ -z "${REVIEWERS}" ] + then + return 1 + fi + # Clean up older review requests + # Turn Needs Review D624: No bug, Automated HSTS ... + # into D624 + for diff in $($ARC list | grep "Needs Review" | grep -E "${BRANCH} repo-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }') + do + echo "Removing old request $diff" + # There is no 'arc abandon', see bug 1452082 + echo '{"transactions": [{"type":"abandon", "value": true}], "objectIdentifier": "'"${diff}"'"}' | $ARC call-conduit -- differential.revision.edit + done + + $ARC diff --verbatim --reviewers "${REVIEWERS}" +} + + + +# Main + +# Parse our command-line options. +while [ $# -gt 0 ]; do + case "$1" in + -h) usage; exit 0 ;; + -p) PRODUCT="$2"; shift ;; + -b) BRANCH="$2"; shift ;; + -n) DRY_RUN=true ;; + -c) CLOSED_TREE=true ;; + -d) DONTBUILD=true ;; + -a) APPROVAL=true ;; + --pinset) DO_PRELOAD_PINSET=true ;; + --hsts) DO_HSTS=true ;; + --hpkp) DO_HPKP=true ;; + --remote-settings) DO_REMOTE_SETTINGS=true ;; + --suffix-list) DO_SUFFIX_LIST=true ;; + --mobile-experiments) DO_MOBILE_EXPERIMENTS=true ;; + -r) REPODIR="$2"; shift ;; + --use-mozilla-central) USE_MC=true ;; + --use-ftp-builds) USE_TC=false ;; + -*) usage + exit 11 ;; + *) break ;; # terminate while loop + esac + shift +done + +# Must supply a code branch to work with. +if [ "${BRANCH}" == "" ]; then + echo "Error: You must specify a branch with -b branchname." >&2 + usage + exit 12 +fi + +# Must choose at least one update action. +if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ] && [ "$DO_MOBILE_EXPERIMENTS" == false ] +then + echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2 + usage + exit 13 +fi + +# per-product constants +case "${PRODUCT}" in + thunderbird) + COMMIT_AUTHOR="tbirdbld " + ;; + firefox) + ;; + *) + echo "Error: Invalid product specified" + usage + exit 14 + ;; +esac + +if [ "${REPODIR}" == "" ]; then + REPODIR="$(basename "${BRANCH}")" +fi + +case "${BRANCH}" in + mozilla-central|comm-central|try ) + HGREPO="https://${HGHOST}/${BRANCH}" + ;; + mozilla-*|comm-* ) + HGREPO="https://${HGHOST}/releases/${BRANCH}" + ;; + * ) + HGREPO="https://${HGHOST}/projects/${BRANCH}" + ;; +esac + +BROWSER_ARCHIVE="target.${PLATFORM_EXT}" +TESTS_ARCHIVE="target.common.tests.tar.gz" + +preflight_cleanup +if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ] +then + if [ "${USE_TC}" == "true" ]; then + download_shared_artifacts_from_tc + else + download_shared_artifacts_from_ftp + fi + unpack_artifacts +fi + +if [ "${DO_HSTS}" == "true" ]; then + if compare_hsts_files + then + HSTS_UPDATED=true + fi +fi +if [ "${DO_HPKP}" == "true" ]; then + if compare_hpkp_files + then + HPKP_UPDATED=true + fi +fi +if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then + if compare_remote_settings_files + then + REMOTE_SETTINGS_UPDATED=true + fi +fi +if [ "${DO_SUFFIX_LIST}" == "true" ]; then + if compare_suffix_lists + then + SUFFIX_LIST_UPDATED=true + fi +fi +if [ "${DO_MOBILE_EXPERIMENTS}" == "true" ]; then + if compare_mobile_experiments + then + MOBILE_EXPERIMENTS_UPDATED=true + fi +fi + + +if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ] && [ "${MOBILE_EXPERIMENTS_UPDATED}" == "false" ]; then + echo "INFO: no updates required. Exiting." + exit 0 +else + if [ "${DRY_RUN}" == "true" ]; then + echo "INFO: Updates are available, not updating hg in dry-run mode." + exit 2 + fi +fi + +clone_repo + +COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update" +if [ "${HSTS_UPDATED}" == "true" ] +then + stage_hsts_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS" +fi + +if [ "${HPKP_UPDATED}" == "true" ] +then + stage_hpkp_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP" +fi + +if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ] +then + stage_remote_settings_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings" +fi + +if [ "${SUFFIX_LIST_UPDATED}" == "true" ] +then + stage_tld_suffix_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} tld-suffixes" +fi + +if [ "${MOBILE_EXPERIMENTS_UPDATED}" == "true" ] +then + stage_mobile_experiments_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} mobile-experiments" +fi + +if [ ${DONTBUILD} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)" +fi +if [ ${CLOSED_TREE} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE" +fi +if [ ${APPROVAL} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update" +fi + + +if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" +then + push_repo +fi + +echo "All done" diff --git a/taskcluster/docker/periodic-updates/setup.sh b/taskcluster/docker/periodic-updates/setup.sh new file mode 100755 index 0000000000..dd4faf5fc2 --- /dev/null +++ b/taskcluster/docker/periodic-updates/setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -ve + +apt-get update -q +apt-get install \ + arcanist \ + curl \ + jq \ + libasound2 \ + libgtk-3-0 \ + libx11-xcb1 \ + libxml2-utils \ + libxt6 \ + libxtst6 \ + shellcheck \ + unzip \ + bzip2 \ + wget + +rm -rf /setup diff --git a/taskcluster/docker/push-to-try/Dockerfile b/taskcluster/docker/push-to-try/Dockerfile new file mode 100644 index 0000000000..35bfbdff7a --- /dev/null +++ b/taskcluster/docker/push-to-try/Dockerfile @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +RUN apt-get update && \ + apt-get install \ + openssh-client + +COPY known_hosts /etc/ssh/ssh_known_hosts +COPY hgrc /etc/mercurial/hgrc.d/push-to-try.rc + +RUN su worker -c "hg clone -r 19a77a5d7b2a90ba225cf5015afd1ac44787abec https://hg.mozilla.org/hgcustom/version-control-tools /builds/worker/.mozbuild/version-control-tools" + +# Create directory for scripts to put an hg config in, for configuring ssh +RUN mkdir /builds/worker/.config/hg +RUN chown -R worker:worker /builds/worker/.config/hg diff --git a/taskcluster/docker/push-to-try/hgrc b/taskcluster/docker/push-to-try/hgrc new file mode 100644 index 0000000000..d1a0318499 --- /dev/null +++ b/taskcluster/docker/push-to-try/hgrc @@ -0,0 +1,2 @@ +[extensions] +push-to-try = ~/.mozbuild/version-control-tools/hgext/push-to-try diff --git a/taskcluster/docker/push-to-try/known_hosts b/taskcluster/docker/push-to-try/known_hosts new file mode 100644 index 0000000000..1dc2e39e70 --- /dev/null +++ b/taskcluster/docker/push-to-try/known_hosts @@ -0,0 +1,2 @@ +hg.mozilla.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEsS2fK+TVkHl4QvvOHB6R5xxngsSYJR+pA4+xDhw4mZT9tgCRU9BBG3LazSLp6PUxnpfok78475/tx6Z8QwbTyUTmLElZ9Z9eJzjaGz/olHzQSWv0VB3kT+VZt0LK7pEuaG+Ph/qwxbtUZZOApYLEvu8uctDlS66doofxZylbsgl1kpRQ5HNu+/DgVo9K9dyMOm9OLoy4tXHSE5pofn4tKYdFRa2lt6OVtIP5/hKNb2i0+JmgM8C3bJTPvzJ4C8p2h83ro29XPUkNAfWrgD5CmAPPqHFXyefDCfdefcvI8B8Za9v4j4LynBDZHsGfII+wIfzyLIxy9K6Op6nqDZgCciBRdgxh4uZQINEhB/JJP03Pxo42ExdG28oU3aL8kRRTORT5ehFtImFfr9QESHaUnbVzBbU5DmOB5voYDMle3RgyY+RXJ7+4OxjLRnJvGks9QCn8QrIvabs/PTCnenI8+yDhMlLUkWTiR4JK8vDBYB2Rm++EmVsN9WjllfDNg3Aj1aYe8XiBD4tS+lg7Ur4rJL8X20H4yMvq56sQ0qfH8PCIQGyGL725E7Yuwj/MHvou5xrPM/Lqo/MtX5T2njrzkeaBmI/zFJaLwbphdrwmrzepbcim7OYJFF2pz8u56KDPD1pUQ7C1gEIAx/4mHiDOGCYooSvyfD+JRdjkZUZMiQ== +hg.mozilla.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGuRsL+/OrvIekv3iBST1zdVBLBuh/DMIj+ZN72N9a0g diff --git a/taskcluster/docker/recipes/clean_packages.sh b/taskcluster/docker/recipes/clean_packages.sh new file mode 100755 index 0000000000..4265ae5593 --- /dev/null +++ b/taskcluster/docker/recipes/clean_packages.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +for task in "$@"; do + echo "removing package source /etc/apt/sources.list.d/99$task.list" + rm -f "/etc/apt/sources.list.d/99$task.list" +done +apt-get update diff --git a/taskcluster/docker/recipes/common.sh b/taskcluster/docker/recipes/common.sh new file mode 100644 index 0000000000..0182154495 --- /dev/null +++ b/taskcluster/docker/recipes/common.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +tooltool_fetch() { + cat >manifest.tt + python3 /setup/tooltool.py fetch + rm manifest.tt +} diff --git a/taskcluster/docker/recipes/debian-test-system-setup.sh b/taskcluster/docker/recipes/debian-test-system-setup.sh new file mode 100644 index 0000000000..80cc47116f --- /dev/null +++ b/taskcluster/docker/recipes/debian-test-system-setup.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ccache') +apt_packages+=('curl') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('gnome-keyring') +apt_packages+=('libasound2-dev') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libgconf2-dev') +apt_packages+=('libiw-dev') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse-dev') +apt_packages+=('libsox-fmt-alsa') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('locales') +apt_packages+=('locales-all') +apt_packages+=('net-tools') +apt_packages+=('openssh-client') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('tar') +apt_packages+=('ttf-dejavu') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xdg-user-dirs') +apt_packages+=('xvfb') +apt_packages+=('zip') + +# use Ubuntu's Python-2.7 (2.7.3 on Precise) +apt_packages+=('python-dev') +apt_packages+=('python-pip') + +apt-get update +apt-get install "${apt_packages[@]}" + +dpkg-reconfigure locales + +# pip 19.3 is causing errors building the docker image, pin to 19.2.3 for now. +# See https://github.com/pypa/pip/issues/7206 +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +# clean up +apt-get autoremove + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/dot-config/pip/pip.conf b/taskcluster/docker/recipes/dot-config/pip/pip.conf new file mode 100644 index 0000000000..73c2b2a52c --- /dev/null +++ b/taskcluster/docker/recipes/dot-config/pip/pip.conf @@ -0,0 +1,2 @@ +[global] +disable-pip-version-check = true diff --git a/taskcluster/docker/recipes/hgrc b/taskcluster/docker/recipes/hgrc new file mode 100755 index 0000000000..f6a2f6643c --- /dev/null +++ b/taskcluster/docker/recipes/hgrc @@ -0,0 +1,33 @@ +# By default the progress bar starts after 3s and updates every 0.1s. We +# change this so it shows and updates every 1.0s. +# We also tell progress to assume a TTY is present so updates are printed +# even if there is no known TTY. +[progress] +delay = 1.0 +refresh = 1.0 +assume-tty = true + +[extensions] +share = +sparse = +robustcheckout = /usr/local/mercurial/robustcheckout.py + +[hostsecurity] +# When running a modern Python, Mercurial will default to TLS 1.1+. +# When running on a legacy Python, Mercurial will default to TLS 1.0+. +# There is no good reason we shouldn't be running a modern Python +# capable of speaking TLS 1.2. And the only Mercurial servers we care +# about should be running TLS 1.2. So make TLS 1.2 the minimum. +minimumprotocol = tls1.2 + +# Settings to make 1-click loaners more useful. +[extensions] +histedit = +rebase = + +[diff] +git = 1 +showfunc = 1 + +[pager] +pager = LESS=FRSXQ less diff --git a/taskcluster/docker/recipes/install-node.sh b/taskcluster/docker/recipes/install-node.sh new file mode 100644 index 0000000000..daeba2edc9 --- /dev/null +++ b/taskcluster/docker/recipes/install-node.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script installs Node v16. +# XXX For now, this should match the version installed in +# taskcluster/scripts/misc/repack-node.sh. Later we'll get the ESLint builder +# to use the linux64-node toolchain directly. + +wget -O node.xz --progress=dot:mega https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz +echo 'c88b52497ab38a3ddf526e5b46a41270320409109c3f74171b241132984fd08f' node.xz | sha256sum -c +tar -C /usr/local -xJ --strip-components 1 < node.xz +node -v # verify +npm -v diff --git a/taskcluster/docker/recipes/setup_packages.sh b/taskcluster/docker/recipes/setup_packages.sh new file mode 100755 index 0000000000..48dbf493f3 --- /dev/null +++ b/taskcluster/docker/recipes/setup_packages.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +TASKCLUSTER_ROOT_URL=$1 +shift + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + + +for task in "$@"; do + echo "adding package source $queue_base/task/$task/artifacts/public/build/" + echo "deb [trusted=yes] $queue_base/task/$task/artifacts/public/build/ apt/" > "/etc/apt/sources.list.d/99$task.list" +done diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh new file mode 100644 index 0000000000..fd2f9634e8 --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +# We do want to install recommended packages. +sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +# To speed up docker image build times as well as number of network/disk I/O +# build a list of packages to be installed nad call it in one go. +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('ccache') +apt_packages+=('compiz') +apt_packages+=('curl') +apt_packages+=('fontconfig') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('git') +apt_packages+=('gnome-icon-theme') +apt_packages+=('gstreamer1.0-gtk3') +apt_packages+=('gstreamer1.0-plugins-base') +apt_packages+=('gstreamer1.0-plugins-good') +apt_packages+=('gstreamer1.0-tools') +apt_packages+=('gstreamer1.0-pulseaudio') +apt_packages+=('language-pack-en-base') +apt_packages+=('libc6-dbg') +apt_packages+=('libasound2-dev') +apt_packages+=('libavcodec-extra57') +apt_packages+=('libcanberra-gtk3-module') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libfreetype6') +apt_packages+=('libgconf2-dev') +apt_packages+=('libgl1-mesa-dri') +apt_packages+=('libgl1-mesa-glx') +apt_packages+=('libgstreamer-plugins-base1.0-dev') +apt_packages+=('libgstreamer1.0-dev') +apt_packages+=('libgtk-3-0') +apt_packages+=('libiw-dev') +apt_packages+=('libx11-xcb1') +apt_packages+=('libxcb1') +apt_packages+=('libxcb-render0') +apt_packages+=('libxcb-shm0') +apt_packages+=('libxcb-glx0') +apt_packages+=('libxcb-shape0') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse0') +apt_packages+=('libpulse-dev') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('mesa-common-dev') +apt_packages+=('net-tools') +apt_packages+=('pulseaudio') +apt_packages+=('pulseaudio-module-bluetooth') +apt_packages+=('pulseaudio-module-gconf') +apt_packages+=('python-dev') +apt_packages+=('python-pip') +apt_packages+=('python3-pip') +apt_packages+=('python3-pyatspi') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('ttf-dejavu') +apt_packages+=('ubuntu-desktop') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xvfb') +apt_packages+=('xwit') +apt_packages+=('yasm') +apt_packages+=('zip') + +# xvinfo for test-linux.sh to monitor Xvfb startup +apt_packages+=('x11-utils') + +# Bug 1232407 - this allows the user to start vnc +apt_packages+=('x11vnc') + +# Bug 1176031 - need `xset` to disable screensavers +apt_packages+=('x11-xserver-utils') + +# Build a list of packages to install from the multiverse repo. +apt_packages+=('ubuntu-restricted-extras') + +# APT update takes very long on Ubuntu. Run it at the last possible minute. +apt-get update + +# Also force the cleanup after installation of packages to reduce image size. +apt-get install --allow-downgrades "${apt_packages[@]}" + +# Enable i386 packages +dpkg --add-architecture i386 +apt-get update + +# Make sure we have libraries for 32-bit tests +apt_packages=() +apt_packages+=('libavcodec-extra57:i386') +apt_packages+=('libpulse0:i386') +apt_packages+=('libxt6:i386') +apt_packages+=('libxtst6:i386') +apt_packages+=('libsecret-1-0:i386') +apt_packages+=('libgtk-3-0:i386') +apt_packages+=('libx11-xcb1:i386') +apt_packages+=('libxcb1:i386') +apt_packages+=('libasound2:i386') + +apt-get install --allow-downgrades "${apt_packages[@]}" +rm -rf /var/lib/apt/lists/* + +# Build a list of packages to purge from the image. +apt_packages=() +apt_packages+=('*cheese*') +apt_packages+=('example-content') +apt_packages+=('git') +apt_packages+=('gnome-calendar') +apt_packages+=('gnome-initial-setup') +apt_packages+=('gnome-mahjongg') +apt_packages+=('gnome-mines') +apt_packages+=('gnome-sudoku') +apt_packages+=('libx11-doc') +apt_packages+=('manpages-dev') +apt_packages+=('orca') +apt_packages+=('rhythmbox') +apt_packages+=('thunderbird') +apt_packages+=('ubuntu-release-upgrader*') +apt_packages+=('update-manager-core') +apt_packages+=('update-manager') +apt_packages+=('yelp') + +# Purge unnecessary packages +apt-get purge "${apt_packages[@]}" + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +# Remove all locale files other than en_US.UTF-8 +rm -rf /usr/share/locale/ /usr/share/locale-langpack/ /usr/share/locales/ + +# Further cleanup +apt-get autoremove + +rm -f "$0" diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh new file mode 100644 index 0000000000..e22b21f51d --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +cd /setup + +# Install tooltool and node now that dependencies are in place. +. /setup/common.sh +. /setup/install-node.sh + +# Upgrade pip and install virtualenv to specified versions. +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +pip3 install -r /setup/psutil_requirements.txt +pip install -r /setup/psutil_requirements.txt + +# Cleanup +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/xvfb.sh b/taskcluster/docker/recipes/xvfb.sh new file mode 100644 index 0000000000..6e0e79f7d4 --- /dev/null +++ b/taskcluster/docker/recipes/xvfb.sh @@ -0,0 +1,75 @@ +#! /bin/bash -x + +set -x + +fail() { + echo # make sure error message is on a new line + echo "[xvfb.sh:error]" "${@}" + exit 1 +} + +cleanup_xvfb() { + # When you call this script with START_VNC or TASKCLUSTER_INTERACTIVE + # we make sure we do not kill xvfb so you do not lose your connection + local xvfb_pid=`pidof Xvfb` + local vnc=${START_VNC:-false} + local interactive=${TASKCLUSTER_INTERACTIVE:-false} + if [ -n "$xvfb_pid" ] && [[ $vnc == false ]] && [[ $interactive == false ]] ; then + kill $xvfb_pid || true + screen -XS xvfb quit || true + fi +} + +# Attempt to start xvfb in a screen session with the given resolution and display +# number. Up to 5 attempts will be made to start xvfb with a short delay +# between retries +try_xvfb() { + screen -dmS xvfb Xvfb :$2 -nolisten tcp -screen 0 $1 \ + > ~/artifacts/xvfb/xvfb.log 2>&1 + export DISPLAY=:$2 + + # Only error code 255 matters, because it signifies that no + # display could be opened. As long as we can open the display + # tests should work. We'll retry a few times with a sleep before + # failing. + local retry_count=0 + local max_retries=5 + xvfb_test=0 + until [ $retry_count -gt $max_retries ]; do + xvinfo || xvfb_test=$? + if [ $xvfb_test != 255 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + echo "Failed to start Xvfb, retry: $retry_count" + sleep 2 + fi + done + if [ $xvfb_test == 255 ]; then + return 1 + else + return 0 + fi +} + +start_xvfb() { + set +e + mkdir -p ~/artifacts/xvfb + local retry_count=0 + local max_retries=2 + local success=1 + until [ $retry_count -gt $max_retries ]; do + try_xvfb $1 $2 + success=$? + if [ $success -eq 0 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + sleep 10 + fi + done + set -e + if [ $success -eq 1 ]; then + fail "Could not start xvfb after ${max_retries} attempts" + fi +} diff --git a/taskcluster/docker/sentry/Dockerfile b/taskcluster/docker/sentry/Dockerfile new file mode 100644 index 0000000000..2f606c5529 --- /dev/null +++ b/taskcluster/docker/sentry/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +LABEL maintainer="Mitchell Hentges mhentges@mozilla.com" + +VOLUME /builds/worker/checkouts + +ADD prepare.sh /setup/prepare-docker.sh +ADD submit_sentry_release.sh /usr/bin/submit_sentry_release.sh +RUN /bin/bash /setup/prepare-docker.sh && rm -R /setup + +# Set a default command useful for debugging +CMD ["/bin/sh", "--login"] diff --git a/taskcluster/docker/sentry/prepare.sh b/taskcluster/docker/sentry/prepare.sh new file mode 100755 index 0000000000..7cce3e1955 --- /dev/null +++ b/taskcluster/docker/sentry/prepare.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o xtrace + +apt-get -y update + +# Install: +# * curl to fetch sentry-cli +# * jq to parse hgmo pushlog +apt-get install -y \ + curl \ + jq + +# Install sentry-cli to publish releases +curl -L https://github.com/getsentry/sentry-cli/releases/download/1.63.1/sentry-cli-Linux-x86_64 -o /usr/bin/sentry-cli +chmod +x /usr/bin/sentry-cli diff --git a/taskcluster/docker/sentry/submit_sentry_release.sh b/taskcluster/docker/sentry/submit_sentry_release.sh new file mode 100644 index 0000000000..136e06a58a --- /dev/null +++ b/taskcluster/docker/sentry/submit_sentry_release.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -o nounset +set -o pipefail + +run() { + revisions=$(curl "$HG_PUSHLOG_URL" | jq -c -r ".pushes[].changesets | @sh" | tr -d \') || return 1 + sentry_api_key=$(curl "http://taskcluster/secrets/v1/secret/$SENTRY_SECRET" | jq -r ".secret.sentryToken") || return 1 + for revision in $revisions; do + SENTRY_AUTH_TOKEN=$sentry_api_key SENTRY_ORG=mozilla sentry-cli --url https://sentry.io/ releases --project mach new "hg-rev-$revision" || return 1 + done +} + +with_backoff() { + failures=0 + while ! "$@"; do + failures=$(( failures + 1 )) + if (( failures >= 5 )); then + echo "[with_backoff] Unable to succeed after 5 tries, failing the job." + return 1 + else + seconds=$(( 2 ** (failures - 1) )) + echo "[with_backoff] Retrying in $seconds second(s)" + sleep $seconds + fi + done +} + +with_backoff run diff --git a/taskcluster/docker/snap-coreXX-build/Dockerfile b/taskcluster/docker/snap-coreXX-build/Dockerfile new file mode 100644 index 0000000000..29a343f6ea --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/Dockerfile @@ -0,0 +1,80 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gabriele Svelto + +RUN mkdir -p /builds +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/checkouts + +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + libavcodec58 \ + libavutil56 \ + jq \ + patch \ + patchelf \ + python3-dev \ + python3-yaml \ + squashfs-tools \ + tar \ + unzip \ + uuid \ + wget \ + zip + +ARG SNAP_BASE + +COPY install-snap.sh /usr/local/bin/ + +RUN install-snap.sh core + +RUN install-snap.sh core20 + +# Snapcraft snap depends on core20 for python3.8 even when we target core22 +RUN install-snap.sh snapcraft + +RUN install-snap.sh core22 + +ARG SNAP_LIST + +RUN for snap in $SNAP_LIST; do install-snap.sh "${snap}"; done + +# Create a snapcraft runner +RUN mkdir -p /snap/bin +RUN echo "#!/bin/sh" > /snap/bin/snapcraft +RUN snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml)" && echo "export SNAP_VERSION=\"$snap_version\"\nexport PATH=\$SNAP/bin/:\$SNAP/libexec/snapcraft/:\$PATH" >> /snap/bin/snapcraft +RUN echo 'exec "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft +RUN chmod +x /snap/bin/snapcraft + +# Generate locale +RUN apt update && apt dist-upgrade --yes && apt install --yes sudo snapd locales && locale-gen en_US.UTF-8 && apt-get autoremove -y + +# Set the proper environment +ENV LANG="en_US.UTF-8" +ENV LANGUAGE="en_US:en" +ENV LC_ALL="en_US.UTF-8" +ENV PATH="/snap/bin:$PATH" +ENV SNAP="/snap/snapcraft/current" +ENV SNAP_NAME="snapcraft" +ENV SNAP_ARCH="amd64" + +# Snapcraft pull will need to sudo apt-get update ... +RUN echo "worker ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/worker && \ + chmod 0440 /etc/sudoers.d/worker + +COPY run.sh /builds/worker/run.sh + +COPY parse.py /builds/worker/parse.py + +COPY patches /builds/worker/patches/ + +RUN chown -R worker:worker /builds/worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/snap-coreXX-build/install-snap.sh b/taskcluster/docker/snap-coreXX-build/install-snap.sh new file mode 100755 index 0000000000..66f64d088d --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/install-snap.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +SNAP_TO_INSTALL=$1 + +if [ -z "${SNAP_TO_INSTALL}" ]; then + echo "Please give a snap name" + exit 1 +fi + +# Grab the requested snap from the stable channel and unpack it in the proper +# place (the 'Snap-CDN: none' header allows building in restricted network +# environments such as Launchpad builders) + +# shellcheck disable=SC2046 +curl -L \ + -H 'Snap-CDN: none' \ + $(curl -H 'X-Ubuntu-Series: 16' "https://api.snapcraft.io/api/v1/snaps/details/${SNAP_TO_INSTALL}?channel=stable" | jq '.download_url' -r) \ + --output "${SNAP_TO_INSTALL}.snap" + +mkdir -p "/snap/${SNAP_TO_INSTALL}" + +unsquashfs -d "/snap/${SNAP_TO_INSTALL}/current" "${SNAP_TO_INSTALL}.snap" diff --git a/taskcluster/docker/snap-coreXX-build/parse.py b/taskcluster/docker/snap-coreXX-build/parse.py new file mode 100644 index 0000000000..82e0e2f4a6 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/parse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import sys + +import yaml + + +def has_pkg_section(p, section): + has_section = section in p.keys() + if has_section: + for pkg in p[section]: + yield pkg + + +def iter_pkgs(part, all_pkgs): + for pkg in has_pkg_section(part, "build-packages"): + if pkg not in all_pkgs: + all_pkgs.append(pkg) + for pkg in has_pkg_section(part, "stage-packages"): + if pkg not in all_pkgs: + all_pkgs.append(pkg) + + +def parse(yaml_file): + all_pkgs = [] + with open(yaml_file, "r") as inp: + snap = yaml.safe_load(inp) + parts = snap["parts"] + for p in parts: + iter_pkgs(parts[p], all_pkgs) + return " ".join(all_pkgs) + + +if __name__ == "__main__": + print(parse(sys.argv[1])) diff --git a/taskcluster/docker/snap-coreXX-build/run.sh b/taskcluster/docker/snap-coreXX-build/run.sh new file mode 100755 index 0000000000..5b3efbb756 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/run.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -ex + +mkdir -p /builds/worker/artifacts/ +mkdir -p /builds/worker/.local/state/snapcraft/ +ln -s /builds/worker/artifacts /builds/worker/.local/state/snapcraft/log + +BRANCH=$1 +DEBUG=${2:-0} + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +export SNAP_ARCH=amd64 +export SNAPCRAFT_BUILD_INFO=1 + +export PATH=$PATH:$HOME/.local/bin/ +unset MOZ_AUTOMATION + +MOZCONFIG=mozconfig.in + +USE_SNAP_FROM_STORE=${USE_SNAP_FROM_STORE:-0} + +TRY=0 +if [ "${BRANCH}" = "try" ]; then + BRANCH=nightly + TRY=1 +fi + +if [ "${USE_SNAP_FROM_STORE}" = "0" ]; then + # ESR currently still has a hard dependency against zstandard==0.17.0 so + # install this specific version here + if [ "${BRANCH}" = "esr" ]; then + sudo apt-get remove -y python3-zstandard && sudo apt-get install -y python3-pip && sudo pip3 install --no-input zstandard==0.17.0 + MOZCONFIG=mozconfig + fi + + # Stable and beta runs out of file descriptors during link with gold + ulimit -n 65536 + + git clone --single-branch --depth 1 --branch "${BRANCH}" https://github.com/canonical/firefox-snap/ + cd firefox-snap/ + + if [ "${TRY}" = "1" ]; then + # Symlink so that we can directly re-use Gecko mercurial checkout + ln -s /builds/worker/checkouts/gecko gecko + fi + + # Force an update to avoid the case of a stale docker image and repos updated + # after + sudo apt-get update + + # shellcheck disable=SC2046 + sudo apt-get install -y $(/usr/bin/python3 /builds/worker/parse.py snapcraft.yaml) + + # CRAFT_PARTS_PACKAGE_REFRESH required to avoid snapcraft running apt-get update + # especially for stage-packages + if [ -d "/builds/worker/patches/${BRANCH}/" ]; then + for p in /builds/worker/patches/"${BRANCH}"/*.patch; do + patch -p1 < "$p" + done; + fi + + if [ "${TRY}" = "1" ]; then + # don't remove hg source, and don't force changeset so we get correct stamp + # still force repo because the try clone is from mozilla-unified but the + # generated link does not work + sed -ri 's|rm -rf .hg||g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|MOZ_SOURCE_REPO=\$\{REPO\}|MOZ_SOURCE_REPO=${GECKO_HEAD_REPOSITORY}|g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|MOZ_SOURCE_CHANGESET=\$\{REVISION\}|MOZ_SOURCE_CHANGESET=${GECKO_HEAD_REV}|g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|hg clone --stream \$REPO -u \$REVISION|cp -r \$SNAPCRAFT_PROJECT_DIR/gecko/. |g' snapcraft.yaml + fi + + if [ "${DEBUG}" = "1" ]; then + { + echo "ac_add_options --enable-debug" + echo "ac_add_options --disable-install-strip" + } >> ${MOZCONFIG} + echo "MOZ_DEBUG=1" >> ${MOZCONFIG} + + # No PGO on debug builds + sed -ri 's/ac_add_options --enable-linker=gold//g' snapcraft.yaml + sed -ri 's/ac_add_options --enable-lto=cross//g' snapcraft.yaml + sed -ri 's/ac_add_options MOZ_PGO=1//g' snapcraft.yaml + fi + + SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY=64G \ + SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \ + CRAFT_PARTS_PACKAGE_REFRESH=0 \ + snapcraft --destructive-mode --verbose +else + mkdir from-snap-store && cd from-snap-store + + CHANNEL="${BRANCH}" + if [ "${CHANNEL}" = "try" ] || [ "${CHANNEL}" = "nightly" ]; then + CHANNEL=edge + fi; + + snap download --channel="${CHANNEL}" firefox + SNAP_DEBUG_NAME=$(find . -maxdepth 1 -type f -name "firefox*.snap" | sed -e 's/\.snap$/.debug/g') + touch "${SNAP_DEBUG_NAME}" +fi + +cp ./*.snap ./*.debug /builds/worker/artifacts/ + +# Those are for fetches usage by the test task +cp ./*.snap /builds/worker/artifacts/firefox.snap +cp ./*.debug /builds/worker/artifacts/firefox.debug + +# Those are for running snap-upstream-test +cd /builds/worker/checkouts/gecko/taskcluster/docker/snap-coreXX-build/snap-tests/ && zip -r9 /builds/worker/artifacts/snap-tests.zip ./* diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md new file mode 100644 index 0000000000..73f78c72bc --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md @@ -0,0 +1,24 @@ +Debugging tests +================ + +You can use the `TEST_FILTER` environment variable, e.g., `TEST_FILTER=xxx` +will filter test named `test_xxx`. + +Setting `TEST_GECKODRIVER_TRACE` to any value will make Selenium dump a trace +log for debugging. + +You can control running headless or not with `TEST_NO_HEADLESS`. Currently, +the copy/paste image test required NOT to run headless. + +More useful for local repro, you can set `TEST_NO_QUIT` if you need to keep +inspecting the browser at the end of a test. + +Data URL containing the diff screenshot will be dumped to stdout/stderr when +`TEST_DUMP_DIFF` is set in the environment. + +Updating reference screenshots +============================== + - `./mach try fuzzy --push-to-lando --full --env TEST_COLLECT_REFERENCE=1 -q "'snap-upstream-test"` + - note the successfull task id you want to source + - you need curl and jq installed + - ./update-references.sh diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py new file mode 100644 index 0000000000..fb8841817b --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + + +import base64 +import io +import json +import os +import sys +import time +import traceback + +from mozlog import formatters, handlers, structuredlog +from PIL import Image, ImageChops +from selenium import webdriver +from selenium.common.exceptions import TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.remote.webelement import WebElement +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + + +class SnapTestsBase: + def __init__(self, exp): + driver_service = Service( + executable_path=r"/snap/firefox/current/usr/lib/firefox/geckodriver", + log_output=os.path.join( + os.environ.get("ARTIFACT_DIR", ""), "geckodriver.log" + ), + ) + options = Options() + if "TEST_GECKODRIVER_TRACE" in os.environ.keys(): + options.log.level = "trace" + options.binary_location = r"/snap/firefox/current/usr/lib/firefox/firefox" + if not "TEST_NO_HEADLESS" in os.environ.keys(): + options.add_argument("--headless") + if "MOZ_AUTOMATION" in os.environ.keys(): + os.environ["MOZ_LOG_FILE"] = os.path.join( + os.environ.get("ARTIFACT_DIR"), "gecko.log" + ) + self._driver = webdriver.Firefox(service=driver_service, options=options) + + self._logger = structuredlog.StructuredLogger(self.__class__.__name__) + self._logger.add_handler( + handlers.StreamHandler(sys.stdout, formatters.TbplFormatter()) + ) + + test_filter = "test_{}".format(os.environ.get("TEST_FILTER", "")) + object_methods = [ + method_name + for method_name in dir(self) + if callable(getattr(self, method_name)) + and method_name.startswith(test_filter) + ] + + self._logger.suite_start(object_methods) + + assert self._dir is not None + + self._wait = WebDriverWait(self._driver, self.get_timeout()) + self._longwait = WebDriverWait(self._driver, 60) + + with open(exp, "r") as j: + self._expectations = json.load(j) + + rv = False + try: + first_tab = self._driver.window_handles[0] + for m in object_methods: + tabs_before = set(self._driver.window_handles) + self._driver.switch_to.window(first_tab) + self._logger.test_start(m) + rv = getattr(self, m)(self._expectations[m]) + self._driver.switch_to.parent_frame() + if rv: + self._logger.test_end(m, status="OK") + else: + self._logger.test_end(m, status="FAIL") + tabs_after = set(self._driver.window_handles) + tabs_opened = tabs_after - tabs_before + self._logger.info("opened {} tabs".format(len(tabs_opened))) + for tab in tabs_opened: + self._driver.switch_to.window(tab) + self._driver.close() + self._wait.until(EC.number_of_windows_to_be(len(tabs_before))) + except Exception as ex: + rv = False + test_status = "ERROR" + if isinstance(ex, AssertionError): + test_status = "FAIL" + elif isinstance(ex, TimeoutException): + test_status = "TIMEOUT" + + test_message = repr(ex) + self.save_screenshot("screenshot_{}.png".format(test_status.lower())) + self._driver.switch_to.parent_frame() + self.save_screenshot("screenshot_{}_parent.png".format(test_status.lower())) + self._logger.test_end(m, status=test_status, message=test_message) + traceback.print_exc() + finally: + self._driver.switch_to.window(first_tab) + + if not "TEST_NO_QUIT" in os.environ.keys(): + self._driver.quit() + + self._logger.info("Exiting with {}".format(rv)) + self._logger.suite_end() + sys.exit(0 if rv is True else 1) + + def get_screenshot_destination(self, name): + final_name = name + if "MOZ_AUTOMATION" in os.environ.keys(): + final_name = os.path.join(os.environ.get("ARTIFACT_DIR"), name) + return final_name + + def save_screenshot(self, name): + final_name = self.get_screenshot_destination(name) + self._logger.info("Saving screenshot '{}' to '{}'".format(name, final_name)) + self._driver.save_screenshot(final_name) + + def get_timeout(self): + if "TEST_TIMEOUT" in os.environ.keys(): + return int(os.getenv("TEST_TIMEOUT")) + else: + return 5 + + def maybe_collect_reference(self): + return "TEST_COLLECT_REFERENCE" in os.environ.keys() + + def open_tab(self, url): + opened_tabs = len(self._driver.window_handles) + + self._driver.switch_to.new_window("tab") + self._wait.until(EC.number_of_windows_to_be(opened_tabs + 1)) + self._driver.get(url) + + return self._driver.current_window_handle + + def assert_rendering(self, exp, element_or_driver): + # wait a bit for things to settle down + time.sleep(0.5) + + # Convert as RGB otherwise we cannot get difference + png_bytes = ( + element_or_driver.screenshot_as_png + if isinstance(element_or_driver, WebElement) + else element_or_driver.get_screenshot_as_png() + ) + svg_png = Image.open(io.BytesIO(png_bytes)).convert("RGB") + svg_png_cropped = svg_png.crop((0, 0, svg_png.width - 20, svg_png.height - 20)) + + if self.maybe_collect_reference(): + new_ref = "new_{}".format(exp["reference"]) + new_ref_file = self.get_screenshot_destination(new_ref) + self._logger.info( + "Collecting new reference screenshot: {} => {}".format( + new_ref, new_ref_file + ) + ) + + with open(new_ref_file, "wb") as current_screenshot: + svg_png_cropped.save(current_screenshot) + + return + + svg_ref = Image.open(os.path.join(self._dir, exp["reference"])).convert("RGB") + diff = ImageChops.difference(svg_ref, svg_png_cropped) + + if diff.getbbox() is not None: + buffered = io.BytesIO() + diff.save(buffered, format="PNG") + + if "TEST_DUMP_DIFF" in os.environ.keys(): + diff_b64 = base64.b64encode(buffered.getvalue()) + self._logger.info( + "data:image/png;base64,{}".format(diff_b64.decode("utf-8")) + ) + + with open( + self.get_screenshot_destination("differences.png"), "wb" + ) as diff_screenshot: + diff_screenshot.write(buffered.getvalue()) + + with open( + self.get_screenshot_destination("current_rendering.png"), "wb" + ) as current_screenshot: + svg_png_cropped.save(current_screenshot) + + assert diff.getbbox() is None, "Mismatching screenshots for {}".format( + exp["reference"] + ) + + +class SnapTests(SnapTestsBase): + def __init__(self, exp): + self._dir = "basic_tests" + super(SnapTests, self).__init__(exp) + + def test_about_support(self, exp): + self.open_tab("about:support") + + version_box = self._wait.until( + EC.visibility_of_element_located((By.ID, "version-box")) + ) + self._wait.until(lambda d: len(version_box.text) > 0) + self._logger.info("about:support version: {}".format(version_box.text)) + assert version_box.text == exp["version_box"], "version text should match" + + distributionid_box = self._wait.until( + EC.visibility_of_element_located((By.ID, "distributionid-box")) + ) + self._wait.until(lambda d: len(distributionid_box.text) > 0) + self._logger.info( + "about:support distribution ID: {}".format(distributionid_box.text) + ) + assert ( + distributionid_box.text == exp["distribution_id"] + ), "distribution_id should match" + + windowing_protocol = self._driver.execute_script( + "return document.querySelector('th[data-l10n-id=\"graphics-window-protocol\"').parentNode.lastChild.textContent;" + ) + self._logger.info( + "about:support windowing protocol: {}".format(windowing_protocol) + ) + assert windowing_protocol == "wayland", "windowing protocol should be wayland" + + return True + + def test_about_buildconfig(self, exp): + self.open_tab("about:buildconfig") + + source_link = self._wait.until( + EC.visibility_of_element_located((By.CSS_SELECTOR, "a")) + ) + self._wait.until(lambda d: len(source_link.text) > 0) + self._logger.info("about:buildconfig source: {}".format(source_link.text)) + assert source_link.text.startswith( + exp["source_repo"] + ), "source repo should exists and match" + + build_flags_box = self._wait.until( + EC.visibility_of_element_located((By.CSS_SELECTOR, "p:last-child")) + ) + self._wait.until(lambda d: len(build_flags_box.text) > 0) + self._logger.info("about:support buildflags: {}".format(build_flags_box.text)) + assert ( + build_flags_box.text.find(exp["official"]) >= 0 + ), "official build flag should be there" + + return True + + def test_youtube(self, exp): + self.open_tab("https://www.youtube.com") + + # Wait for the consent dialog and accept it + self._logger.info("Wait for consent form") + try: + self._wait.until( + EC.visibility_of_element_located( + (By.CSS_SELECTOR, "button[aria-label*=Accept]") + ) + ).click() + except TimeoutException: + self._logger.info("Wait for consent form: timed out, maybe it is not here") + + try: + # Find first video and click it + self._logger.info("Wait for one video") + self._wait.until( + EC.visibility_of_element_located((By.ID, "video-title-link")) + ).click() + except TimeoutException: + # We might have got the "try searching to get started" + # link to News channel + self._driver.get("https://www.youtube.com/channel/UCYfdidRxbB8Qhf0Nx7ioOYw") + self._logger.info("Wait again for one video") + self._wait.until( + EC.visibility_of_element_located((By.ID, "video-title-link")) + ).click() + + # Wait for duration to be set to something + self._logger.info("Wait for video to start") + video = self._wait.until( + EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video")) + ) + self._wait.until(lambda d: type(video.get_property("duration")) == float) + self._logger.info("video duration: {}".format(video.get_property("duration"))) + assert ( + video.get_property("duration") > exp["duration"] + ), "youtube video should have duration" + + self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) + self._logger.info("video played: {}".format(video.get_property("currentTime"))) + assert ( + video.get_property("currentTime") > exp["playback"] + ), "youtube video should perform playback" + + return True + + +if __name__ == "__main__": + SnapTests(exp=sys.argv[1]) diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in new file mode 100644 index 0000000000..92e26dd699 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in @@ -0,0 +1,14 @@ +{ + "test_about_support": { + "version_box": "#RUNTIME_VERSION#", + "distribution_id": "canonical-002" + }, + "test_about_buildconfig": { + "source_repo": "https://hg.mozilla.org/", + "official": "MOZILLA_OFFICIAL=1" + }, + "test_youtube": { + "duration": 1, + "playback": 2 + } +} diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py new file mode 100644 index 0000000000..781000981e --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + + +import os +import random +import tempfile +import time + +from basic_tests import SnapTestsBase +from selenium.common.exceptions import StaleElementReferenceException, TimeoutException +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.select import Select + + +class QATests(SnapTestsBase): + def __init__(self): + self._dir = "qa_tests" + + super(QATests, self).__init__( + exp=os.path.join(self._dir, "qa_expectations.json") + ) + + def _test_audio_playback( + self, url, iframe_selector=None, click_to_play=False, video_selector=None + ): + self._logger.info("open url {}".format(url)) + if url: + self.open_tab(url) + + if iframe_selector: + self._logger.info("find iframe") + iframe = self._driver.find_element(By.CSS_SELECTOR, iframe_selector) + self._driver.switch_to.frame(iframe) + + self._logger.info("find video") + video = self._wait.until( + EC.visibility_of_element_located( + (By.CSS_SELECTOR, video_selector or "video") + ) + ) + self._wait.until(lambda d: type(video.get_property("duration")) == float) + assert video.get_property("duration") > 0.0, "