diff options
Diffstat (limited to 'comm/taskcluster')
173 files changed, 12362 insertions, 0 deletions
diff --git a/comm/taskcluster/.yamllint b/comm/taskcluster/.yamllint new file mode 100644 index 0000000000..0c454fb02a --- /dev/null +++ b/comm/taskcluster/.yamllint @@ -0,0 +1,12 @@ +--- +extends: default + +rules: + document-end: + present: false + document-start: + present: true + indentation: + spaces: 4 + # Checks currently failing + line-length: disable diff --git a/comm/taskcluster/ci/are-we-esmified-yet/kind.yml b/comm/taskcluster/ci/are-we-esmified-yet/kind.yml new file mode 100644 index 0000000000..f6001d5200 --- /dev/null +++ b/comm/taskcluster/ci/are-we-esmified-yet/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + check: + description: Extracts statistics on remaining JSMs in the codebase + run-on-projects: [] + attributes: + cron: true + index: + product: thunderbird + job-name: are-we-esmified-yet + treeherder: + platform: lint/opt + kind: other + symbol: js(esm) + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 3600 + artifacts: + - type: file + name: public/are-we-esmified-yet.json + path: /builds/worker/artifacts/are-we-esmified-yet.json + run: + using: run-task + comm-checkout: true + command: >- + mkdir -p /builds/worker/artifacts && + cd ${GECKO_PATH}/comm && + python3 ./taskcluster/scripts/are-we-esmified-yet.py > /builds/worker/artifacts/are-we-esmified-yet.json diff --git a/comm/taskcluster/ci/balrog/kind.yml b/comm/taskcluster/ci/balrog/kind.yml new file mode 100644 index 0000000000..cbe3cbaef0 --- /dev/null +++ b/comm/taskcluster/ci/balrog/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.balrog_submit:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-repackage + +only-for-attributes: + - shippable diff --git a/comm/taskcluster/ci/beetmover-checksums/kind.yml b/comm/taskcluster/ci/beetmover-checksums/kind.yml new file mode 100644 index 0000000000..87e688403d --- /dev/null +++ b/comm/taskcluster/ci/beetmover-checksums/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.beetmover_checksums:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-repackage + +only-for-attributes: + - shippable + +job-template: + shipping-phase: promote + attributes: + artifact_prefix: public + artifact_map: + by-release-type: + beta|release: comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml + default: comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml diff --git a/comm/taskcluster/ci/beetmover-repackage/kind.yml b/comm/taskcluster/ci/beetmover-repackage/kind.yml new file mode 100644 index 0000000000..01ddecb991 --- /dev/null +++ b/comm/taskcluster/ci/beetmover-repackage/kind.yml @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: single-locale + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.beetmover_repackage_l10n:transforms + - gecko_taskgraph.transforms.beetmover_repackage:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build + - build-signing + - build-mac-signing + - build-mac-notarization + - repackage + - repackage-signing + - shippable-l10n + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + - shippable-l10n-signing + - repackage-l10n + - repackage-signing-l10n + - partials + - partials-signing + - repackage-signing-msi + - repackage-signing-shippable-l10n-msix + - mar-signing + - mar-signing-l10n + +primary-dependency: + - repackage + - repackage-l10n + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux64-asan-reporter-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-asan-reporter-shippable/opt + +job-template: + shipping-phase: promote + attributes: + artifact_map: + by-release-type: + beta|release: comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml + default: comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml diff --git a/comm/taskcluster/ci/beetmover-source/kind.yml b/comm/taskcluster/ci/beetmover-source/kind.yml new file mode 100644 index 0000000000..a6b3159f4b --- /dev/null +++ b/comm/taskcluster/ci/beetmover-source/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.beetmover:transforms + - gecko_taskgraph.transforms.beetmover_source:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-source-signing + +job-template: + shipping-phase: promote + attributes: + artifact_map: comm/taskcluster/comm_taskgraph/manifests/source_files.yml diff --git a/comm/taskcluster/ci/beetmover-strings-source/kind.yml b/comm/taskcluster/ci/beetmover-strings-source/kind.yml new file mode 100644 index 0000000000..fae5d05e47 --- /dev/null +++ b/comm/taskcluster/ci/beetmover-strings-source/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.beetmover:transforms + - gecko_taskgraph.transforms.beetmover_source:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - shippable-l10n-pre-signing + +job-template: + shipping-phase: promote + shipping-product: thunderbird + attributes: + artifact_map: comm/taskcluster/comm_taskgraph/manifests/strings_source.yml diff --git a/comm/taskcluster/ci/bouncer-locations/kind.yml b/comm/taskcluster/ci/bouncer-locations/kind.yml new file mode 100644 index 0000000000..be2854e5b0 --- /dev/null +++ b/comm/taskcluster/ci/bouncer-locations/kind.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-beetmover-dummy + +transforms: + - gecko_taskgraph.transforms.bouncer_locations:transforms + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: nightly bouncer locations job + attributes: + shippable: true + worker-type: bouncer + worker: + implementation: bouncer-locations + scopes: + by-project: + comm-central: + - project:comm:thunderbird:releng:bouncer:action:locations + - project:comm:thunderbird:releng:bouncer:server:production + default: + - project:comm:thunderbird:releng:bouncer:action:locations + - project:comm:thunderbird:releng:bouncer:server:staging + shipping-phase: ship + treeherder: + symbol: BncLoc + kind: other + tier: 1 + +jobs: + thunderbird: + run-on-releases: ['nightly'] + run-on-projects: ['comm-central'] + shipping-product: thunderbird + bouncer-products: + - thunderbird-nightly-latest + - thunderbird-nightly-latest-ssl + - thunderbird-nightly-latest-l10n + - thunderbird-nightly-latest-l10n-ssl + - thunderbird-nightly-msi-latest-ssl + - thunderbird-nightly-msi-latest-l10n-ssl + - thunderbird-nightly-pkg-latest-ssl + - thunderbird-nightly-pkg-latest-l10n-ssl + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/build-mac-notarization/kind.yml b/comm/taskcluster/ci/build-mac-notarization/kind.yml new file mode 100644 index 0000000000..fd20f843e4 --- /dev/null +++ b/comm/taskcluster/ci/build-mac-notarization/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.mac_notarization:transforms + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build-mac-signing + +only-for-attributes: + - shippable + +job-template: + treeherder: + symbol: BMN + upstream-artifacts: + - taskType: signing + paths: + - public/build/target.tar.gz + - public/build/target.pkg + formats: ["apple_notarization"] + taskId: + task-reference: <build-mac-signing> diff --git a/comm/taskcluster/ci/build-mac-signing/kind.yml b/comm/taskcluster/ci/build-mac-signing/kind.yml new file mode 100644 index 0000000000..9719a38882 --- /dev/null +++ b/comm/taskcluster/ci/build-mac-signing/kind.yml @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.build_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - build + +only-for-attributes: + - enable-build-signing + +only-for-build-platforms: + - macosx64-shippable/opt + - macosx64/opt + - macosx64/debug + +job-template: + treeherder: + symbol: BMS + enable-signing-routes: false diff --git a/comm/taskcluster/ci/build-signing/kind.yml b/comm/taskcluster/ci/build-signing/kind.yml new file mode 100644 index 0000000000..ec5bd89727 --- /dev/null +++ b/comm/taskcluster/ci/build-signing/kind.yml @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: platform + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.upstream_artifact_task:transforms + - gecko_taskgraph.transforms.build_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - build + +primary-dependency: + - build + +only-for-attributes: + - shippable + - enable-build-signing + +not-for-build-platforms: + - macosx64-shippable/opt + - macosx64/opt + - macosx64/debug diff --git a/comm/taskcluster/ci/build/kind.yml b/comm/taskcluster/ci/build/kind.yml new file mode 100644 index 0000000000..1c1b441996 --- /dev/null +++ b/comm/taskcluster/ci/build/kind.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + - toolchain + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - comm_taskgraph.transforms.tb_build:transforms + - gecko_taskgraph.transforms.artifact:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - linux.yml + - macosx.yml + - windows.yml + +job-defaults: + worker: + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + use-system-python: false + run: + mozconfig-variant: + by-release-type: + beta: release + release.*: release + default: nightly + # Note: These settings are only honored by nightly (i.e. shipping) builds + update-channel: + by-release-type: + nightly: nightly + beta: beta + release.*: release + default: null + mar-channel-id: + by-release-type: + release: thunderbird-comm-release + beta: thunderbird-comm-beta + default: thunderbird-comm-central + accepted-mar-channel-ids: + by-release-type: + release: thunderbird-comm-release + beta: thunderbird-comm-beta,thunderbird-comm-release + default: thunderbird-comm-central diff --git a/comm/taskcluster/ci/build/linux.yml b/comm/taskcluster/ci/build/linux.yml new file mode 100644 index 0000000000..c7898a79e7 --- /dev/null +++ b/comm/taskcluster/ci/build/linux.yml @@ -0,0 +1,405 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + worker-type: b-linux-gcp + run: + using: mozharness + comm-checkout: true + actions: [build] + config-paths: + - comm/testing/mozharness/configs + script: "mozharness/scripts/fx_desktop_build.py" + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-pkgconf + - linux64-rust-size + - sysroot-wasm32-wasi + + +linux64/opt: + description: "Linux64 Opt" + index: + product: thunderbird + job-name: linux64-opt + treeherder: + platform: linux64/opt + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-libotr + - linux64-rust + - linux64-sccache + - sysroot-x86_64-linux-gnu + +linux64-shippable/opt: + description: "Linux64 Shippable" + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: thunderbird + index: + product: thunderbird + job-name: linux64-opt + type: shippable + treeherder: + platform: linux64-shippable/opt + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + run-on-projects: ['release'] + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-libotr + - linux64-rust + - sysroot-x86_64-linux-gnu + +linux64/debug: + description: "Linux64 Debug" + index: + product: thunderbird + job-name: linux64-debug + treeherder: + platform: linux64/debug + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + custom-build-variant-cfg: debug + mozconfig-variant: debug + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-libotr + - linux64-rust + - linux64-sccache + - sysroot-x86_64-linux-gnu + +linux64-asan/opt: + description: "Linux64 Opt ASAN" + index: + product: thunderbird + job-name: linux64-asan-opt + treeherder: + platform: linux64/asan + symbol: Bo + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt asan" + ASAN_OPTIONS: "detect_leaks=0" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + custom-build-variant-cfg: asan-tc + mozconfig-variant: nightly-asan + tooltool-downloads: public + run-on-projects: ['comm-central'] + use-sccache: true + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux64-libotr + - linux64-llvm-symbolizer + - linux64-rust-dev + - linux64-sccache + - sysroot-x86_64-linux-gnu + +linux64-asan-reporter-shippable/opt: + description: "Linux64 Opt ASAN Reporter (Shippable)" + attributes: + shippable: true + shipping-product: + by-release-type: + nightly: thunderbird + default: null + index: + product: thunderbird + job-name: linux64-asan-reporter-opt + type: shippable + treeherder: + platform: linux64-asan-reporter/opt + symbol: BoR + run-on-projects: ['comm-central'] + worker: + env: + PERFHERDER_EXTRA_OPTIONS: asan-reporter + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 3600 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + custom-build-variant-cfg: asan-reporter-tc + mozconfig-variant: nightly-asan-reporter + mar-channel-id: + thunderbird-comm-central-asan + accepted-mar-channel-ids: + thunderbird-comm-central-asan + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux64-libotr + - linux64-llvm-symbolizer + - linux64-rust + - linux64-sccache + - sysroot-x86_64-linux-gnu + +linux64-tsan/opt: + description: "Linux64 Opt TSAN" + index: + product: thunderbird + job-name: linux64-tsan-opt + treeherder: + platform: linux64/tsan + symbol: Bo + worker: + env: + PERFHERDER_EXTRA_OPTIONS: "opt tsan" + LSAN_OPTIONS: "verbosity=1:log_threads=1" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + max-run-time: 5400 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + custom-build-variant-cfg: tsan-tc + mozconfig-variant: tsan + tooltool-downloads: public + run-on-projects: ['comm-central'] + use-sccache: true + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux64-libotr + - linux64-llvm-symbolizer + - linux64-rust-dev + - linux64-sccache + - sysroot-x86_64-linux-gnu + +linux64-aarch64/opt: + description: "Linux64 aarch64 Opt" + index: + product: thunderbird + job-name: linux64-aarch64-opt + attributes: + enable-full-crashsymbols: true + treeherder: + platform: linux64-aarch64/opt + symbol: B + tier: 2 + worker: + max-run-time: 7200 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + extra-config: + mozconfig_platform: linux64-aarch64 + mozconfig-variant: opt + tooltool-downloads: public + run-on-projects: ['comm-central'] + use-sccache: true + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux-aarch64-libotr + - linux64-llvm-symbolizer + - linux64-rust + - linux64-sccache + - sysroot-aarch64-linux-gnu + - sysroot-x86_64-linux-gnu + +linux64-ccov/opt: + description: "Linux64-CCov Opt" + index: + product: thunderbird + job-name: linux64-ccov-opt + treeherder: + platform: linux64-ccov/opt + symbol: B + tier: 2 + run-on-projects: ['comm-central'] + worker: + artifacts: + - name: public/code-coverage-grcov.zip + path: /builds/worker/workspace/obj-build/code-coverage-grcov.zip + type: file + max-run-time: 7200 + env: + RUSTC_BOOTSTRAP: '1' + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + using: mozharness + comm-checkout: true + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + custom-build-variant-cfg: code-coverage-opt + mozconfig-variant: code-coverage-opt + use-sccache: false + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-libotr + - linux64-grcov + - linux64-rust + - sysroot-x86_64-linux-gnu + +linux64-ccov/debug: + description: "Linux64-CCov Debug" + index: + product: thunderbird + job-name: linux64-ccov-debug + treeherder: + platform: linux64-ccov/debug + symbol: B + tier: 2 + run-on-projects: [] + worker: + max-run-time: 7200 + env: + RUSTC_BOOTSTRAP: '1' + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + custom-build-variant-cfg: code-coverage-debug + mozconfig-variant: code-coverage-debug + use-sccache: false + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-libotr + - linux64-grcov + - linux64-rust + - sysroot-x86_64-linux-gnu + +linux/opt: + description: "Linux32 Opt" + index: + product: thunderbird + job-name: linux32-opt + treeherder: + platform: linux32/opt + symbol: B + worker: + max-run-time: 7200 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_32_builds.py + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux32-libotr + - linux64-rust + - linux64-sccache + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + +linux-shippable/opt: + description: "Linux32 Shippable" + attributes: + shippable: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: thunderbird + index: + product: thunderbird + job-name: linux32-opt + type: shippable + treeherder: + platform: linux32-shippable/opt + symbol: B + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_32_builds.py + - taskcluster_nightly.py + run-on-projects: ['release'] + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux32-libotr + - linux64-rust + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu + +linux/debug: + description: "Linux32 Debug" + index: + product: thunderbird + job-name: linux32-debug + treeherder: + platform: linux32/debug + symbol: B + worker: + max-run-time: 7200 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_32_builds.py + custom-build-variant-cfg: debug + mozconfig-variant: debug + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux32-libotr + - linux64-rust + - linux64-sccache + - sysroot-i686-linux-gnu + - sysroot-x86_64-linux-gnu diff --git a/comm/taskcluster/ci/build/macosx.yml b/comm/taskcluster/ci/build/macosx.yml new file mode 100644 index 0000000000..d8510362d4 --- /dev/null +++ b/comm/taskcluster/ci/build/macosx.yml @@ -0,0 +1,288 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + index: + product: thunderbird + worker-type: b-linux-gcp + run: + using: mozharness + comm-checkout: true + actions: [build] + config-paths: + - comm/testing/mozharness/configs + script: "mozharness/scripts/fx_desktop_build.py" + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-cctools-port + - linux64-clang + - linux64-dump_syms + - linux64-hfsplus + - linux64-libdmg + - linux64-nasm + - linux64-node + - linux64-rust-macos + - linux64-rust-size + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + +macosx64/debug: + description: "MacOS X x64 Cross-compile" + index: + job-name: macosx64-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: osx-cross/debug + symbol: B + tier: 1 + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_mac_64_cross_builds.py + custom-build-variant-cfg: cross-debug + mozconfig-variant: debug + use-sccache: true + fetches: + toolchain: + - linux64-sccache + - macosx64-x64-libotr + - macosx64-sdk + +macosx64/opt: + description: "MacOS X x64 Cross-compile" + index: + job-name: macosx64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: osx-cross/opt + symbol: B + tier: 1 + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_mac_64_cross_builds.py + use-sccache: true + fetches: + toolchain: + - linux64-sccache + - macosx64-x64-libotr + - macosx64-sdk + +macosx64-x64-shippable/opt: + description: "MacOS X x64 Cross-compile" + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: thunderbird + index: + product: thunderbird + job-name: macosx64-x64-opt + type: shippable + treeherder: + platform: osx-shippable/opt + symbol: B + tier: 1 + run-on-projects: ['release'] + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + fetches: + toolchain: + - macosx64-x64-libotr + - macosx64-sdk + +macosx64-aarch64-shippable/opt: + description: "MacOS X aarch64 Cross-compile" + attributes: + enable-full-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: thunderbird + index: + product: thunderbird + job-name: macosx64-aarch64-opt + type: shippable + treeherder: + platform: osx-aarch64-shippable/opt + symbol: B + tier: 1 + run-on-projects: ['release'] + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + PERFHERDER_EXTRA_OPTIONS: aarch64 + run: + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_mac_64_cross_builds.py + - taskcluster_nightly.py + extra-config: + mozconfig_platform: macosx64-aarch64 + script: "mozharness/scripts/fx_desktop_build.py" + fetches: + toolchain: + - macosx64-aarch64-libotr + - macosx64-sdk + +macosx64-shippable/opt: + description: "MacOS Universal" + attributes: + shippable: true + enable-build-signing: true + skip-upload-crashsymbols: true + skip-verify-test-packaging: true + shipping-phase: build + shipping-product: thunderbird + index: + product: thunderbird + job-name: macosx64-opt + type: shippable + treeherder: + platform: osx-shippable/opt + symbol: U + tier: 1 + run-on-projects: ['release'] + worker: + max-run-time: 1800 + run: + job-script: taskcluster/scripts/misc/unify.sh + config: + - builds/taskcluster_base_thunderbird.py + dependencies: + macosx64-x64-shippable-opt: build-macosx64-x64-shippable/opt + macosx64-aarch64-shippable-opt: build-macosx64-aarch64-shippable/opt + fetches: + macosx64-x64-shippable-opt: + - artifact: target.dmg + extract: false + dest: x64 + - artifact: buildhub.json + extract: false + dest: ../artifacts + - artifact: host/bin/mar + extract: false + dest: ../artifacts/host/bin + - artifact: host/bin/mbsdiff + extract: false + dest: ../artifacts/host/bin + - artifact: target.txt + extract: false + dest: ../artifacts + - artifact: target_info.txt + extract: false + dest: ../artifacts + - artifact: target.json + extract: false + dest: ../artifacts + - artifact: target.mozinfo.json + extract: false + dest: ../artifacts + - artifact: target.test_packages.json + extract: false + dest: ../artifacts + - artifact: target.perftests.tests.tar.gz + extract: false + dest: ../artifacts + - artifact: target.common.tests.tar.gz + dest: x64/common + - artifact: target.cppunittest.tests.tar.gz + dest: x64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: x64/gtest + - artifact: target.jittest.tests.tar.gz + dest: x64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: x64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: x64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: x64/reftest + - artifact: target.talos.tests.tar.gz + dest: x64/talos + - artifact: target.raptor.tests.tar.gz + dest: x64/raptor + - artifact: target.condprof.tests.tar.gz + dest: x64/confprof + - artifact: target.awsy.tests.tar.gz + dest: x64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: x64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: x64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: x64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: x64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: x64/jsshell + - artifact: target.generated-files.tar.gz + extract: false + dest: ../artifacts + - artifact: target.langpack.xpi + extract: false + dest: ../artifacts + - artifact: mozharness.zip + extract: false + dest: ../artifacts + macosx64-aarch64-shippable-opt: + - artifact: target.dmg + extract: false + dest: aarch64 + - artifact: target.common.tests.tar.gz + dest: aarch64/common + - artifact: target.cppunittest.tests.tar.gz + dest: aarch64/cppunittest + - artifact: target.gtest.tests.tar.gz + dest: aarch64/gtest + - artifact: target.jittest.tests.tar.gz + dest: aarch64/jittest + - artifact: target.jsreftest.tests.tar.gz + dest: aarch64/jsreftest + - artifact: target.mochitest.tests.tar.gz + dest: aarch64/mochitest + - artifact: target.reftest.tests.tar.gz + dest: aarch64/reftest + - artifact: target.talos.tests.tar.gz + dest: aarch64/talos + - artifact: target.raptor.tests.tar.gz + dest: aarch64/raptor + - artifact: target.condprof.tests.tar.gz + dest: aarch64/confprof + - artifact: target.awsy.tests.tar.gz + dest: aarch64/awsy + - artifact: target.xpcshell.tests.tar.gz + dest: aarch64/xpcshell + - artifact: target.web-platform.tests.tar.gz + dest: aarch64/web-platform + - artifact: target.updater-dep.tests.tar.gz + dest: aarch64/updater-dep + - artifact: target.crashreporter-symbols.zip + dest: aarch64/crashreporter-symbols + - artifact: target.jsshell.zip + dest: aarch64/jsshell diff --git a/comm/taskcluster/ci/build/windows.yml b/comm/taskcluster/ci/build/windows.yml new file mode 100644 index 0000000000..6592ccd4aa --- /dev/null +++ b/comm/taskcluster/ci/build/windows.yml @@ -0,0 +1,536 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: mozharness + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + +win32/opt: + description: "Win32 Opt (Cross)" + index: + product: thunderbird + job-name: win32-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-32/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win32-libotr + - vs + fetch: + - upx-3.95-win + +win32-shippable/opt: + description: "Win32 Opt Shippable" + index: + product: thunderbird + job-name: win32-opt + type: shippable + attributes: + shippable: true + enable-build-signing: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: thunderbird + treeherder: + platform: windows2012-32-shippable/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win32 + mozconfig_platform: win32 + run-on-projects: ['release'] + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win32-libotr + - vs + fetch: + - upx-3.95-win + +win32/debug: + description: "Win32 Debug (Cross)" + index: + product: thunderbird + job-name: win32-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-32/debug + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win32/debug.py + mozconfig-variant: debug + extra-config: + mozconfig_platform: win32 + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win32-libotr + - vs + fetch: + - upx-3.95-win + +win64/opt: + description: "Win64 Opt (Cross)" + index: + product: thunderbird + job-name: win64-opt + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-64/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - vs + fetch: + - upx-3.95-win + +win64-shippable/opt: + description: "Win64 Opt Shippable" + index: + product: thunderbird + job-name: win64-opt + type: shippable + attributes: + shippable: true + enable-build-signing: true + enable-full-crashsymbols: true + shipping-phase: build + shipping-product: thunderbird + treeherder: + platform: windows2012-64-shippable/opt + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64 + mozconfig_platform: win64 + run-on-projects: ['release'] + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - vs + fetch: + - upx-3.95-win + +win64/debug: + description: "Win64 Debug (Cross)" + index: + product: thunderbird + job-name: win64-debug + attributes: + enable-build-signing: true + enable-full-crashsymbols: true + treeherder: + platform: windows2012-64/debug + symbol: B + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: cross + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/debug.py + mozconfig-variant: debug + extra-config: + mozconfig_platform: win64 + use-sccache: true + optimization: + skip-suite-only: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - vs + fetch: + - upx-3.95-win + + +win64-plain/debug: + description: "Win64 Debug Plain" + index: + product: thunderbird + job-name: win64-plain-debug + treeherder: + platform: windows2012-64/debug + symbol: Bp + tier: 2 + worker-type: b-win2012 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + extra-config: + disable_package_metrics: true + stage_platform: win64 + mozconfig-variant: plain-debug + run-on-projects: ['comm-central'] + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - nsis + - sysroot-wasm32-wasi + - win64-cbindgen + - win64-clang + - win64-dump_syms + - win64-libotr + - win64-mozmake + - win64-node + - win64-nasm + - win64-rust + - vs + - win64-winchecksec + +win64-plain/opt: + description: "Win64 Opt Plain" + index: + product: thunderbird + job-name: win64-plain-opt + treeherder: + platform: windows2012-64/opt + symbol: Bp + tier: 2 + worker-type: b-win2012 + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: plain + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/taskcluster_base_windows.py + - builds/taskcluster_base_win64.py + extra-config: + disable_package_metrics: true + stage_platform: win64 + mozconfig-variant: plain-opt + run-on-projects: ['comm-central'] + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - nsis + - sysroot-wasm32-wasi + - win64-cbindgen + - win64-clang + - win64-dump_syms + - win64-libotr + - win64-mozmake + - win64-node + - win64-nasm + - win64-rust + - vs + - win64-winchecksec + +win64-asan/opt: + description: "Win64 Opt ASAN" + index: + product: thunderbird + job-name: win64-asan-opt + treeherder: + platform: windows2012-64/asan + symbol: Bo + tier: 1 + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: "opt asan" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + extra-config: + stage_platform: win64-asan + mozconfig_platform: win64 + mozconfig-variant: nightly-asan + run-on-projects: ['comm-central'] + use-sccache: true + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win + +win64-asan-reporter-shippable/opt: + description: "Win64 Opt ASAN Reporter (Shippable)" + attributes: + shippable: true + shipping-product: + by-release-type: + nightly: thunderbird + default: null + index: + product: thunderbird + job-name: win64-asan-reporter-opt + type: shippable + treeherder: + platform: win64-asan-reporter/opt + symbol: BoR + worker-type: b-linux-gcp + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: "asan-reporter" + MOZ_AUTOMATION_PACKAGE_TESTS: "1" + run: + actions: [build] + options: [append-env-variables-from-configs] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/asan_reporter_opt.py + - taskcluster_nightly.py + extra-config: + stage_platform: win64-asan-reporter + mozconfig_platform: win64 + mozconfig-variant: nightly-asan-reporter + mar-channel-id: + thunderbird-comm-central-asan + accepted-mar-channel-ids: + thunderbird-comm-central-asan + run-on-projects: ['comm-central'] + optimization: + skip-unless-backstop-no-suite: null + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - win64-llvm-symbolizer + - vs + fetch: + - upx-3.95-win diff --git a/comm/taskcluster/ci/code-review/kind.yml b/comm/taskcluster/ci/code-review/kind.yml new file mode 100644 index 0000000000..da76da0260 --- /dev/null +++ b/comm/taskcluster/ci/code-review/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.code_review:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - source-test + +jobs: + issues: + label: code-review-issues + description: List all issues found in static analysis and linting tasks + worker-type: b-linux-gcp + + # Only run by using the code-review target_tasks_method + run-on-projects: [] + + # This option permits to run the task + # regardless of the soft-dependencies tasks exit status + # as we are interested in the task failures + requires: all-resolved + + # Publish on pulse + routes: + - project.relman.codereview.v1.try_ending + + # Dummy execution for now, we only need the pulse message + worker: + docker-image: + in-tree: debian11-amd64-build + max-run-time: 600 + run: + using: run-task + checkout: false + command: /bin/true diff --git a/comm/taskcluster/ci/config.yml b/comm/taskcluster/ci/config.yml new file mode 100644 index 0000000000..25f6583be4 --- /dev/null +++ b/comm/taskcluster/ci/config.yml @@ -0,0 +1,310 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +trust-domain: comm +project-repo-param-prefix: comm_ +product-dir: comm/mail + +treeherder: + group-names: + # Images/Packages/Toolchain + 'Deb8': 'Packages for Debian 8' + 'Deb8-32': 'Packages for Debian 8 32-bits' + 'Deb11': 'Packages for Debian 11' + 'Fetch': 'Fetch and store content' + 'Ub18': 'Packages for Ubuntu 18.04' + 'Ub20': 'Packages for Ubuntu 20.04' + 'I': 'Docker Image Builds' + 'TA': 'Toolchain builds for Android' + 'TL': 'Toolchain builds for Linux 64-bits' + 'TM': 'Toolchain builds for OSX' + 'TMW': 'Toolchain builds for Windows MinGW' + 'TW32': 'Toolchain builds for Windows 32-bits' + 'TW64': 'Toolchain builds for Windows 64-bits' + + # Release promotion + 'BM': 'Beetmover' + 'BMcs': 'Beetmover checksums,' + 'BMcss': 'Beetmover checksums for source' + 'BMR': 'Beetmover repackages' + 'c-Up': 'Balrog submission of complete updates' + 'cs': 'Checksum signing' + 'css': 'Checksum signing for source' + 'Flatpak': 'Flatpak image generation' + 'L10n': 'Localised Repacks' + 'L10n-Rpk': 'Localized Repackaged Repacks' + 'l10n-bump': 'L10n Bumper' + 'langpack': 'Langpack uploads to ATN' + 'ms': 'Complete MAR signing' + 'MSI': 'Repack installers into MSIs' + 'MSIs': 'Signing of Repacked installers of MSIs' + 'MSIX': 'Repack into MSIX package' + 'MSIXs': 'Signing of Repack into MSIX package' + 'p': 'Partial generation' + 'ps': 'Partials signing' + 'Rel': 'Release promotion' + 'rs': 'Repackage signing' + 'UV': 'Update verify' + + # mozlint + 'cpp': 'C/C++ checks' + 'js': 'JavaScript checks' + 'l10n': 'Localization checks' + 'misc': 'Misc checks' + 'pedantic': 'pedantic checks' + 'py': 'Python checks' + 'text': 'Check on texts' + + # Tests + 'X': 'Xpcshell tests' + 'X-msix': 'Xpcshell tests on msix packages' + 'M': 'Mochitests' + 'M-msix': 'Mochitests from MSIX builds' + + 'Searchfox': 'Searchfox builds' + 'doc': 'Source docs' + +index: + products: + - thunderbird + - static-analysis + +try: + ridealong-builds: {} + +release-promotion: + products: + - 'thunderbird' + flavors: + promote_thunderbird: + product: thunderbird + target-tasks-method: promote_desktop + partial-updates: true + push_thunderbird: + product: thunderbird + target-tasks-method: push_desktop + partial-updates: true + ship_thunderbird: + product: thunderbird + target-tasks-method: ship_desktop + version-bump: true + partial-updates: true + +# Paths in this section should not be prefixed by comm/ +# Behavior names are prefixed with 'comm-' +merge-automation: + behaviors: + comm-central-to-beta: + fetch-version-from: "mail/config/version.txt" + version-files: + - filename: "mail/config/version.txt" + new-suffix: '' + - filename: "mail/config/version_display.txt" + new-suffix: 'b1' + replacements: + - - mail/config/mozconfigs/l10n-common + - ac_add_options --with-branding=comm/mail/branding/nightly + - ac_add_options --enable-official-branding + - - .gecko_rev.yml + - "GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central" + - "GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta" + - - suite/config/version.txt + - a1 + - '' + - - suite/config/version_display.txt + - a1 + - b1 + merge-old-head: true + base-tag: 'BETA_{major_version}_BASE' + end-tag: 'BETA_{major_version}_END' + from-repo: 'https://hg.mozilla.org/comm-central' + from-branch: 'comm' + to-repo: 'https://hg.mozilla.org/releases/comm-beta' + to-branch: 'comm-beta' + comm-bump-central: + fetch-version-from: "mail/config/version.txt" + version-files: + - filename: "mail/config/version.txt" + version-bump: "major" + new-suffix: 'a1' + - filename: "mail/config/version_display.txt" + version-bump: "major" + new-suffix: 'a1' + replacements: + - - suite/config/version.txt + - '{old_suite_version}' + - '{new_suite_version}' + - - suite/config/version_display.txt + - '{old_suite_version}' + - '{new_suite_version}' + merge-old-head: false + end-tag: 'NIGHTLY_{major_version}_END' + to-repo: 'https://hg.mozilla.org/comm-central' + to-branch: 'comm' + +scriptworker: + scope-prefix: 'project:comm:thunderbird:releng' + +partner-urls: + release-partner-attribution: null + release-partner-repack: null + release-eme-free-repack: null + +task-priority: + by-project: + 'comm-esr.*': 'very-high' + 'comm-beta': 'high' + 'comm-central': 'medium' + 'default': 'low' + +taskgraph: + register: comm_taskgraph:register + decision-parameters: 'comm_taskgraph.decision:get_decision_parameters' + +workers: + aliases: + b-linux.*: + provisioner: 'comm-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + b-win2012: + provisioner: 'comm-{level}' + implementation: generic-worker + os: windows + worker-type: '{alias}-azure' + images: + provisioner: 'comm-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + images-gcp: + provisioner: '{trust-domain}-{level}' + implementation: docker-worker + os: linux + worker-type: '{alias}' + balrog: + provisioner: scriptworker-k8s + implementation: balrog + os: scriptworker + worker-type: + by-release-level: + production: comm-3-balrog + staging: comm-1-balrog + bouncer: + provisioner: scriptworker-k8s + # Note that this implementation doesn't correspond with a + # payload_builder, there are several `bouncer-*` implementations. + implementation: bouncer + os: scriptworker + worker-type: + by-release-level: + production: comm-3-bouncer + staging: comm-1-bouncer + beetmover: + provisioner: scriptworker-k8s + implementation: beetmover + os: scriptworker + worker-type: + by-release-level: + production: comm-3-beetmover + staging: comm-1-beetmover + linux-depsigning: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: linux + worker-type: comm-t-signing + linux-signing: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: linux + worker-type: + by-release-level: + production: comm-3-signing + staging: comm-t-signing + mac-depsigning: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: macosx + worker-type: tb-depsigning-mac-v1 + mac-signing: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: macosx + worker-type: + by-release-level: + production: tb-signing-mac-v1 + staging: tb-depsigning-mac-v1 + tree: + provisioner: scriptworker-k8s + implementation: treescript + os: scriptworker + worker-type: + by-release-level: + production: comm-3-tree + staging: comm-1-tree + t-linux(-large|-xlarge|-xlarge-source): + provisioner: comm-t + implementation: docker-worker + os: linux + worker-type: '{alias}-gcp' + t-osx-1015-r8: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + t-osx-1100-m1: + provisioner: releng-hardware + implementation: generic-worker + os: macosx + worker-type: 'gecko-{alias}' + win11-64-2009: + provisioner: comm-t + implementation: generic-worker + os: windows + worker-type: '{alias}' + succeed: + provisioner: built-in + implementation: succeed + os: none + worker-type: succeed + misc: + provisioner: comm-t + implementation: docker-worker + os: linux + worker-type: misc-gcp + +mac-notarization: + mac-entitlements: + by-platform: + macosx64.*: + by-release-level: + production: comm/build/macosx/hardenedruntime/production.entitlements.xml + default: comm/build/macosx/hardenedruntime/developer.entitlements.xml + default: '' + mac-requirements: + by-platform: + macosx64.*: build/package/mac_osx/requirements.plist + default: '' + +expiration-policy: + by-project: + try-comm-central: + default: 1 month + shortest: 7 days + short: 14 days + medium: 1 month + long: 1 month + ash: + default: 1 month + shortest: 7 days + short: 1 month + medium: 1 month + long: 1 month + default: + default: 3 months + shortest: 7 days + short: 1 month + medium: 1 year + long: 1 year diff --git a/comm/taskcluster/ci/docker-image/docker-image.yml b/comm/taskcluster/ci/docker-image/docker-image.yml new file mode 100644 index 0000000000..93b6a172d9 --- /dev/null +++ b/comm/taskcluster/ci/docker-image/docker-image.yml @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +tb-debian-mingw: + symbol: I(tb-mingw) + parent: debian11-amd64-build + +tb-flatpak: + symbol: I(flatpak) + +tb-atn: + symbol: I(tb-atn) + parent: debian11-base diff --git a/comm/taskcluster/ci/docker-image/kind.yml b/comm/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 0000000000..5865f118da --- /dev/null +++ b/comm/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: comm_taskgraph.loader.merge:loader +reference-base-path: taskcluster/ci + +kind-dependencies: + - packages + +reference-jobs: + - android-build + - debian8-i386-packages + - debian8-i386-raw + - debian8-packages + - debian8-raw + - deb11-toolchain-build + - debian11-amd64-build + - debian11-base + - debian11-packages + - debian11-raw + - fetch + - funsize-update-generator + - image_builder + - index-task + - lint + - push-to-try + - static-analysis-build + - ubuntu1804-base + - ubuntu1804-i386-packages + - ubuntu1804-i386-raw + - ubuntu1804-packages + - ubuntu1804-raw + - ubuntu1804-test + - ubuntu1804-test-base + - ubuntu2004-base + - ubuntu2004-packages + - ubuntu2004-raw + - update-verify + +transforms: + - gecko_taskgraph.transforms.docker_image:transforms + - gecko_taskgraph.transforms.cached_tasks:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - docker-image.yml diff --git a/comm/taskcluster/ci/fetch/kind.yml b/comm/taskcluster/ci/fetch/kind.yml new file mode 100644 index 0000000000..aa79cc9c6a --- /dev/null +++ b/comm/taskcluster/ci/fetch/kind.yml @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: comm_taskgraph.loader.reference:loader +reference-base-path: taskcluster/ci diff --git a/comm/taskcluster/ci/l10n-bump/kind.yml b/comm/taskcluster/ci/l10n-bump/kind.yml new file mode 100644 index 0000000000..b6e6628745 --- /dev/null +++ b/comm/taskcluster/ci/l10n-bump/kind.yml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_version_bump:transforms + - gecko_taskgraph.transforms.task:transforms + + +job-defaults: + worker-type: tree + worker: + implementation: treescript + tags: [] + bump: false + dontbuild: true + ignore-closed-tree: false + push: + by-project: + comm-(central|beta): true + default: false + source-repo: + by-release-type: + release: https://hg.mozilla.org/releases/comm-esr115 + beta: https://hg.mozilla.org/releases/comm-beta + default: https://hg.mozilla.org/comm-central + l10n-bump-info: + by-release-type: + beta|release: + name: Thunderbird l10n changesets + path: mail/locales/l10n-changesets.json + version-path: mail/config/version.txt + l10n-repo-url: "https://hg.mozilla.org/projects/comm-l10n/json-pushes?version=2&tipsonly=1" + ignore-config: + ja: + - macosx64 + ja-JP-mac: + - linux + - linux64 + - win32 + - win64 + platform-configs: [{ + "platforms": [ + "linux", + "linux64", + "macosx64", + "win32", + "win64", + ], + "path": "mail/locales/shipped-locales", + }] + default: + name: Thunderbird l10n changesets + path: mail/locales/l10n-changesets.json + version-path: mail/config/version.txt + ignore-config: + ja: + - macosx64 + ja-JP-mac: + - linux + - linux64 + - win32 + - win64 + platform-configs: [{ + "platforms": [ + "linux", + "linux64", + "macosx64", + "win32", + "win64", + ], + "path": "mail/locales/all-locales", + }] + +jobs: + l10n-bumper: + name: l10n_bumper + description: l10n changesets bumper (action/cron) + run-on-projects: [] + treeherder: + kind: build + platform: linux64/opt + symbol: l10n-bump(lb) + tier: 1 diff --git a/comm/taskcluster/ci/l10n-cross-channel/kind.yml b/comm/taskcluster/ci/l10n-cross-channel/kind.yml new file mode 100644 index 0000000000..ff1e645918 --- /dev/null +++ b/comm/taskcluster/ci/l10n-cross-channel/kind.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - comm_taskgraph.transforms.tb_cross_channel:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + quarantine: + description: Push strings from all shipping trains to comm-strings-quarantine + run-on-projects: [] + worker-type: b-linux-gcp + ssh-key-secret: + by-level: + "3": project/comm/thunderbird/releng/build/level-3/l10n-cross-channel-quarantine-ssh + default: null + worker: + docker-image: {in-tree: push-to-try} + max-run-time: 3600 + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts + env: + TASK_ID: {"task-reference": "<self>"} + TASKCLUSTER_PROXY_URL: http://taskcluster + treeherder: + platform: thunderbird-release/opt + tier: 1 + kind: build + symbol: Rel(l10n-cross-channel) + run: + using: mach + comm-checkout: true + actions: + by-level: + "3": ["prep", "create", "push", "clean"] + default: ["prep", "create", "clean"] diff --git a/comm/taskcluster/ci/l10n-pre/kind.yml b/comm/taskcluster/ci/l10n-pre/kind.yml new file mode 100644 index 0000000000..78d2152938 --- /dev/null +++ b/comm/taskcluster/ci/l10n-pre/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - comm_taskgraph.transforms.l10n_pre:transforms + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + symbol: L10n-pre + worker-type: b-linux-gcp + worker: + max-run-time: 900 + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + locale-list: comm/mail/locales/onchange-locales + comm-locales-file: comm/mail/locales/l10n-changesets.json + browser-locales-file: browser/locales/l10n-changesets.json + run: + using: mozharness + comm-checkout: true + actions: + - clone-gecko-locales + - clone-monorepo + - merge-repos + - pack-merged + script: comm/taskcluster/scripts/desktop_comm_l10n.py + config-paths: + - comm/testing/mozharness/configs + config: + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/thunderbird_split_l10n.py + - taskcluster_nightly.py + options: [] + run-on-projects: ['ash'] + optimization: null + + +jobs: + l10n-pre/opt: + description: Pre-Localization artifact build + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/l10n/kind.yml b/comm/taskcluster/ci/l10n/kind.yml new file mode 100644 index 0000000000..cb9fbbb735 --- /dev/null +++ b/comm/taskcluster/ci/l10n/kind.yml @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: platform + +transforms: + - comm_taskgraph.transforms.l10n:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build + - build-signing + - fetch + - l10n-pre + - repackage + - toolchain + +primary-dependency: build + +only-for-build-platforms: + - linux64/opt + - linux/opt + - macosx64/opt + - win32/opt + - win64/opt + +job-template: + description: Localization + locales-file: comm/mail/locales/onchange-locales + ignore-locales: + by-build-platform: + # OSX has a special locale for japanese + macosx64.*: [ja, en-US] + default: [ja-JP-mac, en-US] + run-time: + by-build-platform: + default: 900 + win.*: 1800 + secrets: false + dependencies: + l10n-pre: l10n-pre-l10n-pre/opt + fetches: + toolchain: + by-build-platform: + default: [] + macosx64.*: + - linux64-libdmg + - linux64-hfsplus + win.*: + - linux64-wine + - nsis + fetch: + by-build-platform: + default: [] + win.*: + - upx-3.95-win + l10n-pre: + - strings_all.tar.zst + tooltool: + by-build-platform: + default: internal + linux.*: public + index: + type: l10n + product: thunderbird + job-name: + by-build-platform: + linux(?!64).*: linux-opt + linux64.*: linux64-opt + macosx64.*: macosx64-opt + win32.*: win32-opt + win64.*: win64-opt + run-on-projects: ['ash'] + worker-type: b-linux-gcp + treeherder: + symbol: L10n + tier: 1 + platform: + by-build-platform: + linux64: linux64/opt + linux: linux32/opt + macosx64: osx-cross/opt + win32: windows2012-32/opt + win64: windows2012-64/opt + env: + by-build-platform: + linux.*: # linux64 and 32 get same treatment here + EN_US_PACKAGE_NAME: target.tar.bz2 + EN_US_BINARY_URL: + artifact-reference: <build/{artifact_prefix}> + macosx64.*: + EN_US_PACKAGE_NAME: target.dmg + EN_US_BINARY_URL: + artifact-reference: <build/{artifact_prefix}> + win.*: + EN_US_PACKAGE_NAME: target.zip + EN_US_BINARY_URL: + artifact-reference: <build-signing/{artifact_prefix}> + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + linux: + - single_locale/linux32.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - single_locale/thunderbird.py + linux64: + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - single_locale/thunderbird.py + macosx64: + - single_locale/macosx64.py + - single_locale/tc_common.py + - single_locale/tc_macosx64.py + - single_locale/thunderbird.py + win32: + - single_locale/win32.py + - single_locale/tc_common.py + - single_locale/tc_win32.py + - single_locale/thunderbird.py + win64: + - single_locale/win64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - single_locale/thunderbird.py + # no default, so we fail on new entries + actions: ['list-locales', 'setup', 'repack', 'summary'] + script: mozharness/scripts/desktop_l10n.py + when: + files-changed: + - comm/calendar/locales/** + - comm/chat/locales/** + - comm/mail/locales/** + - comm/testing/mozharness/configs/single_locale/** + - testing/mozharness/configs/single_locale/** + - testing/mozharness/mozharness/mozilla/l10n/locales.py + - testing/mozharness/scripts/desktop_l10n.py + - third_party/python/compare-locales/** + - toolkit/locales/** + - toolkit/mozapps/installer/** diff --git a/comm/taskcluster/ci/mar-signing-l10n/kind.yml b/comm/taskcluster/ci/mar-signing-l10n/kind.yml new file mode 100644 index 0000000000..46f138165c --- /dev/null +++ b/comm/taskcluster/ci/mar-signing-l10n/kind.yml @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.mar_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-l10n + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux64-asan-reporter-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +job-template: + shipping-phase: promote + treeherder-group: ms + description-suffix: 'mar signing' + required_signoffs: + - mar-signing diff --git a/comm/taskcluster/ci/mar-signing/kind.yml b/comm/taskcluster/ci/mar-signing/kind.yml new file mode 100644 index 0000000000..af9331487f --- /dev/null +++ b/comm/taskcluster/ci/mar-signing/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.mar_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux64-asan-reporter-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + - win64-asan-reporter-shippable/opt + +job-template: + shipping-phase: promote + treeherder-group: ms + description-suffix: 'mar signing' + required_signoffs: + - mar-signing diff --git a/comm/taskcluster/ci/merge-automation/kind.yml b/comm/taskcluster/ci/merge-automation/kind.yml new file mode 100644 index 0000000000..2c5290966b --- /dev/null +++ b/comm/taskcluster/ci/merge-automation/kind.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.merge_automation:transforms + - comm_taskgraph.transforms.merge_automation:transforms + - gecko_taskgraph.transforms.task:transforms + + +jobs: + merge-automation: + name: merge-automation + label: merge-automation + treeherder: + symbol: Rel(merge-automation) + tier: 1 + kind: build + platform: thunderbird-release/opt + description: Merge repositories for release management. + worker-type: tree + scopes: [] + worker: + implementation: treescript + tags: [] + bump: false + dontbuild: false + ignore-closed-tree: true + push: false + l10n-bump-info: + by-behavior: + comm-central-to-beta: + name: Thunderbird l10n changesets + path: mail/locales/l10n-changesets.json + version-path: mail/config/version.txt + l10n-repo-url: "https://hg.mozilla.org/projects/comm-l10n/json-pushes?version=2&tipsonly=1" + ignore-config: + ja: + - macosx64 + ja-JP-mac: + - linux + - linux64 + - win32 + - win64 + platform-configs: [{ + "platforms": [ + "linux", + "linux64", + "macosx64", + "win32", + "win64", + ], + "path": "mail/locales/shipped-locales", + }] + default: null + run-on-projects: [] diff --git a/comm/taskcluster/ci/packages/kind.yml b/comm/taskcluster/ci/packages/kind.yml new file mode 100644 index 0000000000..aa79cc9c6a --- /dev/null +++ b/comm/taskcluster/ci/packages/kind.yml @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: comm_taskgraph.loader.reference:loader +reference-base-path: taskcluster/ci diff --git a/comm/taskcluster/ci/partials-signing/kind.yml b/comm/taskcluster/ci/partials-signing/kind.yml new file mode 100644 index 0000000000..c01adaedd0 --- /dev/null +++ b/comm/taskcluster/ci/partials-signing/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.mar_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - partials + +job-template: + shipping-phase: promote + treeherder-group: ps + description-suffix: 'partial signing' + required_signoffs: + - mar-signing diff --git a/comm/taskcluster/ci/partials/kind.yml b/comm/taskcluster/ci/partials/kind.yml new file mode 100644 index 0000000000..fdea013cb4 --- /dev/null +++ b/comm/taskcluster/ci/partials/kind.yml @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.partials:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.partials:transforms + +kind-dependencies: + - repackage + - repackage-l10n + +only-for-attributes: + - shippable + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +job-template: + shipping-phase: promote diff --git a/comm/taskcluster/ci/post-balrog-dummy/kind.yml b/comm/taskcluster/ci/post-balrog-dummy/kind.yml new file mode 100644 index 0000000000..6b5b90afaf --- /dev/null +++ b/comm/taskcluster/ci/post-balrog-dummy/kind.yml @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.per_platform_dummy:transforms + - gecko_taskgraph.transforms.reverse_chunk_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - balrog + +job-template: + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + worker-type: succeed + worker: + implementation: succeed diff --git a/comm/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml b/comm/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml new file mode 100644 index 0000000000..a136d22089 --- /dev/null +++ b/comm/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.reverse_chunk_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-checksums + - release-beetmover-source-checksums + +jobs: + thunderbird-promote: + name: post-beetmover-checksums-dummy + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + shipping-product: thunderbird + worker-type: succeed + worker: + implementation: succeed diff --git a/comm/taskcluster/ci/post-beetmover-dummy/kind.yml b/comm/taskcluster/ci/post-beetmover-dummy/kind.yml new file mode 100644 index 0000000000..adbdcd7306 --- /dev/null +++ b/comm/taskcluster/ci/post-beetmover-dummy/kind.yml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.per_platform_dummy:transforms + - gecko_taskgraph.transforms.reverse_chunk_deps:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-checksums + - beetmover-repackage + +job-template: + description: Dummy task to deal with max_dependencies + run-on-projects: [] + shipping-phase: promote + worker-type: succeed + worker: + implementation: succeed diff --git a/comm/taskcluster/ci/release-balrog-scheduling/kind.yml b/comm/taskcluster/ci/release-balrog-scheduling/kind.yml new file mode 100644 index 0000000000..ea5574cc75 --- /dev/null +++ b/comm/taskcluster/ci/release-balrog-scheduling/kind.yml @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-bouncer-check + - release-push-langpacks + +job-defaults: + run-on-projects: [] + shipping-phase: ship + worker-type: balrog + worker: + implementation: balrog + balrog-action: schedule + +jobs: + thunderbird: + description: Schedule Thunderbird publishing in balrog + name: release-thunderbird_schedule_publishing_in_balrog + shipping-product: thunderbird + worker: + product: thunderbird + publish-rules: + by-release-level: + production: + by-release-type: + beta: [43] + release: [17829] + default: [] + staging: + by-release-type: + beta: [43] + release: [903] + default: [] + background-rate: + by-release-type: + beta: + by-beta-number: + '1': 15 + '(2|3)': 25 + '(4|5|6)': 35 + default: null + default: null + force-fallback-mapping-update: + by-release-type: + beta: + by-beta-number: + '1': true + default: false + default: false + treeherder: + platform: thunderbird-release/opt + symbol: Rel(BS) + tier: 1 + kind: build diff --git a/comm/taskcluster/ci/release-balrog-submit-toplevel/kind.yml b/comm/taskcluster/ci/release-balrog-submit-toplevel/kind.yml new file mode 100644 index 0000000000..64732d2056 --- /dev/null +++ b/comm/taskcluster/ci/release-balrog-submit-toplevel/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.scriptworker:add_balrog_scopes + - gecko_taskgraph.transforms.balrog_toplevel:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + run-on-projects: [] # to make sure this never runs in CI + shipping-phase: promote + worker-type: balrog + worker: + implementation: balrog + balrog-action: v2-submit-toplevel + require-mirrors: true + archive-domain: + by-release-level: + production: archive.mozilla.org + staging: ftp.stage.mozaws.net + download-domain: + by-release-level: + production: download.mozilla.org + staging: bouncer-bouncer-releng.stage.mozaws.net + whats-new-config: comm/mail/config/whats_new_page.yml + +jobs: + thunderbird: + name: submit-toplevel-thunderbird-release-to-balrog + description: Submit toplevel Thunderbird release to balrog + shipping-product: thunderbird + worker: + product: thunderbird + channel-names: + by-release-type: + beta: ["beta", "beta-localtest", "beta-cdntest"] + release: ["release", "release-localtest", "release-cdntest", "release-localtest-next"] + default: [] + rules-to-update: + by-release-type: + beta: ["thunderbird-beta-cdntest", "thunderbird-beta-localtest"] + release: ["thunderbird-release115-cdntest", "thunderbird-release115-localtest", "thunderbird-release115-localtest-next"] + default: [] + platforms: ["linux", "linux64", "macosx64", "win32", "win64"] + treeherder: + platform: thunderbird-release/opt + symbol: Rel(BPTb) + tier: 1 + kind: build diff --git a/comm/taskcluster/ci/release-beetmover-push-to-release/kind.yml b/comm/taskcluster/ci/release-beetmover-push-to-release/kind.yml new file mode 100644 index 0000000000..67209a2409 --- /dev/null +++ b/comm/taskcluster/ci/release-beetmover-push-to-release/kind.yml @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.beetmover_push_to_release:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-generate-checksums-beetmover + +job-defaults: + run-on-projects: [] + shipping-phase: push + +jobs: + thunderbird-push-to-release: + name: thunderbird_push_to_release + product: thunderbird + shipping-product: thunderbird + treeherder-platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/release-beetmover-source-checksums/kind.yml b/comm/taskcluster/ci/release-beetmover-source-checksums/kind.yml new file mode 100644 index 0000000000..7d4560306f --- /dev/null +++ b/comm/taskcluster/ci/release-beetmover-source-checksums/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.beetmover_source_checksums:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-source-checksums-signing + +job-template: + shipping-phase: promote + attributes: + artifact_prefix: public + artifact_map: comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml diff --git a/comm/taskcluster/ci/release-bouncer-aliases/kind.yml b/comm/taskcluster/ci/release-bouncer-aliases/kind.yml new file mode 100644 index 0000000000..9f4adfb229 --- /dev/null +++ b/comm/taskcluster/ci/release-bouncer-aliases/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.bouncer_aliases:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-bouncer-check + +job-defaults: + description: Update bouncer aliases job + worker-type: bouncer + worker: + implementation: bouncer-aliases + scopes: + by-release-level: + production: + - project:comm:thunderbird:releng:bouncer:action:aliases + - project:comm:thunderbird:releng:bouncer:server:production + staging: + - project:comm:thunderbird:releng:bouncer:action:aliases + - project:comm:thunderbird:releng:bouncer:server:staging + run-on-projects: [] + shipping-phase: ship + treeherder: + symbol: Rel(BncAlias) + kind: test + tier: 1 + +jobs: + thunderbird: + bouncer-products-per-alias: + by-release-type: + beta: + thunderbird-beta-latest-ssl: installer-ssl + thunderbird-beta-latest: installer + thunderbird-beta-msi-latest-ssl: msi + thunderbird-beta-msix-latest-ssl: msix + release: + # Disable on comm-esr115 when the next esr is released + thunderbird-latest-ssl: installer-ssl + thunderbird-latest: installer + thunderbird-msi-latest-ssl: msi + thunderbird-msix-latest-ssl: msix + default: {} + shipping-product: thunderbird + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/release-bouncer-check/kind.yml b/comm/taskcluster/ci/release-bouncer-check/kind.yml new file mode 100644 index 0000000000..97b15d72c4 --- /dev/null +++ b/comm/taskcluster/ci/release-bouncer-check/kind.yml @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - release-beetmover-push-to-release + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.bouncer_check:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: bouncer-check + description: bouncer check + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: push + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + docker-image: {in-tree: "update-verify"} + run: + sparse-profile: mozharness + attributes: + build_platform: linux64 + build_type: opt + treeherder: + symbol: Rel(BncChk) + kind: test + tier: 1 + +jobs: + thunderbird: + shipping-product: thunderbird + index: + product: thunderbird + job-name: thunderbird-bouncer-check + run: + comm-checkout: true + config: + - comm/testing/mozharness/configs/releases/bouncer_thunderbird.py + extra-config: + by-project: + comm-(beta|esr.*): + locales: ['en-US', 'fr', 'de'] + bouncer_prefix: "https://download.mozilla.org/" + default: + locales: ['en-US', 'fr', 'de'] + bouncer_prefix: "https://bouncer-bouncer-releng.stage.mozaws.net/" + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/release-bouncer-sub/kind.yml b/comm/taskcluster/ci/release-bouncer-sub/kind.yml new file mode 100644 index 0000000000..8142655f84 --- /dev/null +++ b/comm/taskcluster/ci/release-bouncer-sub/kind.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.bouncer_submission:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + description: release bouncer submission job + worker-type: bouncer + worker: + implementation: bouncer-submission + scopes: + by-release-level: + production: + - project:comm:thunderbird:releng:bouncer:action:submission + - project:comm:thunderbird:releng:bouncer:server:production + staging: + - project:comm:thunderbird:releng:bouncer:action:submission + - project:comm:thunderbird:releng:bouncer:server:staging + run-on-projects: [] + shipping-phase: promote + locales-file: comm/mail/locales/shipped-locales + treeherder: + symbol: Rel(BncSub) + kind: test + tier: 1 + +jobs: + thunderbird: + bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64'] + bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'msi', 'msix'] + shipping-product: thunderbird + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/release-final-verify/kind.yml b/comm/taskcluster/ci/release-final-verify/kind.yml new file mode 100644 index 0000000000..c45e9dc4b9 --- /dev/null +++ b/comm/taskcluster/ci/release-final-verify/kind.yml @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - release-bouncer-check + - release-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.final_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: final-verify + run-on-projects: [] # to make sure this never runs as part of CI + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: [1] + +jobs: + thunderbird: + description: final verify + shipping-phase: push + shipping-product: thunderbird + treeherder: + platform: thunderbird-release/opt + kind: test + symbol: Rel(FV) + tier: 1 diff --git a/comm/taskcluster/ci/release-flatpak-push/kind.yml b/comm/taskcluster/ci/release-flatpak-push/kind.yml new file mode 100644 index 0000000000..5b6ca98363 --- /dev/null +++ b/comm/taskcluster/ci/release-flatpak-push/kind.yml @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - comm_taskgraph.transforms.release_flatpak_push:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-flatpak-repackage + +job-defaults: + description: Pushes Flatpaks onto Flathub + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [beta, release] + shipping-phase: ship + treeherder: + platform: linux64/opt + kind: build + tier: 2 + worker-type: + by-release-level: + production: scriptworker-k8s/comm-3-pushflatpak + staging: scriptworker-k8s/comm-1-pushflatpak + worker: + implementation: push-flatpak + channel: + by-release-type: + beta: beta + release: stable + default: mock + +jobs: + thunderbird: + shipping-product: thunderbird + flathub-scope: "flathub" + treeherder: + symbol: Flatpak(push) diff --git a/comm/taskcluster/ci/release-flatpak-repackage/kind.yml b/comm/taskcluster/ci/release-flatpak-repackage/kind.yml new file mode 100644 index 0000000000..723fd25c7c --- /dev/null +++ b/comm/taskcluster/ci/release-flatpak-repackage/kind.yml @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_flatpak_repackage:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + +job-defaults: + description: Generates flatpak by repackaging the existing .tar.bz2 + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [beta, release] + shipping-phase: promote + scopes: [] + treeherder: + platform: linux64-shippable/opt + kind: build + tier: 2 + worker-type: b-linux-gcp + worker: + implementation: docker-worker + os: linux + max-run-time: 7200 + docker-image: {in-tree: tb-flatpak} + artifacts: + - name: public/build + type: directory + path: /home/worker/artifacts/ + command: + - /bin/bash + - -cx + - ./runme.sh + env: + VERSION: "{release_config[version]}" + BUILD_NUMBER: "{release_config[build_number]}" + CANDIDATES_DIR: + by-release-level: + staging: + https://ftp.stage.mozaws.net/pub/{task[shipping-product]}/candidates + production: + https://archive.mozilla.org/pub/{task[shipping-product]}/candidates + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + L10N_CHANGESETS: "{config_params[comm_head_repository]}/raw-file/{config_params[comm_head_rev]}/mail/locales/l10n-changesets.json" + FLATPAK_BRANCH: + by-project: + comm-esr.*: stable + default: beta + MANIFEST_URL: "{config_params[comm_head_repository]}/file/{config_params[comm_head_rev]}/taskcluster/docker/tb-flatpak/runme.sh" + RELEASE_NOTES_URL: + by-project: + comm-esr.*: "https://www.thunderbird.net/en-US/thunderbird/{config_params[app_version]}/releasenotes/" + default: "https://www.thunderbird.net/en-US/thunderbird/{config_params[app_version]}beta/releasenotes/" + chain-of-trust: true + +jobs: + thunderbird: + shipping-product: thunderbird + attributes: + build_platform: linux64-shippable + build_type: opt + treeherder: + symbol: Flatpak(r) diff --git a/comm/taskcluster/ci/release-generate-checksums-beetmover/kind.yml b/comm/taskcluster/ci/release-generate-checksums-beetmover/kind.yml new file mode 100644 index 0000000000..57de655b03 --- /dev/null +++ b/comm/taskcluster/ci/release-generate-checksums-beetmover/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.release_generate_checksums_beetmover:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-generate-checksums-signing + +job-template: + shipping-phase: promote + attributes: + artifact_map: comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml diff --git a/comm/taskcluster/ci/release-generate-checksums-signing/kind.yml b/comm/taskcluster/ci/release-generate-checksums-signing/kind.yml new file mode 100644 index 0000000000..2e763e5434 --- /dev/null +++ b/comm/taskcluster/ci/release-generate-checksums-signing/kind.yml @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +kind-dependencies: + - release-generate-checksums + +transforms: + - gecko_taskgraph.transforms.release_generate_checksums_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +job-template: + shipping-phase: promote diff --git a/comm/taskcluster/ci/release-generate-checksums/kind.yml b/comm/taskcluster/ci/release-generate-checksums/kind.yml new file mode 100644 index 0000000000..0aa175b5ad --- /dev/null +++ b/comm/taskcluster/ci/release-generate-checksums/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - beetmover-source + - post-beetmover-checksums-dummy + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_generate_checksums:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: generate-checksums + description: generates checksums + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + max-run-time: 1200 + artifacts: + - name: public/build/SHA256SUMMARY + path: /builds/worker/SHA256SUMMARY + type: file + - name: public/build/SHA256SUMS + path: /builds/worker/SHA256SUMS + type: file + - name: public/build/SHA512SUMMARY + path: /builds/worker/SHA512SUMMARY + type: file + - name: public/build/SHA512SUMS + path: /builds/worker/SHA512SUMS + type: file + run: + using: mozharness + comm-checkout: true + config: [] + config-paths: + - comm/testing/mozharness/configs + actions: [create-virtualenv, collect-individual-checksums, create-big-checksums, create-summary] + options: + - "version={version}" + - "build-number={build_number}" + script: "mozharness/scripts/release/generate-checksums.py" + treeherder: + symbol: Rel(GenChcks) + kind: test + tier: 1 + optimization: null + +jobs: + thunderbird: + shipping-product: thunderbird + attributes: + build_platform: thunderbird-release + build_type: opt + run: + extra-config: + by-release-level: + production: + stage_product: "thunderbird" + bucket_name: "moz-fx-productdelivery-pr-38b5-productdelivery" + staging: + stage_product: "thunderbird" + bucket_name: "moz-fx-productdelivery-no-7d6a-productdelivery" + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/release-mark-as-shipped/kind.yml b/comm/taskcluster/ci/release-mark-as-shipped/kind.yml new file mode 100644 index 0000000000..ce35138454 --- /dev/null +++ b/comm/taskcluster/ci/release-mark-as-shipped/kind.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_mark_as_shipped:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-balrog-scheduling + - release-bouncer-aliases + - release-version-bump + +job-defaults: + description: mark release as shipped in Ship-It + worker-type: + by-release-level: + production: scriptworker-k8s/comm-3-shipit + staging: scriptworker-k8s/comm-1-shipit + worker: + implementation: shipit-shipped + scopes: + by-release-level: + production: + - project:comm:thunderbird:releng:ship-it:server:production + - project:comm:thunderbird:releng:ship-it:action:mark-as-shipped + staging: + - project:comm:thunderbird:releng:ship-it:server:staging + - project:comm:thunderbird:releng:ship-it:action:mark-as-shipped + run-on-projects: [] + shipping-phase: ship + +jobs: + thunderbird: + name: release-thunderbird_mark_as_shipped + shipping-product: thunderbird + treeherder: + platform: thunderbird-release/opt + symbol: Rel(ShipIt) + tier: 1 + kind: build diff --git a/comm/taskcluster/ci/release-notify-av-announce/kind.yml b/comm/taskcluster/ci/release-notify-av-announce/kind.yml new file mode 100644 index 0000000000..ed528c4ad4 --- /dev/null +++ b/comm/taskcluster/ci/release-notify-av-announce/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + +job-defaults: + name: notify-av-announce + description: Sends email to antivirus vendors telling them that the release is available for scanning. + run-on-projects: [] + shipping-phase: push + worker-type: succeed + worker: + implementation: succeed + notifications: + # !t is a custom format option that calls .title() on the string. + subject: "{task[shipping-product]!t} {release_config[version]} build{release_config[build_number]} release" + message: | + Company Name: MZLA Technologies + Contact Department: Release Engineering + Contact email address: tb-builds@thunderbird.net + Product information: {task[shipping-product]} {release_config[version]} + Product location: https://releases.mozilla.org/pub/{task[shipping-product]}/candidates/{release_config[version]}-candidates/build{release_config[build_number]}/ + emails: + by-project: + comm-(beta|esr.*): ["av-vendor-release-announce@mozilla.org"] + try-comm-central: ["{config[params][owner]}"] + default: [] + +jobs: + thunderbird: + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-notify-promote/kind.yml b/comm/taskcluster/ci/release-notify-promote/kind.yml new file mode 100644 index 0000000000..281e5b5b88 --- /dev/null +++ b/comm/taskcluster/ci/release-notify-promote/kind.yml @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - post-beetmover-dummy + - release-generate-checksums-beetmover + - release-bouncer-sub + - beetmover-strings-source + +job-defaults: + name: notify-release-drivers-promote + description: Sends email to tb-drivers telling release was promoted. + run-on-projects: [] + shipping-phase: promote + worker-type: succeed + worker: + implementation: succeed + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory" + emails: + by-project: + comm-(beta|esr.*): ["thunderbird-drivers@mozilla.org"] + try-comm-central: ["{config[params][owner]}"] + default: [] + +jobs: + thunderbird: + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-notify-push/kind.yml b/comm/taskcluster/ci/release-notify-push/kind.yml new file mode 100644 index 0000000000..afb5a66b08 --- /dev/null +++ b/comm/taskcluster/ci/release-notify-push/kind.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-beetmover-push-to-release + +job-defaults: + name: notify-release-drivers-push + description: Sends email to tb-drivers telling release was pushed. + run-on-projects: [] + shipping-phase: push + worker-type: succeed + worker: + implementation: succeed + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest" + emails: + by-project: + comm-(beta|esr.*): ["thunderbird-drivers@mozilla.org"] + try-comm-central: ["{config[params][owner]}"] + default: [] + +jobs: + thunderbird: + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-notify-ship/kind.yml b/comm/taskcluster/ci/release-notify-ship/kind.yml new file mode 100644 index 0000000000..bbbec12317 --- /dev/null +++ b/comm/taskcluster/ci/release-notify-ship/kind.yml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_notifications:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-balrog-scheduling + - release-bouncer-aliases + - release-version-bump + - release-mark-as-shipped + +job-defaults: + name: notify-release-drivers-ship + description: Sends email to release-drivers telling release was shipped. + run-on-projects: [] + shipping-phase: ship + worker-type: succeed + worker: + implementation: succeed + notifications: + emails: + by-project: + comm-(beta|esr.*): ["thunderbird-drivers@mozilla.org"] + try-comm-central: ["{config[params][owner]}"] + default: [] + +jobs: + thunderbird: + shipping-product: thunderbird + notifications: + subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" + message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!" diff --git a/comm/taskcluster/ci/release-notify-started/kind.yml b/comm/taskcluster/ci/release-notify-started/kind.yml new file mode 100644 index 0000000000..1c62b5ad38 --- /dev/null +++ b/comm/taskcluster/ci/release-notify-started/kind.yml @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - comm_taskgraph.transforms.release_started:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: notify-release-drivers-started + description: Sends email to release-drivers telling release was started. + run-on-projects: [] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: "debian11-base"} + max-run-time: 600 + emails: + by-project: + comm-(beta|esr.*): ["thunderbird-drivers@mozilla.org"] + try-comm-central: ["{config[params][owner]}"] + default: [] + +jobs: + thunderbird: + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-push-langpacks/kind.yml b/comm/taskcluster/ci/release-push-langpacks/kind.yml new file mode 100644 index 0000000000..3d02277800 --- /dev/null +++ b/comm/taskcluster/ci/release-push-langpacks/kind.yml @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - comm_taskgraph.transforms.push_langpacks:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build + - shippable-l10n + + +only-for-build-platforms: + - linux64-shippable/opt # ATN only support 1 platform per locale. That's why we use linux64 + - macosx64-shippable/opt # Although, we need the special locale "ja-JP-Mac" from this platform + + +job-template: + description: Sends {locales} XPIs for platform to ATN + worker-type: b-linux-gcp + worker: + docker-image: {in-tree: "tb-atn"} + implementation: docker-worker + os: linux + max-run-time: 1800 + env: + TASK_ID: {"task-reference": "<self>"} + TASKCLUSTER_PROXY_URL: http://taskcluster + PATH: /builds/worker/bin:/usr/local/bin:/usr/bin:/bin + channel: + by-project: + comm-esr115: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64.*: unlisted + default: listed + comm-beta: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64.*: unlisted + default: listed + default: unlisted + command: + - /builds/worker/bin/run-task + - "--" + - /builds/worker/venv/bin/python + - /builds/worker/bin/atn_langpack.py + run-on-projects: [] + scopes: + - secrets:get:project/comm/thunderbird/releng/build/level-{level}/atn_langpack + shipping-phase: push + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-source-checksums-signing/kind.yml b/comm/taskcluster/ci/release-source-checksums-signing/kind.yml new file mode 100644 index 0000000000..379aff10bc --- /dev/null +++ b/comm/taskcluster/ci/release-source-checksums-signing/kind.yml @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.source_checksums_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-source + +job-template: + shipping-phase: promote diff --git a/comm/taskcluster/ci/release-source-signing/kind.yml b/comm/taskcluster/ci/release-source-signing/kind.yml new file mode 100644 index 0000000000..2b6c3376ab --- /dev/null +++ b/comm/taskcluster/ci/release-source-signing/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.build_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-source + +job-template: + shipping-phase: promote diff --git a/comm/taskcluster/ci/release-source/kind.yml b/comm/taskcluster/ci/release-source/kind.yml new file mode 100644 index 0000000000..514f9f1270 --- /dev/null +++ b/comm/taskcluster/ci/release-source/kind.yml @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + shipping-phase: promote + treeherder: + symbol: Src + worker-type: b-linux-gcp + worker: + max-run-time: 3600 + env: + PKG_SRCPACK_BASENAME: source + attributes: + shippable: true + run: + using: mozharness + comm-checkout: true + actions: [package-source] + script: "mozharness/scripts/fx_desktop_build.py" + config-paths: + - comm/testing/mozharness/configs + config: + - builds/thunderbird_source.py + options: [] + tooltool-downloads: public + run-on-projects: [] + optimization: null + +jobs: + thunderbird-source/opt: + description: "Thunderbird source builder" + treeherder: + platform: thunderbird-release/opt + shipping-product: thunderbird diff --git a/comm/taskcluster/ci/release-update-verify-config-next/kind.yml b/comm/taskcluster/ci/release-update-verify-config-next/kind.yml new file mode 100644 index 0000000000..34b0475ce6 --- /dev/null +++ b/comm/taskcluster/ci/release-update-verify-config-next/kind.yml @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - comm_taskgraph.transforms.update_verify_config:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-config-next + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [release] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 3600 + artifacts: + - name: public/build/update-verify.cfg + path: /builds/worker/checkouts/gecko/update-verify.cfg + type: file + run: + sparse-profile: mozharness + treeherder: + symbol: UVCnext + kind: test + tier: 1 + extra: + app-name: mail + branch-prefix: comm + archive-prefix: + by-release-level: + staging: "http://ftp.stage.mozaws.net/pub" + production: "https://archive.mozilla.org/pub" + previous-archive-prefix: + by-release-level: + staging: "https://archive.mozilla.org/pub" + production: null + aus-server: + by-release-level: + staging: "https://stage.balrog.nonprod.cloudops.mozgcp.net" + production: "https://aus5.mozilla.org" + override-certs: + by-release-level: + staging: dep + production: null + updater-platform: linux-x86_64 + product: thunderbird + channel: "release-localtest-next" + include-version: release-next + last-watershed: "91.9.0" + +jobs: + thunderbird-next-linux: + shipping-product: thunderbird + treeherder: + platform: linux32-shippable/opt + attributes: + build_platform: linux-shippable + extra: + platform: linux-i686 + thunderbird-next-linux64: + shipping-product: thunderbird + treeherder: + platform: linux64-shippable/opt + attributes: + build_platform: linux64-shippable + extra: + platform: linux-x86_64 + thunderbird-next-macosx64: + shipping-product: thunderbird + treeherder: + platform: osx-shippable/opt + attributes: + build_platform: macosx64-shippable + extra: + platform: mac + thunderbird-next-win32: + shipping-product: thunderbird + treeherder: + platform: windows2012-32-shippable/opt + attributes: + build_platform: win32-shippable + extra: + platform: win32 + thunderbird-next-win64: + shipping-product: thunderbird + treeherder: + platform: windows2012-64-shippable/opt + attributes: + build_platform: win64-shippable + extra: + platform: win64 diff --git a/comm/taskcluster/ci/release-update-verify-config/kind.yml b/comm/taskcluster/ci/release-update-verify-config/kind.yml new file mode 100644 index 0000000000..eb6081778b --- /dev/null +++ b/comm/taskcluster/ci/release-update-verify-config/kind.yml @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - comm_taskgraph.transforms.update_verify_config:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-config + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + docker-image: + in-tree: "update-verify" + max-run-time: 3600 + artifacts: + - name: public/build/update-verify.cfg + path: /builds/worker/checkouts/gecko/update-verify.cfg + type: file + run: + sparse-profile: mozharness + extra: + app-name: mail + branch-prefix: comm + archive-prefix: + by-release-level: + staging: "http://ftp.stage.mozaws.net/pub" + production: "https://archive.mozilla.org/pub" + previous-archive-prefix: + by-release-level: + staging: "https://archive.mozilla.org/pub" + production: null + aus-server: + by-release-level: + staging: "https://stage.balrog.nonprod.cloudops.mozgcp.net" + production: "https://aus5.mozilla.org" + override-certs: + by-release-level: + staging: dep + production: null + include-version: + by-release-type: + beta: beta + release: nonbeta + default: beta + last-watershed: + by-release-type: + beta: "79.0b2" + release: "115.0" + default: "default" + +jobs: + thunderbird-linux: + shipping-product: thunderbird + treeherder: + symbol: UVC + platform: linux32-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux-shippable + extra: + product: thunderbird + platform: linux-i686 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release: "release-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + thunderbird-linux64: + shipping-product: thunderbird + treeherder: + symbol: UVC + platform: linux64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: linux64-shippable + extra: + product: thunderbird + platform: linux-x86_64 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release: "release-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + thunderbird-macosx64: + shipping-product: thunderbird + treeherder: + symbol: UVC + platform: osx-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: macosx64-shippable + extra: + product: thunderbird + platform: mac + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release: "release-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + thunderbird-win32: + shipping-product: thunderbird + treeherder: + symbol: UVC + platform: windows2012-32-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win32-shippable + extra: + product: thunderbird + platform: win32 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release: "release-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null + + thunderbird-win64: + shipping-product: thunderbird + treeherder: + symbol: UVC + platform: windows2012-64-shippable/opt + kind: test + tier: 1 + attributes: + build_platform: win64-shippable + extra: + product: thunderbird + platform: win64 + updater-platform: linux-x86_64 + channel: + by-release-type: + beta: "beta-localtest" + release: "release-localtest" + default: "default" + mar-channel-id-override: + by-release-type: + beta: beta + default: null diff --git a/comm/taskcluster/ci/release-update-verify-next/kind.yml b/comm/taskcluster/ci/release-update-verify-next/kind.yml new file mode 100644 index 0000000000..bd898b27ba --- /dev/null +++ b/comm/taskcluster/ci/release-update-verify-next/kind.yml @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-balrog-submit-toplevel + - release-update-verify-config-next + +transforms: + - gecko_taskgraph.transforms.release:run_on_releases + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.update_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify-next + run-on-projects: [] # to make sure this never runs as part of CI + run-on-releases: [release] + shipping-phase: promote + worker-type: b-linux-gcp + worker: + artifacts: + - name: 'public/build/diff-summary.log' + path: '/builds/worker/tools/release/updates/diff-summary.log' + type: file + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: + - 255 + env: + CHANNEL: "release-localtest-next" + treeherder: + symbol: UV(UVnext) + kind: test + extra: + chunks: 12 + +jobs: + thunderbird-next-linux64: + description: linux64 release-next update verify + shipping-product: thunderbird + attributes: + build_platform: linux64-shippable + + thunderbird-next-linux: + description: linux release-next update verify + shipping-product: thunderbird + attributes: + build_platform: linux-shippable + + thunderbird-next-win64: + description: win64 release-next update verify + shipping-product: thunderbird + attributes: + build_platform: win64-shippable + + thunderbird-next-win32: + description: win32 release-next update verify + shipping-product: thunderbird + attributes: + build_platform: win32-shippable + + thunderbird-next-macosx64: + description: macosx64 release-next update verify + shipping-product: thunderbird + attributes: + build_platform: macosx64-shippable diff --git a/comm/taskcluster/ci/release-update-verify/kind.yml b/comm/taskcluster/ci/release-update-verify/kind.yml new file mode 100644 index 0000000000..c8c21829cb --- /dev/null +++ b/comm/taskcluster/ci/release-update-verify/kind.yml @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - post-balrog-dummy + - post-beetmover-dummy + - release-balrog-submit-toplevel + - release-update-verify-config + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.update_verify:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + name: update-verify + run-on-projects: [] # to make sure this never runs as part of CI + shipping-phase: promote + worker-type: b-linux-gcp + worker: + artifacts: + - name: 'public/build/diff-summary.log' + path: '/builds/worker/checkouts/gecko/diff-summary.log' + type: file + docker-image: + in-tree: "update-verify" + max-run-time: 7200 + retry-exit-status: + - 255 + treeherder: + symbol: UV(UV) + kind: test + extra: + chunks: 12 + +jobs: + thunderbird-linux64: + description: linux64 update verify + shipping-product: thunderbird + attributes: + build_platform: linux64-shippable + + thunderbird-linux: + description: linux update verify + shipping-product: thunderbird + attributes: + build_platform: linux-shippable + + thunderbird-win64: + description: win64 update verify + shipping-product: thunderbird + attributes: + build_platform: win64-shippable + + thunderbird-win32: + description: win32 update verify + shipping-product: thunderbird + attributes: + build_platform: win32-shippable + + thunderbird-macosx64: + description: macosx64 update verify + shipping-product: thunderbird + attributes: + build_platform: macosx64-shippable diff --git a/comm/taskcluster/ci/release-version-bump/kind.yml b/comm/taskcluster/ci/release-version-bump/kind.yml new file mode 100644 index 0000000000..a2328651c2 --- /dev/null +++ b/comm/taskcluster/ci/release-version-bump/kind.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.release_deps:transforms + - gecko_taskgraph.transforms.release_version_bump:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - release-beetmover-push-to-release + +job-defaults: + description: Release Promotion version bump/tag + run-on-projects: [] + shipping-phase: ship + worker-type: tree + worker: + implementation: treescript + dontbuild: true + tags: ['buildN', 'release'] + bump: true + bump-files: + by-project: + default: ["mail/config/version_display.txt"] + comm-(beta|esr.*): + - "mail/config/version.txt" + - "mail/config/version_display.txt" + push: + by-project: + comm-(beta|esr.*): true + default: false + +jobs: + thunderbird-comm: + name: thunderbird-comm-version-bump + worker: + repo-param-prefix: comm_ + shipping-product: thunderbird + treeherder: + platform: thunderbird-release/opt + symbol: Rel(VerBump) + tier: 1 + kind: build diff --git a/comm/taskcluster/ci/repackage-l10n/kind.yml b/comm/taskcluster/ci/repackage-l10n/kind.yml new file mode 100644 index 0000000000..fd909a5d06 --- /dev/null +++ b/comm/taskcluster/ci/repackage-l10n/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind:transforms + - gecko_taskgraph.transforms.repackage_l10n:transforms + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - shippable-l10n-signing + - shippable-l10n-mac-signing + - shippable-l10n-mac-notarization + - toolchain + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +job-template: + upstream-mac-kind: + by-build-type: + debug: shippable-l10n-mac-signing + default: + by-project: + comm-(central|beta|esr.*): shippable-l10n-mac-notarization + default: shippable-l10n-mac-signing + worker-type: 'b-linux-gcp' + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + linux-.*: + - repackage/base.py + - repackage/linux32_signed.py + linux64.*: + - repackage/base.py + - repackage/linux64_signed.py + macosx64.*: + - repackage/base.py + - repackage/osx_signed.py + win32.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win32_signed.py + win64.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win64_signed.py + package-formats: + by-build-platform: + linux-.*: [mar] + linux64.*: [mar] + macosx64.*: [mar, dmg, pkg] + win32.*: [mar, installer] + win64.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx + treeherder-group: L10n-Rpk diff --git a/comm/taskcluster/ci/repackage-msi/kind.yml b/comm/taskcluster/ci/repackage-msi/kind.yml new file mode 100644 index 0000000000..5020884a9a --- /dev/null +++ b/comm/taskcluster/ci/repackage-msi/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-signing + - repackage-signing-l10n + - fetch + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + +job-template: + worker-type: 'b-win2012' + mozharness: + use-magic-mh-args: false + use-caches: false + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win32_signed.py + win64\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msi] + fetches: + fetch: + - wix-3.14.0 diff --git a/comm/taskcluster/ci/repackage-msix/kind.yml b/comm/taskcluster/ci/repackage-msix/kind.yml new file mode 100644 index 0000000000..2c700b0509 --- /dev/null +++ b/comm/taskcluster/ci/repackage-msix/kind.yml @@ -0,0 +1,89 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + # We're going to generate tasks based on this. We `depend` on and fetch the package from this. + - build-signing + - toolchain + +group-by: platform + +only-for-build-platforms: + - win32/opt + - win64/opt + - win64/debug + - win32-shippable/opt + - win64-shippable/opt + +job-template: + worker-type: b-linux-gcp + worker: + max-run-time: 2700 + # This is for fast try iteration; `repackage-shippable-l10n-msix` is the "real one". + run-on-projects: [] + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win32_signed.py + win64\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msix, msix-store] + fetches: + toolchain: + - linux64-msix-packaging + msix: + vendor: Mozilla Thunderbird + channel: + by-release-type: + beta.*: beta + release.*: official + default: nightly + identity-name: + by-package-format: + msix: + by-release-type: + beta.*: MozillaThunderbirdBeta + release.*: MozillaThunderbird + default: MozillaThunderbirdDaily + msix-store: + by-release-type: + beta.*: ThunderbirdBeta + release.*: MZLA + default: ThunderbirdDaily + publisher: + by-package-format: + msix: + by-level: + "1": "CN=Mozilla Fake SPC" + "2": "CN=Mozilla Fake SPC" + default: + # This value needs to line up with the signing certificate + # we're going to sign with. We only sign potential release + # or nightly builds with proper certificates, so even on + # level 3 repositories, some build types are expected to + # be signed with our fake certificate. + by-build-platform: + .*-shippable: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + default: "CN=Mozilla Fake SPC" + msix-store: "CN=D72CA2AC-1021-4C81-8ED2-0CF10CF5F5B6" + publisher-display-name: + by-package-format: + msix: "MZLA Technologies" + msix-store: "Mozilla Thunderbird" diff --git a/comm/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml b/comm/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml new file mode 100644 index 0000000000..304b0e9d2f --- /dev/null +++ b/comm/taskcluster/ci/repackage-shippable-l10n-msix/kind.yml @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage:transforms + - comm_taskgraph.transforms.repackage_msix:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + # We're going to generate tasks based on this. We `depend` on and fetch the package from this. + - build-signing + # We're not going to generate tasks based on these. But we are going to `depend` on and fetch + # langpacks from these. + - shippable-l10n + - toolchain + +not-for-kind-dependencies: + - shippable-l10n + +group-by: platform + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + +job-template: + worker-type: b-linux-gcp + worker: + max-run-time: 2700 + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + win32\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win32_signed.py + win64\b.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win64_signed.py + package-formats: [msix, msix-store] + fetches: + toolchain: + - linux64-msix-packaging + msix: + vendor: Mozilla Thunderbird + channel: + by-release-type: + beta.*: beta + release.*: official + default: nightly + identity-name: + by-package-format: + msix: + by-release-type: + beta.*: MozillaThunderbirdBeta + release.*: MozillaThunderbird + default: MozillaThunderbirdDaily + msix-store: + by-release-type: + beta.*: ThunderbirdBeta + release.*: MZLA + default: ThunderbirdDaily + publisher: + by-package-format: + msix: + by-level: + "1": "CN=Mozilla Fake SPC" + "2": "CN=Mozilla Fake SPC" + default: + # This value needs to line up with the signing certificate + # we're going to sign with. We only sign potential release + # or nightly builds with proper certificates, so even on + # level 3 repositories, some build types are expected to + # be signed with our fake certificate. + by-build-platform: + .*-shippable: "CN=Mozilla Corporation, OU=Firefox Engineering Operations, O=Mozilla Corporation, L=Mountain View, S=California, C=US" + default: "CN=Mozilla Fake SPC" + msix-store: "CN=D72CA2AC-1021-4C81-8ED2-0CF10CF5F5B6" + publisher-display-name: + by-package-format: + msix: "MZLA Technologies" + msix-store: "Mozilla Thunderbird" diff --git a/comm/taskcluster/ci/repackage-signing-l10n/kind.yml b/comm/taskcluster/ci/repackage-signing-l10n/kind.yml new file mode 100644 index 0000000000..a2b100cf24 --- /dev/null +++ b/comm/taskcluster/ci/repackage-signing-l10n/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage_signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + + +kind-dependencies: + - repackage-l10n + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt diff --git a/comm/taskcluster/ci/repackage-signing-msi/kind.yml b/comm/taskcluster/ci/repackage-signing-msi/kind.yml new file mode 100644 index 0000000000..eb71971e94 --- /dev/null +++ b/comm/taskcluster/ci/repackage-signing-msi/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-msi + +only-for-build-platforms: + - win32-shippable/opt + - win32/opt + - win64-shippable/opt + - win64/opt diff --git a/comm/taskcluster/ci/repackage-signing-msix/kind.yml b/comm/taskcluster/ci/repackage-signing-msix/kind.yml new file mode 100644 index 0000000000..38604e8375 --- /dev/null +++ b/comm/taskcluster/ci/repackage-signing-msix/kind.yml @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-msix diff --git a/comm/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml b/comm/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml new file mode 100644 index 0000000000..e4e1fd841f --- /dev/null +++ b/comm/taskcluster/ci/repackage-signing-shippable-l10n-msix/kind.yml @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage_signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - repackage-shippable-l10n-msix diff --git a/comm/taskcluster/ci/repackage-signing/kind.yml b/comm/taskcluster/ci/repackage-signing/kind.yml new file mode 100644 index 0000000000..13eb3f46ce --- /dev/null +++ b/comm/taskcluster/ci/repackage-signing/kind.yml @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage_signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + + +kind-dependencies: + - repackage + +only-for-build-platforms: + - win32-shippable/opt + - win64-shippable/opt + - win64-asan-reporter-shippable/opt diff --git a/comm/taskcluster/ci/repackage/kind.yml b/comm/taskcluster/ci/repackage/kind.yml new file mode 100644 index 0000000000..e5f842c189 --- /dev/null +++ b/comm/taskcluster/ci/repackage/kind.yml @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.repackage_set_upstream_mac_kind:transforms + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.repackage:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - build-signing + - build-mac-signing + - build-mac-notarization + - toolchain + +only-for-build-platforms: + - linux-shippable/opt + - linux64-shippable/opt + - linux64-asan-reporter-shippable/opt + - macosx64-shippable/opt + - macosx64/opt + - macosx64/debug + - win32-shippable/opt + - win64-shippable/opt + - win64-asan-reporter-shippable/opt + +job-template: + upstream-mac-kind: + by-build-type: + debug: build-mac-signing + default: + by-project: + comm-(central|beta|esr.*): build-mac-notarization + default: build-mac-signing + worker-type: b-linux-gcp + worker: + max-run-time: + by-build-platform: + .*asan.*: 2700 + default: 900 + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + by-build-platform: + linux-.*: + - repackage/base.py + - repackage/linux32_signed.py + linux64.*: + - repackage/base.py + - repackage/linux64_signed.py + macosx64.*: + - repackage/base.py + - repackage/osx_signed.py + win32.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win32_signed.py + win64.*: + - repackage/base.py + - repackage/tb_win32_sfx_stub.py + - repackage/win64_signed.py + package-formats: + by-build-platform: + linux-.*: [mar] + linux64.*: [mar] + macosx64.*: [mar, dmg, pkg] + win32.*: [mar, installer] + win64.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx diff --git a/comm/taskcluster/ci/searchfox/kind.yml b/comm/taskcluster/ci/searchfox/kind.yml new file mode 100644 index 0000000000..337c931e93 --- /dev/null +++ b/comm/taskcluster/ci/searchfox/kind.yml @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.build_lints:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + index: + product: thunderbird + treeherder: + symbol: Searchfox(idx) + kind: build + tier: 2 + run-on-projects: [] + optimization: null + use-system-python: false + +jobs: + linux64-searchfox/debug: + description: "Linux64 Debug Searchfox" + index: + job-name: linux64-searchfox-debug + treeherder: + platform: linux64/debug + worker-type: b-linux-gcp + worker: + max-run-time: 36000 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "linux" + run: + using: mozharness + comm-checkout: true + actions: [build] + config-paths: + - comm/testing/mozharness/configs + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_sub_linux_configs/64_searchfox_and_debug.py + script: "mozharness/scripts/fx_desktop_build.py" + tooltool-downloads: public + keep-artifacts: false + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-libotr + - linux64-nasm + - linux64-node + - linux64-pkgconf + - linux64-rust + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + + macosx64-searchfox/debug: + description: "MacOS X x64 Debug Cross-compile Searchfox" + index: + job-name: macosx64-searchfox-debug + treeherder: + platform: osx-cross/debug + worker-type: b-linux-gcp + worker: + max-run-time: 36000 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "macosx" + run: + using: mozharness + comm-checkout: true + actions: [build] + config-paths: + - comm/testing/mozharness/configs + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: cross-debug-searchfox + keep-artifacts: false + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-cctools-port + - linux64-clang + - linux64-dump_syms + - linux64-hfsplus + - linux64-libdmg + - linux64-nasm + - linux64-node + - linux64-rust-macos + - macosx64-x64-libotr + - macosx64-sdk + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + + win64-searchfox/debug: + description: "Win64 Searchfox Debug (clang)" + index: + job-name: win64-searchfox-debug + treeherder: + platform: windows2012-64/debug + worker-type: b-linux-gcp + worker: + max-run-time: 36000 + env: + PERFHERDER_EXTRA_OPTIONS: searchfox + MOZSEARCH_PLATFORM: "windows" + run: + using: mozharness + comm-checkout: true + options: [append-env-variables-from-configs] + actions: [build] + config-paths: + - comm/testing/mozharness/configs + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_base_thunderbird.py + - builds/releng_base_linux_64_builds.py + - builds/taskcluster_sub_win64/searchfox_debug.py + tooltool-downloads: internal + extra-config: + mozconfig_platform: win64 + use-sccache: true + fetches: + toolchain: + - linux64-binutils + - linux64-cbindgen + - linux64-clang + - linux64-dump_syms + - linux64-nasm + - linux64-node + - linux64-rust-cross + - linux64-rust-size + - linux64-sccache + - linux64-winchecksec + - linux64-wine + - nsis + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - win64-libotr + - vs + fetch: + - upx-3.95-win diff --git a/comm/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml b/comm/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml new file mode 100644 index 0000000000..45bb772008 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n-mac-notarization/kind.yml @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: chunk-locales + +transforms: + - gecko_taskgraph.transforms.mac_notarization:transforms + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.upstream_artifact_task:transforms + - gecko_taskgraph.transforms.shippable_l10n_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - shippable-l10n-mac-signing + +only-for-attributes: + - shippable + +job-template: + max-run-time: 1800 + treeherder: + symbol: BMN diff --git a/comm/taskcluster/ci/shippable-l10n-mac-signing/kind.yml b/comm/taskcluster/ci/shippable-l10n-mac-signing/kind.yml new file mode 100644 index 0000000000..3a59d2fc30 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n-mac-signing/kind.yml @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: chunk-locales + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.shippable_l10n_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - shippable-l10n + +only-for-build-platforms: + - macosx64-shippable/opt + - macosx64/opt + - macosx64/debug + +job-template: + treeherder: + symbol: BMS + attributes: + shipping_phase: promote diff --git a/comm/taskcluster/ci/shippable-l10n-pre-signing/kind.yml b/comm/taskcluster/ci/shippable-l10n-pre-signing/kind.yml new file mode 100644 index 0000000000..ac3d0c28e6 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n-pre-signing/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.single_dep:loader + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - comm_taskgraph.transforms.l10n_source_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - shippable-l10n-pre + +job-template: + shipping-phase: promote diff --git a/comm/taskcluster/ci/shippable-l10n-pre/kind.yml b/comm/taskcluster/ci/shippable-l10n-pre/kind.yml new file mode 100644 index 0000000000..d2765c2f27 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n-pre/kind.yml @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - comm_taskgraph.transforms.l10n_pre:transforms + - gecko_taskgraph.transforms.build:transforms + - gecko_taskgraph.transforms.build_attrs:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +job-defaults: + shipping-phase: promote + shipping-product: thunderbird + treeherder: + symbol: L10n-pre-S + worker-type: b-linux-gcp + worker: + max-run-time: 900 + env: + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + attributes: + shippable: true + locale-list: + by-release-type: + beta|release: comm/mail/locales/shipped-locales + default: comm/mail/locales/all-locales + comm-locales-file: comm/mail/locales/l10n-changesets.json + browser-locales-file: browser/locales/l10n-changesets.json + run: + using: mozharness + comm-checkout: true + actions: + - clone-gecko-locales + - clone-monorepo + - merge-repos + - pack-merged + - gen-changesets + script: comm/taskcluster/scripts/desktop_comm_l10n.py + config-paths: + - comm/testing/mozharness/configs + config: + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/thunderbird_split_l10n.py + - taskcluster_nightly.py + options: [] + run-on-projects: [] + optimization: null + + +jobs: + shippable-l10n-pre/opt: + description: Pre-Localization artifact build + treeherder: + platform: thunderbird-release/opt diff --git a/comm/taskcluster/ci/shippable-l10n-signing/kind.yml b/comm/taskcluster/ci/shippable-l10n-signing/kind.yml new file mode 100644 index 0000000000..8e0ddd06c1 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n-signing/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: chunk-locales + +transforms: + - gecko_taskgraph.transforms.name_sanity:transforms + - gecko_taskgraph.transforms.upstream_artifact_task:transforms + - gecko_taskgraph.transforms.shippable_l10n_signing:transforms + - gecko_taskgraph.transforms.signing:transforms + - gecko_taskgraph.transforms.task:transforms + - comm_taskgraph.transforms.signing:transforms + +kind-dependencies: + - shippable-l10n + +primary-dependency: + - shippable-l10n + +only-for-attributes: + - shippable + +not-for-build-platforms: + - macosx64-shippable/opt + - macosx64/opt + - macosx64/debug diff --git a/comm/taskcluster/ci/shippable-l10n/kind.yml b/comm/taskcluster/ci/shippable-l10n/kind.yml new file mode 100644 index 0000000000..35599ffde9 --- /dev/null +++ b/comm/taskcluster/ci/shippable-l10n/kind.yml @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: platform + +transforms: + - comm_taskgraph.transforms.l10n:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build + - build-signing + - fetch + - shippable-l10n-pre + - repackage + - toolchain + +primary-dependency: build + +only-for-build-platforms: + - linux64-shippable/opt + - linux-shippable/opt + - macosx64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +job-template: + description: Localization + locales-file: + by-release-type: + beta|release: comm/mail/locales/shipped-locales + default: comm/mail/locales/all-locales + locales-per-chunk: 5 + run-on-projects: ['release'] + attributes: + shipping_phase: promote + ignore-locales: + by-build-platform: + # OSX has a special locale for japanese + macosx64.*: [ja, en-US] + default: [ja-JP-mac, en-US] + run-time: + by-build-platform: + default: 900 + win.*: 1800 + secrets: false + dependencies: + shippable-l10n-pre: shippable-l10n-pre-shippable-l10n-pre/opt + fetches: + toolchain: + by-build-platform: + default: [] + macosx64.*: + - linux64-libdmg + - linux64-hfsplus + win.*: + - linux64-wine + - nsis + fetch: + by-build-platform: + default: [] + win.*: + - upx-3.95-win + shippable-l10n-pre: + - strings_all.tar.zst + tooltool: + by-build-platform: + default: internal + linux.*: public + index: + type: shippable-l10n + product: thunderbird + job-name: + by-build-platform: + linux-shippable: linux-opt + linux64-shippable: linux64-opt + macosx64-shippable: macosx64-opt + win32-shippable: win32-opt + win64-shippable: win64-opt + worker-type: b-linux-gcp + treeherder: + symbol: L10n(B) + tier: 1 + platform: + by-build-platform: + linux64-shippable: linux64-shippable/opt + linux-shippable: linux32-shippable/opt + macosx64-shippable: osx-shippable/opt + win32-shippable: windows2012-32-shippable/opt + win64-shippable: windows2012-64-shippable/opt + env: + by-build-platform: + linux.*: # linux64 and 32 get same treatment here + EN_US_PACKAGE_NAME: target.tar.bz2 + EN_US_BINARY_URL: + artifact-reference: <build-signing/{artifact_prefix}> + macosx64.*: + EN_US_PACKAGE_NAME: target.dmg + EN_US_BINARY_URL: + artifact-reference: <repackage/{artifact_prefix}> + win.*: + EN_US_PACKAGE_NAME: target.zip + EN_US_BINARY_URL: + artifact-reference: <build-signing/{artifact_prefix}> + mozharness: + comm-checkout: true + config-paths: + - comm/testing/mozharness/configs + config: + # no default, so we fail on new entries + by-build-platform: + linux-shippable: + - single_locale/linux32.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - single_locale/thunderbird.py + - taskcluster_nightly.py + linux64-shippable: + - single_locale/linux64.py + - single_locale/tc_common.py + - single_locale/tc_linux_common.py + - single_locale/thunderbird.py + - taskcluster_nightly.py + macosx64-shippable: + - single_locale/macosx64.py + - single_locale/tc_common.py + - single_locale/tc_macosx64.py + - single_locale/thunderbird.py + - taskcluster_nightly.py + win32-shippable: + - single_locale/win32.py + - single_locale/tc_common.py + - single_locale/tc_win32.py + - single_locale/thunderbird.py + - taskcluster_nightly.py + win64-shippable: + - single_locale/win64.py + - single_locale/tc_common.py + - single_locale/tc_win64.py + - single_locale/thunderbird.py + - taskcluster_nightly.py + actions: ['list-locales', 'setup', 'repack', + 'summary'] + script: mozharness/scripts/desktop_l10n.py diff --git a/comm/taskcluster/ci/source-docs/kind.yml b/comm/taskcluster/ci/source-docs/kind.yml new file mode 100644 index 0000000000..8218c65ae4 --- /dev/null +++ b/comm/taskcluster/ci/source-docs/kind.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + rtd-webhook: + description: Triggers a build of the in-tree source documentation at source-docs.thunderbird.net + run-on-projects: ['trunk'] + index: + product: thunderbird + job-name: source-docs + treeherder: + platform: doc/opt + kind: test + symbol: doc(RTD) + tier: 2 + worker-type: t-linux-large + worker: + docker-image: {in-tree: "debian11-amd64-build"} + max-run-time: 1800 + env: + TASK_ID: {"task-reference": "<self>"} + TASKCLUSTER_PROXY_URL: http://taskcluster + run: + using: run-task + comm-checkout: true + cwd: '{checkout}' + command: > + ./comm/taskcluster/scripts/build-source-docs.sh + scopes: + - secrets:get:project/comm/thunderbird/releng/build/level-{level}/rtd-webhook + when: + files-changed: + - comm/docs/** diff --git a/comm/taskcluster/ci/source-test/clang.yml b/comm/taskcluster/ci/source-test/clang.yml new file mode 100644 index 0000000000..899b3aadc7 --- /dev/null +++ b/comm/taskcluster/ci/source-test/clang.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + always-target: true + platform: linux64/debug + worker-type: t-linux-xlarge-source + attributes: + code-review: true + worker: + docker-image: {in-tree: static-analysis-build} + max-run-time: 5400 + treeherder: + kind: other + tier: 2 + run: + using: run-task + tooltool-downloads: public + comm-checkout: true + fetches: + toolchain: + - linux64-cbindgen + - linux64-nasm + - linux64-node + optimization: + skip-unless-changed-no-suite: + - 'comm/**/*.c' + - 'comm/**/*.cc' + - 'comm/**/*.cpp' + - 'comm/**/*.m' + - 'comm/**/*.mm' + - 'comm/**/*.h' + +format: + description: Run clang-format on C/C++ patches + treeherder: + symbol: cpp(clang-format) + run: + cwd: '{checkout}' + command: >- + ./comm/taskcluster/scripts/source-test-clang-format.sh {changed_files} + + fetches: + toolchain: + - linux64-clang + - linux64-clang-tidy + - linux64-rust + - sysroot-x86_64-linux-gnu + - sysroot-wasm32-wasi + worker: + artifacts: + - type: file + name: public/code-review/clang-format.json + path: /builds/worker/clang-format.json + - type: file + name: public/code-review/clang-format.diff + path: /builds/worker/clang-format.diff diff --git a/comm/taskcluster/ci/source-test/kind.yml b/comm/taskcluster/ci/source-test/kind.yml new file mode 100644 index 0000000000..cf6079c02b --- /dev/null +++ b/comm/taskcluster/ci/source-test/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.source_test:transforms + - comm_taskgraph.transforms.source_test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - fetch + - toolchain + +jobs-from: + - clang.yml + - mozlint.yml + - python.yml + +job-defaults: + attributes: + retrigger: true diff --git a/comm/taskcluster/ci/source-test/mozlint.yml b/comm/taskcluster/ci/source-test/mozlint.yml new file mode 100644 index 0000000000..2ef12d70c6 --- /dev/null +++ b/comm/taskcluster/ci/source-test/mozlint.yml @@ -0,0 +1,287 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + always-target: true + attributes: + code-review: true + treeherder: + kind: test + tier: 1 + platform: lint/opt + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 1800 + artifacts: + - type: file + name: public/code-review/mozlint.json + path: /builds/worker/mozlint.json + env: + MOZLINT_NO_SUITE: '1' + run: + using: mach + comm-checkout: true + +codespell: + description: Checks for misspellings in text files + treeherder: + symbol: text(spell) + # We don't want patches to be backed out because of a typo, a follow-up + # bug will be good enough + tier: 2 + run: + mach: > + lint -v -l codespell -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: codespell.yml + +eslint: + description: JS lint check + treeherder: + symbol: js(ES) + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + cp -r /build/node_modules_eslint-plugin-mozilla/ tools/lint/eslint/eslint-plugin-mozilla/node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules && + ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules && + ./mach commlint -v -l eslint -f treeherder -f json:/builds/worker/mozlint.json comm/ + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: eslint.yml + +file-perm: + description: Check for incorrect permissions on source files + platform: lint/opt + treeherder: + symbol: pedantic(file-perm) + # We don't want patches to be backed out because of this, a follow-up + # bug will be good enough + tier: 2 + run: + mach: > + lint -v -l file-perm -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: file-perm.yml + +file-whitespace: + description: Check for trailing whitespaces and Windows CR + platform: lint/opt + treeherder: + symbol: pedantic(file-whitespace) + # We don't want patches to be backed out because of this, a follow-up + # bug will be good enough + tier: 2 + run: + mach: > + lint -v -l file-whitespace -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: file-whitespace.yml + +fluent-lint: + description: Check for problems with Fluent files. + treeherder: + symbol: l10n(fluent) + run: + mach: > + lint -l fluent-lint -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: fluent-lint.yml + +license: + description: Check for license blocks in source files. + treeherder: + symbol: pedantic(license) + run: + mach: > + lint -v -l license -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: license.yml + +l10n-conflicts: + description: l10n tests for strings with errors and conflicts with cross-channel + treeherder: + symbol: l10n(l1nt-conflicts) + always-target: false + run-on-projects: [] + run: + mach: > + lint -W -l l10n -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: l10n.yml + +localization: + description: l10n tests for strings with errors + treeherder: + symbol: l10n(l1nt) + run: + mach: > + lint -v -l l10n -f treeherder + --config-path=comm/tools/lint + comm/* + attributes: + code-review: false + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: l10n.yml + +lintpref: + description: Check for duplicates between all.js and StaticPrefList.yaml. + treeherder: + symbol: misc(pref) + run: + mach: > + lint -v -l lintpref -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: lintpref.yml + +mingw-cap: + description: lint for MinGW Capitalization issues + treeherder: + symbol: misc(mingw) + run: + mach: > + lint -v -l mingw-capitalization -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: mingw-capitalization.yml + +py-black: + description: black run over the gecko codebase + treeherder: + symbol: py(black) + run: + mach: > + lint -v -l black -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: black.yml + +py-ruff: + description: Run ruff over the codebase + treeherder: + symbol: py(ruff) + run: + mach: > + commlint -v -l ruff -f treeherder + -f json:/builds/worker/mozlint.json + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: ruff.yml + +shellcheck: + description: shellcheck run over the Thunderbird codebase + treeherder: + symbol: misc(shell) + run-on-projects: [] + run: + mach: > + lint -v -l shellcheck -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: shellcheck.yml + +stylelint: + description: CSS lint check + treeherder: + symbol: stylelint + run: + using: run-task + cwd: '{checkout}' + command: > + cp -r /build/node_modules_eslint node_modules && + ./mach commlint -v -l stylelint -f treeherder + -f json:/builds/worker/mozlint.json + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: stylelint.yml + +trojan-source: + description: Trojan Source attack - CVE-2021-42572 + platform: lint/opt + treeherder: + symbol: misc(trojan-source) + run: + mach: > + lint -v -l trojan-source -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/* + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: trojan-source.yml + +yaml: + description: yamllint run over the taskcluster configuration + treeherder: + symbol: pedantic(yaml) + run: + mach: > + lint -v -l yaml -f treeherder + -f json:/builds/worker/mozlint.json + --config-path=comm/tools/lint + comm/mail/config + comm/taskcluster + comm/.*.yml + optimization: + skip-unless-mozlint: + root-path: comm + mozlint-config: yaml.yml diff --git a/comm/taskcluster/ci/source-test/python.yml b/comm/taskcluster/ci/source-test/python.yml new file mode 100644 index 0000000000..b3ffc9bc32 --- /dev/null +++ b/comm/taskcluster/ci/source-test/python.yml @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + platform: linux1804-64/opt + always-target: true + worker-type: t-linux-xlarge-source + worker: + # A build image instead of the lint image m-c uses, because of the + # comm-checkout requirement. + docker-image: {in-tree: "debian11-amd64-build"} + max-run-time: 3600 + env: + MOZCONFIG: comm/mail/config/mozconfigs/linux64/source + treeherder: + kind: test + tier: 1 + run: + comm-checkout: true + +comm_taskgraph-tests: + description: taskcluster/taskgraph unit tests + treeherder: + symbol: py(ctg) + run: + using: run-task + cwd: "{checkout}" + command: > + ./mach configure && + ./mach python-test --subsuite comm_taskgraph + optimization: + skip-unless-changed-no-suite: + - "taskcluster/**/*.py" + - "comm/taskcluster/**/*.py" diff --git a/comm/taskcluster/ci/test/compiled.yml b/comm/taskcluster/ci/test/compiled.yml new file mode 100644 index 0000000000..67098d1798 --- /dev/null +++ b/comm/taskcluster/ci/test/compiled.yml @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run-without-variant: false + variants: ["1proc"] + mozharness: + script: desktop_unittest.py + config: + by-test-platform: + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + - unittests/thunderbird_extra.py + macosx.*: + - unittests/mac_unittest.py + - unittests/thunderbird_extra.py + windows.*: + - unittests/win_unittest.py + - unittests/thunderbird_extra.py + supports-artifact-builds: false + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: GTest + instance-size: xlarge + run-on-projects: + by-test-platform: + macosx.*: [] # currently fails on macOS + default: built-projects + test-manifest-loader: null # don't load tests in the taskgraph + target: null + tier: default diff --git a/comm/taskcluster/ci/test/kind.yml b/comm/taskcluster/ci/test/kind.yml new file mode 100644 index 0000000000..13061808f1 --- /dev/null +++ b/comm/taskcluster/ci/test/kind.yml @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.test:loader + +kind-dependencies: + - build + - build-mac-signing + - build-signing + - repackage-signing-shippable-l10n-msix + - repackage-signing-msix + - fetch + - toolchain + +transforms: + - comm_taskgraph.transforms.tests:transforms + - gecko_taskgraph.transforms.test:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - compiled.yml + - tests.yml + +# These apply to all included test yml files listed in jobs-from +job-defaults: + attributes: + retrigger: true + optimization: + by-test-platform: + .*-tsan.*/opt: + skip-unless-backstop-no-suite: null + .*-asan.*/opt: + skip-unless-backstop-no-suite: null + default: + skip-suite-only: null + mozharness: + requires-signed-builds: + by-test-platform: + windows11-64.*: + by-variant: + msix: true + default: false + default: false + fetches: + toolchain: + by-test-platform: + win.*: + - win32-fix-stacks + - win32-minidump-stackwalk + macosx.*: + - macosx64-fix-stacks + - macosx64-minidump-stackwalk + default: + - linux64-fix-stacks + - linux64-minidump-stackwalk diff --git a/comm/taskcluster/ci/test/test-platforms.yml b/comm/taskcluster/ci/test/test-platforms.yml new file mode 100644 index 0000000000..985eda9d57 --- /dev/null +++ b/comm/taskcluster/ci/test/test-platforms.yml @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file maps build platforms to test platforms. In some cases, a +# single build may be tested on multiple test platforms, but a single test +# platform can only link to one build platform. Both build and test platforms +# are represented as <platform>/<type>, where <type> is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + +# Linux platforms (matching /linux.*/) + +linux1804-64-qr/opt: + build-platform: linux64/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +linux1804-64-shippable-qr/opt: + build-platform: linux64-shippable/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +linux1804-64-qr/debug: + build-platform: linux64/debug + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +linux1804-64-asan-qr/opt: + build-platform: linux64-asan/opt + test-sets: + - mochitest-tests + - xpcshell-tests + +linux1804-64-tsan-qr/opt: + build-platform: linux64-tsan/opt + test-sets: + - mochitest-tests + - xpcshell-tests + +linux1804-64-ccov-qr/opt: + build-platform: linux64-ccov/opt + test-sets: + - mochitest-tests + - xpcshell-tests + +# Windows platforms (matching /windows.*/) + +# windows 11 - 32bit +windows11-32-2009-qr/opt: + build-platform: win32/opt + build-signing-label: build-signing-win32/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +windows11-32-2009-shippable-qr/opt: + build-platform: win32-shippable/opt + build-signing-label: build-signing-win32-shippable/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +windows11-32-2009-qr/debug: + build-platform: win32/debug + build-signing-label: build-signing-win32/debug + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +# windows 11 - 64bit +windows11-64-2009-qr/opt: + build-platform: win64/opt + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/opt + default: build-signing-win64/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +windows11-64-2009-shippable-qr/opt: + build-platform: win64-shippable/opt + build-signing-label: + by-variant: + msix: repackage-signing-shippable-l10n-msix-win64-shippable/opt + default: build-signing-win64-shippable/opt + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +windows11-64-2009-qr/debug: + build-platform: win64/debug + build-signing-label: + by-variant: + msix: repackage-signing-msix-win64/debug + default: build-signing-win64/debug + test-sets: + - gtest-tests + - mochitest-tests + - xpcshell-tests + +windows11-64-2009-asan-qr/opt: + build-platform: win64-asan/opt + test-sets: + - mochitest-tests + +# MacOS X platforms (matching /macosx.*/) + +macosx1015-64-qr/opt: + build-platform: macosx64/opt + test-sets: + - mochitest-tests + - xpcshell-tests + +macosx1015-64-qr/debug: + build-platform: macosx64/debug + test-sets: + - mochitest-tests + - xpcshell-tests + +macosx1015-64-shippable-qr/opt: + build-platform: macosx64-shippable/opt + test-sets: + - mochitest-tests + - xpcshell-tests + +# macosx1100-64-shippable/opt: +# build-platform: macosx64-shippable/opt +# test-sets: +# - mochitest-tests +# - xpcshell-tests diff --git a/comm/taskcluster/ci/test/test-sets.yml b/comm/taskcluster/ci/test/test-sets.yml new file mode 100644 index 0000000000..5e487e1050 --- /dev/null +++ b/comm/taskcluster/ci/test/test-sets.yml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# + +gtest-tests: + - gtest + +mochitest-tests: + - mochitest-thunderbird + +xpcshell-tests: + - xpcshell diff --git a/comm/taskcluster/ci/test/tests.yml b/comm/taskcluster/ci/test/tests.yml new file mode 100644 index 0000000000..9d3105d5e9 --- /dev/null +++ b/comm/taskcluster/ci/test/tests.yml @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# Each stanza here describes a particular test suite or sub-suite. These are +# processed through the transformations described in kind.yml to produce a +# bunch of tasks. See the schema in `taskcluster/gecko_taskgraph/transforms/tests.py` +# for a description of the fields used here. + +# Note that these are in lexical order, as enforced by the task loader. + +# Adding a new test type or running tests on a new platform? Be sure to review +# https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy + +xpcshell: + description: "xpcshell test run" + suite: + category: xpcshell + name: + by-variant: + msix: xpcshell-msix + default: xpcshell + target: + by-test-platform: + windows11-64.*: + by-variant: + msix: target.installer.msix + default: null + default: null + treeherder-symbol: X(X) + variants: + - msix + run-on-projects: + by-variant: + msix: + by-test-platform: + windows11-64.*: built-projects + default: [] + default: built-projects + tier: + by-variant: + msix: 2 + default: + by-test-platform: + .*-asan.*: 2 + .*-tsan.*: 2 + macosx1100.*: 2 + windows11.*: 1 + default: default + chunks: + by-test-platform: + .*-ccov.*/.*: 8 + .*-tsan.*/opt: 8 + default: 4 + instance-size: default + max-run-time: + by-variant: + msix: 7200 + default: + by-test-platform: + windows.*/opt: 5400 + .*-ccov(-qr)?/.*: 7200 + .*-tsan.*/opt: 7200 + default: 2700 + allow-software-gl-layers: false + mozharness: + script: desktop_unittest.py + config: + by-test-platform: + windows.*: + - unittests/win_unittest.py + - unittests/thunderbird_extra.py + macosx.*64.*: + - unittests/mac_unittest.py + - unittests/thunderbird_extra.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + - unittests/thunderbird_extra.py + extra-options: + by-test-platform: + windows1.*: + - --threads=2 + default: [] + requires-signed-builds: + by-test-platform: + windows1.-64-asan-qr/opt: false + windows.*: true + default: false + fetches: + toolchain: + by-test-platform: + linux.*: + - linux64-fix-stacks + - linux64-node + - linux64-minidump-stackwalk + macosx.*: + - macosx64-fix-stacks + - macosx64-node + - macosx64-minidump-stackwalk + win.*64.*: + - win32-fix-stacks + - win64-node + - win32-minidump-stackwalk + win.*32.*: + - win32-fix-stacks + - win32-node + - win32-minidump-stackwalk + default: [] + +mochitest-thunderbird: + description: "Mochitest browser-chrome-thunderbird run" + suite: + category: mochitest + name: mochitest-browser-chrome-thunderbird + target: + by-test-platform: + windows11-64.*: + by-variant: + msix: target.installer.msix + default: null + default: null + treeherder-symbol: M(bct) + built-projects-only: true + test-manifest-loader: null # don't load tests in the taskgraph + loopback-video: true + max-run-time: + by-variant: + msix: 5400 + default: + by-test-platform: + linux.*64-ccov.*/.*: 9000 + linux.*64-tsan.*/opt: 9000 + default: 3600 + variants: + - msix + run-on-projects: + by-variant: + msix: + by-test-platform: + windows11-64.*: built-projects + default: [] + default: built-projects + tier: + by-variant: + msix: 2 + default: + by-test-platform: + .*-asan.*: 2 + .*-tsan.*: 2 + macosx1100.*: 2 + windows11.*: 1 + default: default + chunks: + by-test-platform: + linux.*64-asan.*/opt: 16 + linux.*64-tsan.*/opt: 15 + windows1.-64-asan.*/opt: 9 + default: 7 + mozharness: + script: desktop_unittest.py + mochitest-flavor: browser + config: + by-test-platform: + windows.*: + - unittests/win_unittest.py + - unittests/thunderbird_extra.py + macosx.*64.*: + - unittests/mac_unittest.py + - unittests/thunderbird_extra.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + - unittests/thunderbird_extra.py + instance-size: + by-test-platform: + linux.*64-tsan.*/opt: xlarge # runs out of memory on default/m3.large + default: default + allow-software-gl-layers: false diff --git a/comm/taskcluster/ci/toolchain/clang.yml b/comm/taskcluster/ci/toolchain/clang.yml new file mode 100644 index 0000000000..a74d39f612 --- /dev/null +++ b/comm/taskcluster/ci/toolchain/clang.yml @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + run-on-projects: [] + run: + using: toolchain-script + resources: + - 'build/build-clang/build-clang.py' + worker-type: b-linux-xlarge-gcp + worker: + max-run-time: 7200 + + +# The clang compilers defined below are the same as what's defined in +# mozilla-central for Firefox builds, except that the Android compiler-rt +# pieces have been left out. + +linux64-clang-16: + description: "Clang 16 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TL(clang-16) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: linux64-clang + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang-16-raw + - linux64-aarch64-compiler-rt-16 + - linux64-x64-compiler-rt-16 + - linux64-x86-compiler-rt-16 + - macosx64-aarch64-compiler-rt-16 + - macosx64-x64-compiler-rt-16 + - wasm32-wasi-compiler-rt-16 + - win32-compiler-rt-16 + - win64-compiler-rt-16 + +win64-clang-16: + description: "Clang-cl 16 toolchain build" + attributes: + local-toolchain: true + treeherder: + symbol: TW64(clang-16) + worker-type: b-linux-gcp + worker: + max-run-time: 600 + run: + script: repack-clang.sh + toolchain-alias: + by-project: + toolchains: null + default: win64-clang + toolchain-artifact: public/build/clang.tar.zst + fetches: + toolchain: + - linux64-cctools-port + - win64-clang-16-raw + - linux64-aarch64-compiler-rt-16 + - linux64-x64-compiler-rt-16 + - linux64-x86-compiler-rt-16 + - macosx64-aarch64-compiler-rt-16 + - macosx64-x64-compiler-rt-16 + - wasm32-wasi-compiler-rt-16 + - win32-compiler-rt-16 + - win64-compiler-rt-16 diff --git a/comm/taskcluster/ci/toolchain/kind.yml b/comm/taskcluster/ci/toolchain/kind.yml new file mode 100644 index 0000000000..c2e321e52d --- /dev/null +++ b/comm/taskcluster/ci/toolchain/kind.yml @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: comm_taskgraph.loader.merge:loader +reference-base-path: taskcluster/ci + +kind-dependencies: + - fetch + - packages + +reference-jobs: + - linux64-aarch64-compiler-rt-16 + - linux64-binutils + - linux64-cbindgen + - linux64-cctools-port + - linux64-clang-16-profile + - linux64-clang-16-raw + - linux64-clang-16-stage1 + - linux64-clang-mingw-x64 + - linux64-clang-mingw-x86 + - linux64-clang-tidy + - linux64-dump_syms + - linux64-fix-stacks + - linux64-grcov + - linux64-hfsplus + - linux64-libdmg + - linux64-llvm-symbolizer + - linux64-makecab + - linux64-mar-tools + - linux64-minidump-stackwalk + - linux64-mkbom + - linux64-msix-packaging + - linux64-nasm + - linux64-node + - linux64-pkgconf + - linux64-python + - linux64-rust + - linux64-rust-cross + - linux64-rust-dev + - linux64-rust-macos + - linux64-rust-macos-toolchain + - linux64-rust-size + - linux64-rust-toolchain + - linux64-rust-windows + - linux64-rust-windows-toolchain + - linux64-sccache + - linux64-toolchain-sysroot + - linux64-upx + - linux64-winchecksec + - linux64-wine + - linux64-x64-compiler-rt-16 + - linux64-x86-compiler-rt-16 + - linux64-xar + - macosx64-aarch64-compiler-rt-16 + - macosx64-clang-16-raw + - macosx64-fix-stacks + - macosx64-minidump-stackwalk + - macosx64-node + - macosx64-sdk + - macosx64-sdk-toolchain + - macosx64-x64-compiler-rt-16 + - mingw32-rust + - nsis + - sysroot-aarch64-linux-gnu + - sysroot-i686-linux-gnu + - sysroot-wasm32-wasi + - sysroot-x86_64-linux-gnu + - wasm32-wasi-compiler-rt + - win32-compiler-rt-16 + - win32-fix-stacks + - win32-minidump-stackwalk + - win32-node + - win64-cbindgen + - win64-clang-16-raw + - win64-clang-16-stage1 + - win64-compiler-rt-16 + - win64-dump_syms + - win64-llvm-symbolizer + - win64-mozmake + - win64-nasm + - win64-node + - win64-python + - win64-rust + - win64-rust-size + - win64-sccache + - win64-vs2019 + - win64-winchecksec + +transforms: + - gecko_taskgraph.transforms.try_job:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.cached_tasks:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs-from: + - clang.yml + - libotr.yml diff --git a/comm/taskcluster/ci/toolchain/libotr.yml b/comm/taskcluster/ci/toolchain/libotr.yml new file mode 100644 index 0000000000..04e7765b05 --- /dev/null +++ b/comm/taskcluster/ci/toolchain/libotr.yml @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +job-defaults: + description: 'libotr library build' + worker-type: b-linux-gcp + worker: + max-run-time: 1800 + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + run-on-projects: [] + run: + using: comm-toolchain-script + script: build-libotr.sh + toolchain-artifact: public/build/libotr.tar.xz + +linux32-libotr: + treeherder: + symbol: TL(lib32otr) + run: + arguments: ['linux32'] + fetches: + toolchain: + - linux64-clang + - linux64-binutils + - sysroot-i686-linux-gnu + +linux64-libotr: + treeherder: + symbol: TL(libotr) + run: + arguments: ['linux64'] + fetches: + toolchain: + - linux64-clang + - linux64-binutils + - sysroot-x86_64-linux-gnu + +linux-aarch64-libotr: + treeherder: + symbol: TL(libotr-aarch64) + tier: 2 + run: + arguments: ['linux-aarch64'] + fetches: + toolchain: + - sysroot-aarch64-linux-gnu + - linux64-clang + - linux64-binutils + +macosx64-x64-libotr: + treeherder: + symbol: TM(libotr) + worker: + env: + MACOS_SDK_DIR: "MacOSX13.3.sdk" + MACOS_TARGET_SDK: "10.12" + run: + arguments: ['macosx64'] + fetches: + toolchain: + - linux64-binutils + - linux64-cctools-port + - linux64-clang + # Uses the sdk for builds, not the -toolchain sdk + - macosx64-sdk + +macosx64-aarch64-libotr: + treeherder: + symbol: TM(libotr-aarch64) + worker: + env: + MACOS_SDK_DIR: "MacOSX13.3.sdk" + MACOS_TARGET_SDK: "11.0" + run: + arguments: ['macosx64-aarch64'] + fetches: + toolchain: + - linux64-binutils + - linux64-cctools-port + - linux64-clang + - macosx64-sdk + +win32-libotr: + worker: + docker-image: {in-tree: tb-debian-mingw} + treeherder: + symbol: TW32(libotr) + run: + arguments: ['win32'] + +win64-libotr: + worker: + docker-image: {in-tree: tb-debian-mingw} + treeherder: + symbol: TW64(libotr) + run: + arguments: ['win64'] diff --git a/comm/taskcluster/ci/upload-symbols/kind.yml b/comm/taskcluster/ci/upload-symbols/kind.yml new file mode 100644 index 0000000000..68a9d439e9 --- /dev/null +++ b/comm/taskcluster/ci/upload-symbols/kind.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.multi_dep:loader + +group-by: platform + +transforms: + - gecko_taskgraph.transforms.upload_symbols:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +kind-dependencies: + - build + - toolchain + +primary-dependency: build + +only-for-build-platforms: + - linux64-shippable/opt + - linux-shippable/opt + - macosx64-x64-shippable/opt + - macosx64-aarch64-shippable/opt + - win32-shippable/opt + - win64-shippable/opt + +job-template: + description: Upload Symbols + worker-type: b-linux-gcp + use-system-python: false + worker: + docker-image: {in-tree: "debian11-base"} + max-run-time: 900 + env: + # {level} gets replaced in the upload_symbols transform + SYMBOL_SECRET: "project/comm/thunderbird/releng/build/level-{level}/gecko-symbol-upload" + MAKECAB: /builds/worker/fetches/makecab/makecab + MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + run: + using: mach + mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/cidata/target.crashreporter-symbols-full.tar.zst>"} + sparse-profile: upload-symbols + scopes: + - secrets:get:project/comm/thunderbird/releng/build/level-{level}/gecko-symbol-upload + fetches: + toolchain: + - linux64-makecab diff --git a/comm/taskcluster/comm_taskgraph/__init__.py b/comm/taskcluster/comm_taskgraph/__init__.py new file mode 100644 index 0000000000..cd54612dfb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/__init__.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +from importlib import import_module + +from gecko_taskgraph import GECKO +from gecko_taskgraph.optimize.schema import set_optimization_schema # noqa: F401 + +from comm_taskgraph.optimize import thunderbird_optimizations + +logger = logging.getLogger(__name__) + +COMM = os.path.join(GECKO, "comm") +COMM_SCRIPTS = os.path.join(COMM, "taskcluster", "scripts") + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + from comm_taskgraph.parameters import register_parameters + + logger.info("{} path registered".format(__name__)) + register_parameters() + + set_optimization_schema(thunderbird_optimizations) + + _import_modules( + [ + "documentation", + "util.docker", + "actions", + "target_tasks", + "transforms.job.toolchain", + ] + ) + + +def _import_modules(modules): + for module in modules: + import_module(".{}".format(module), package=__name__) diff --git a/comm/taskcluster/comm_taskgraph/actions.py b/comm/taskcluster/comm_taskgraph/actions.py new file mode 100644 index 0000000000..6095f3e6c8 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/actions.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.actions.util import create_tasks, fetch_graph_and_labels +from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS + + +def is_release_promotion_available(parameters): + return parameters["project"] in RELEASE_PROMOTION_PROJECTS + + +@register_callback_action( + name="l10n-bump", + title="L10n Bumper Automation", + symbol="l10n_bump", + description="L10n bumper action.", + order=500, + context=[], + available=is_release_promotion_available, +) +def l10n_bump_action(parameters, graph_config, _input, task_group_id, task_id): + """ + Runs the 'l10n_bump' task. + """ + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + to_run = [label for label, entry in full_task_graph.tasks.items() if "l10n-bump" in entry.kind] + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) diff --git a/comm/taskcluster/comm_taskgraph/decision.py b/comm/taskcluster/comm_taskgraph/decision.py new file mode 100644 index 0000000000..a075b47cf5 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/decision.py @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import sys + +from redo import retry +from taskgraph.decision import ( + _determine_more_accurate_base_ref, + _determine_more_accurate_base_rev, + _get_env_prefix, +) +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.taskcluster import get_artifact +from taskgraph.util.vcs import get_repository + +from gecko_taskgraph.util.backstop import is_backstop +from gecko_taskgraph.util.partials import populate_release_history +from gecko_taskgraph.util.taskgraph import ( + find_decision_task, + find_existing_tasks_from_previous_kinds, +) + +from . import COMM +from comm_taskgraph.parameters import get_defaults + +logger = logging.getLogger(__name__) + +BALROG_PRODUCT = "Thunderbird" + +# Backstop defaults +BACKSTOP_TIME_INTERVAL = 60 * 22 # minutes +INTEGRATION_PROJECTS = {"comm-central"} + +PER_PROJECT_PARAMETERS = { + "ash": { + "target_tasks_method": "ash_tasks", + }, + "jamun": { + "target_tasks_method": "nightly_desktop", + "release_type": "nightly", + }, + "try-comm-central": { + "enable_always_target": True, + "target_tasks_method": "try_cc_tasks", + }, + "comm-central": { + "target_tasks_method": "comm_central_tasks", + "release_type": "nightly", + }, + "comm-beta": { + "target_tasks_method": "mozilla_beta_tasks", + "release_type": "beta", + }, + "comm-esr115": { + "target_tasks_method": "mozilla_esr115_tasks", + "release_type": "release", + }, +} + +CRON_OPTIONS = { + "nightly_desktop": { + "existing_tasks": lambda parameters, graph_config: get_existing_tasks( + parameters, graph_config + ), + "release_history": lambda parameters, graph_config: populate_release_history( + BALROG_PRODUCT, parameters["project"] + ), + } +} + + +def get_decision_parameters(graph_config, parameters): + logger.info("{}.get_decision_parameters called".format(__name__)) + + # Apply default values for all Thunderbird CI projects + parameters.update(get_defaults(graph_config.vcs_root)) + + # If the target method is nightly, we should build partials. This means + # knowing what has been released previously. + # An empty release_history is fine, it just means no partials will be built + project = parameters["project"] + + if project in PER_PROJECT_PARAMETERS: + # Upstream will set target_tasks_method to "default" when nothing is set + if parameters["target_tasks_method"] == "default": + del parameters["target_tasks_method"] + + # If running from .cron.yml, do not overwrite existing parameters + update_parameters = [ + (_k, _v) + for _k, _v in PER_PROJECT_PARAMETERS[project].items() + if _k not in parameters or not parameters[_k] + ] + parameters.update(update_parameters) + logger.info("project parameters set for project {} from {}.".format(project, __file__)) + else: + # Projects without a target_tasks_method should not exist for Thunderbird CI + raise Exception("No target_tasks_method is defined for project {}.".format(project)) + + del parameters["backstop"] + parameters["backstop"] = is_backstop( + parameters, + trust_domain="comm", + time_interval=BACKSTOP_TIME_INTERVAL, + integration_projects=INTEGRATION_PROJECTS, + ) + for n in ( + "COMM_BASE_REPOSITORY", + "COMM_BASE_REV", + "COMM_HEAD_REPOSITORY", + "COMM_HEAD_REV", + "COMM_HEAD_REF", + ): + val = os.environ.get(n, "") + parameters[n.lower()] = val + + repo_path = COMM + repo = get_repository(repo_path) + logger.info("Determining comm_base_ref...") + parameters["comm_base_ref"] = _determine_more_accurate_base_ref( + repo, + candidate_base_ref="", + head_ref=parameters.get("comm_head_ref"), + base_rev=parameters.get("comm_base_rev"), + ) + logger.info("Determining comm_base_rev...") + parameters["comm_base_rev"] = _determine_more_accurate_base_rev( + repo, + base_ref=parameters["comm_base_ref"], + candidate_base_rev=parameters.get("comm_base_rev"), + head_rev=parameters.get("comm_head_rev"), + env_prefix=_get_env_prefix(graph_config), + ) + + parameters.setdefault("release_history", dict()) + if parameters.get("tasks_for", "") == "cron": + for key, _callable in CRON_OPTIONS.get(parameters["target_tasks_method"], {}).items(): + result = _callable(parameters, graph_config) + parameters[key] = result + + +def get_existing_tasks(parameters, graph_config): + """ + Find the decision task corresponding to the on-push graph, and return + a mapping of labels to task-ids from it. + """ + try: + decision_task = retry( + find_decision_task, + args=(parameters, graph_config), + attempts=4, + sleeptime=5 * 60, + ) + except Exception: + logger.exception("Didn't find existing push task.") + sys.exit(1) + + _, task_graph = TaskGraph.from_json(get_artifact(decision_task, "public/full-task-graph.json")) + return find_existing_tasks_from_previous_kinds(task_graph, [decision_task], []) diff --git a/comm/taskcluster/comm_taskgraph/documentation.py b/comm/taskcluster/comm_taskgraph/documentation.py new file mode 100644 index 0000000000..76e8f979cd --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/documentation.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +from gecko_taskgraph.util.verify import documentation_paths + +from . import COMM + +documentation_paths.add(os.path.join(COMM, "taskcluster", "docs")) diff --git a/comm/taskcluster/comm_taskgraph/files_changed.py b/comm/taskcluster/comm_taskgraph/files_changed.py new file mode 100644 index 0000000000..10deb65004 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/files_changed.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Support for optimizing tasks based on the set of files that have changed. +""" + +import logging +import os +from subprocess import CalledProcessError + +from taskgraph.util.memoize import memoize +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path + +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.hg import get_json_automationrelevance +from mozversioncontrol import InvalidRepoPath, get_repository_object + +logger = logging.getLogger(__name__) + + +@memoize +def get_changed_files(repository, revision): + """ + Get the set of files changed in the push headed by the given revision. + Responses are cached, so multiple calls with the same arguments are OK. + """ + contents = get_json_automationrelevance(repository, revision) + try: + changesets = contents["changesets"] + except KeyError: + # We shouldn't hit this error in CI. + if os.environ.get("MOZ_AUTOMATION"): + raise + + # We're likely on an unpublished commit, grab changed files from + # version control. + return get_locally_changed_files(GECKO) + + logger.debug("{} commits influencing task scheduling:".format(len(changesets))) + changed_files = set() + for c in changesets: + desc = "" # Support empty desc + if c["desc"]: + desc = c["desc"].splitlines()[0].encode("ascii", "ignore") + logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc)) + changed_files |= set(c["files"]) + + return changed_files + + +def get_files_changed_extended(params): + """ + Get the set of files changed in the push head from possibly multiple + head_repositories. + """ + changed_files = set() + + repo_keys = [key for key in params.keys() if key.endswith("head_repository")] + + def prefix_changed(_changed, prefix): + if not prefix: + return _changed + else: + return {join_path(prefix, file) for file in _changed} + + for repo_key in repo_keys: + repo_prefix = repo_key.replace("head_repository", "") + rev_key = f"{repo_prefix}head_rev" + repo_subdir_key = f"{repo_prefix}src_path" + + repository = params.get(repo_key) + revision = params.get(rev_key) + repo_subdir = params.get(repo_subdir_key, "") + + if not repository or not revision: + logger.warning( + f"Missing `{repo_key}` or `{rev_key}` parameters; " + "assuming all files have changed" + ) + return True + + changed_files |= prefix_changed(get_changed_files(repository, revision), repo_subdir) + + return changed_files + + +def check(params, file_patterns): + """Determine whether any of the files changed in the indicated push to + https://hg.mozilla.org match any of the given file patterns.""" + changed_files = get_files_changed_extended(params) + if not changed_files: + logger.warning( + "changed_files from automationrelevance is empty; assuming all files have changed" + ) + return True + + for pattern in file_patterns: + for path in changed_files: + if match_path(path, pattern): + return True + + return False + + +@memoize +def get_locally_changed_files(repo): + try: + vcs = get_repository_object(repo) + return set(vcs.get_outgoing_files("AM")) + except (InvalidRepoPath, CalledProcessError): + return set() diff --git a/comm/taskcluster/comm_taskgraph/loader/__init__.py b/comm/taskcluster/comm_taskgraph/loader/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/comm/taskcluster/comm_taskgraph/loader/merge.py b/comm/taskcluster/comm_taskgraph/loader/merge.py new file mode 100644 index 0000000000..00b600cd0d --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/merge.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from gecko_taskgraph.loader.transform import loader as transform_loader + +from comm_taskgraph.loader.reference import loader as reference_loader + + +def loader(kind, path, config, params, loaded_tasks): + """ + Look up jobs via reference loader at reference-base-path using the list + reference-jobs-from, followed by jobs-from. + + This loader has been tested with "fetch" jobs successfully. Anything else + is likely to have bugs. + """ + # Make a copy of config for reference_loader. Use pop here to remove the + # fields that aren't used by the transform loader + reference_config = { + "kind-dependencies": config.get("kind-dependencies", None), + "reference-base-path": config.pop("reference-base-path"), + "reference-jobs": config.pop("reference-jobs", None), + } + for job in reference_loader(kind, path, reference_config, params, loaded_tasks): + yield job + + for job in transform_loader(kind, path, config, params, loaded_tasks): + yield job diff --git a/comm/taskcluster/comm_taskgraph/loader/reference.py b/comm/taskcluster/comm_taskgraph/loader/reference.py new file mode 100644 index 0000000000..cb4d8f0565 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/reference.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from taskgraph.util.python_path import find_object +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.yaml import load_yaml + +logger = logging.getLogger(__name__) + + +def _get_aliases(kind, job, project): + aliases = {job["name"]} + + if kind == "toolchain": + if job["run"].get("toolchain-alias"): + resolve_keyed_by( + job["run"], + "toolchain-alias", + item_name=f"{kind}-{job['name']}", + project=project, + ) + aliaslist = job["run"].get("toolchain-alias") + if aliaslist is not None: + if isinstance(aliaslist, str): + aliaslist = [aliaslist] + for alias in aliaslist: + aliases.add(alias) + + return aliases + + +def _expand_aliases(kind, inputs, project): + """Given the list of all "reference-jobs" pulled in from upstream, return a + set with all job names and aliases. + For example "linux64-clang" is an alias of "linux64-clang-13", and both + of those names will be included in the returned set.""" + rv = set() + for input_job in inputs: + for alias in _get_aliases(kind, input_job, project): + rv.add(alias) + return rv + + +def _get_loader(path, config): + try: + _loader = config["loader"] + except KeyError: + raise KeyError("{!r} does not define `loader`".format(path)) + return find_object(_loader) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Loads selected jobs from a different taskgraph hierarchy. + + This loads jobs of the given kind from the taskgraph rooted at `base-path`, + and includes all the jobs with names or aliases matching the names in the + `jobs` key. + """ + base_path = config.pop("reference-base-path") + sub_path = os.path.join(base_path, kind) + + logger.debug("Reference loader: load tasks from {}".format(sub_path)) + sub_config = load_yaml(sub_path, "kind.yml") + _loader = _get_loader(sub_path, sub_config) + inputs = _loader(kind, sub_path, sub_config, params, loaded_tasks) + + jobs = config.pop("reference-jobs", None) + + config.update(sub_config) + project = params["project"] + + if jobs is not None: + jobs = set(jobs) + + found_reference_jobs = [job for job in inputs if (_get_aliases(kind, job, project) & jobs)] + + # Check for jobs listed as a reference job in Thunderbird's config + # that do not exist in upstream. + reference_alias_names = _expand_aliases(kind, found_reference_jobs, project) + if reference_alias_names >= jobs: + return found_reference_jobs + else: + missing_jobs = jobs - reference_alias_names + raise Exception( + "Reference jobs not found in kind {}: {}".format(kind, ", ".join(missing_jobs)) + ) + else: + return inputs diff --git a/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml new file mode 100644 index 0000000000..81a11b2793 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-release: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-generate-checksums: build + release-generate-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-generate-checksums-signing + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number} + +# Configuration for individual files. Extends 'default', above. +mapping: + SHA256SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 256 hashes" + from: + - release-generate-checksums + pretty_name: SHA256SUMMARY + checksums_path: SHA256SUMMARY + SHA512SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 512 hashes" + from: + - release-generate-checksums + pretty_name: SHA512SUMMARY + checksums_path: SHA512SUMMARY + KEY: + <<: *default + description: "Public side of the key that was used to sign the release artifacts" + pretty_name: KEY + checksums_path: KEY + SHA256SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha512 hashes" + pretty_name: SHA256SUMS + checksums_path: SHA256SUMS + SHA256SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA256SUMS.asc + checksums_path: SHA256SUMS.asc + SHA512SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha256 hashes" + pretty_name: SHA512SUMS + checksums_path: SHA512SUMS + SHA512SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA512SUMS.asc + checksums_path: SHA512SUMS.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml new file mode 100644 index 0000000000..13fab07342 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-source: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-checksums-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/beetmover-checksums/source + +# Configuration for individual files. Extends 'default', above. +mapping: + target-source.checksums: + <<: *default + description: "Checksums file for the source zip files" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.beet + checksums_path: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.beet + target-source.checksums.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.asc + checksums_path: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_files.yml b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml new file mode 100644 index 0000000000..53270eb347 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-source: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/source + +# Configuration for individual files. Extends 'default', above. +mapping: + source.tar.xz: + <<: *default + description: "Source file with the in-tree code archived" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.source.tar.xz + checksums_path: + by-platform: + thunderbird-source: source/thunderbird-${version}.source.tar.xz + source.tar.xz.asc: + <<: *default + description: "Detached signature for the source file" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.source.tar.xz.asc + checksums_path: + by-platform: + thunderbird-source: source/thunderbird-${version}.source.tar.xz.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml new file mode 100644 index 0000000000..b6fa63ec84 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + shippable-l10n-pre: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + shippable-l10n-pre-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - shippable-l10n-pre-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/source + +# Configuration for individual files. Extends 'default', above. +mapping: + strings_all.tar.zst: + <<: *default + description: "Strings sources for localuized builds and langpacks" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst + strings_all.tar.zst.asc: + <<: *default + description: "Detached signature for the strings source file" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst.asc + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst.asc + l10n-changesets.json: + <<: *default + description: "Source info used to build strings_all.tar.zst" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json + l10n-changesets.json.asc: + <<: *default + description: "Detached signature for changesets file" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json.asc + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml new file mode 100644 index 0000000000..26540e677c --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml @@ -0,0 +1,334 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/candidates +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing + release-sign-and-push-langpacks: scriptworker +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + tools_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + filename_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + - ${version}-candidates/build${build_number}/${path_platform} + +mapping: + buildhub.json: + <<: *default + all_locales: false + description: "Build related information to be consumed by Buildhub service" + pretty_name: buildhub.json + checksums_path: ${path_platform}/${locale}/buildhub.json + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: thunderbird-${version}.common.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: thunderbird-${version}.cppunittest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: thunderbird-${version}.crashreporter-symbols.zip + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.crashreporter-symbols.zip + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: thunderbird-${version}.mochitest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.mozinfo.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: thunderbird-${version}.reftest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: thunderbird-${version}.talos.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: thunderbird-${version}.awsy.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: thunderbird-${version}.test_packages.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.test_packages.json + target.txt: + <<: *default + description: "File containing buildid and revision" + pretty_name: thunderbird-${version}.txt + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.txt + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: thunderbird-${version}.web-platform.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: thunderbird-${version}.xpcshell.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + locale_prefix: '' + pretty_name: ${filename_platform}_info.txt + checksums_path: ${filename_platform}_info.txt + destinations: + - ${version}-candidates/build${build_number} + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: ${path_platform}/${locale}/mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + locale_prefix: '' + pretty_name: jsshell-${path_platform}.zip + checksums_path: jsshell/jsshell-${path_platform}.zip + destinations: + - ${version}-candidates/build${build_number}/jsshell + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird" + locale_prefix: '' + from: + - build + only_for_platforms: + - linux-shippable + - linux64-shippable + - macosx64-shippable + - win32-shippable + - win64-shippable + pretty_name: ${locale}.xpi + checksums_path: ${path_platform}/xpi/${locale}.xpi + destinations: + - ${version}-candidates/build${build_number}/${path_platform}/xpi + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mar + checksums_path: mar-tools/${tools_platform}/mar + not_for_platforms: + - win32-shippable + - win64-shippable + destinations: + - ${version}-candidates/build${build_number}/mar-tools + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mbsdiff + checksums_path: mar-tools/${tools_platform}/mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + destinations: + - ${version}-candidates/build${build_number}/mar-tools + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + pretty_name: thunderbird-${version}.tar.bz2 + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2 + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + pretty_name: thunderbird-${version}.tar.bz2.asc + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2.asc + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: Thunderbird ${version}.pkg + checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.pkg + target.dmg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + pretty_name: Thunderbird ${version}.dmg + checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.dmg + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.zip + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + from: + - repackage-signing + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: Thunderbird Setup ${version}.exe + checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.exe + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: Thunderbird Setup ${version}.msi + checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.msi + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + locale_prefix: 'multi/' + pretty_name: Thunderbird Setup ${version}.msix + checksums_path: ${path_platform}/multi/Thunderbird Setup ${version}.msix + target.complete.mar: + <<: *default + description: "Complete MAR to serve as updates" + all_locales: true + from: + - mar-signing + pretty_name: thunderbird-${version}.complete.mar + checksums_path: update/${path_platform}/${locale}/thunderbird-${version}.complete.mar + update_balrog_manifest: true + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: thunderbird-${previous_version}-${version}.partial.mar + checksums_path: update/${path_platform}/${locale}/thunderbird-${previous_version}-${version}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml new file mode 100644 index 0000000000..f3b2225f96 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/candidates +default_locales: + - en-US +tasktype_map: + beetmover-repackage: beetmover + release-beetmover-signed-langpacks: signing +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + linux: 'linux-i686' + linux64: 'linux-x86_64' + macosx64: 'mac' + win32: 'win32' + win64: 'win64' + +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: '' + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform} + +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: thunderbird-${version}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/${locale}/thunderbird-${version}.checksums.beet + target-langpack.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename for the langpack" + locale_prefix: '' + from: + - release-beetmover-signed-langpacks + pretty_name: ${locale}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}/xpi diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml new file mode 100644 index 0000000000..ee2ca6c800 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml @@ -0,0 +1,434 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/nightly +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + stage_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +mapping: + buildhub.json: + <<: *default + description: "Build related information to be consumed by Buildhub service" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + KEY: + <<: *default + from: + - signing + description: "Public side of the key that was used to sign the release artifacts" + pretty_name: KEY + checksums_path: KEY + only_for_platforms: + - linux64-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch} + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch} + - latest-${branch}-l10n + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json + target.txt: + <<: *default + description: "File containing buildid and revision" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.txt + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.txt + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}_info.txt + checksums_path: thunderbird-${version}.${locale}.${filename_platform}_info.txt + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + pretty_name: jsshell-${filename_platform}.zip + checksums_path: jsshell-${filename_platform}.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird" + from: + - build + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + - macosx64-shippable + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.langpack.xpi + checksums_path: thunderbird-${version}.${locale}.langpack.xpi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n/${filename_platform}/xpi + - latest-${branch}-l10n/${filename_platform}/xpi + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mar + checksums_path: mar + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mbsdiff + checksums_path: mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2 + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2 + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.pkg + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.pkg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.dmg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.dmg + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.dmg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.zip + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + from: + - repackage-signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.exe + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.exe + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.msi + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.msi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + all_locales: true + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.multi.${filename_platform}.installer.msix + checksums_path: thunderbird-${version}.multi.${filename_platform}.installer.msix + destinations: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + target.complete.mar: + <<: *default + description: "The main installer we ship our mobile products baked within" + all_locales: true + from: + - mar-signing + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.complete.mar + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.complete.mar + update_balrog_manifest: true + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + checksums_path: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + by-locale: + en-US: + - partials/${year}/${month}/${upload_date}-${branch} + default: + - partials/${year}/${month}/${upload_date}-${branch}-l10n diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml new file mode 100644 index 0000000000..4de3ae780e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + - pub/thunderbird/nightly +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + beetmover-repackage: beetmover +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +# Configuration for individual files. Extends 'default', above. +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.checksums + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.checksums diff --git a/comm/taskcluster/comm_taskgraph/optimize.py b/comm/taskcluster/comm_taskgraph/optimize.py new file mode 100644 index 0000000000..24f7e67dfa --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/optimize.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird specific taskgraph optimizers. +""" + +import logging + +from taskgraph.optimize.base import Any, OptimizationStrategy, register_strategy +from taskgraph.util.memoize import memoize +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path +from taskgraph.util.yaml import load_yaml +from voluptuous import Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.optimize.schema import default_optimizations +from mozlint.pathutils import filterpaths + +from comm_taskgraph import files_changed + +logger = logging.getLogger(__name__) + + +def is_excluded(check_path, file_patterns): + for pattern in file_patterns: + if match_path(check_path, pattern): + return True + return False + + +def is_suite(check_path): + suite_patterns = ("editor", "suite") + return is_excluded(check_path, suite_patterns) + + +@memoize +def get_non_suite_changed_files(repository, revision): + """ + Returns the list of changed files from comm- repository (no prefixing) + with suite/** and editor/** files removed. + """ + return { + file + for file in files_changed.get_changed_files(repository, revision) + if not is_suite(file) + } + + +@register_strategy("comm-skip-unless-changed") +class SkipUnlessChanged(OptimizationStrategy): + def should_remove_task(self, task, params, file_patterns): + # pushlog_id == -1 - this is the case when run from a cron.yml job + if params.get("pushlog_id") == -1: + return False + + changed = files_changed.check(params, file_patterns) + if not changed: + logger.debug( + "no files found matching a pattern in `skip-unless-changed` for " + task.label + ) + return True + return False + + +@register_strategy("skip-suite-only") +class SkipSuiteOnly(OptimizationStrategy): + def should_remove_task(self, task, params, arg): + # pushlog_id == -1 - this is the case when run from a cron.yml job + if params.get("pushlog_id") == -1: + return False + + if params.get("project") == "try-comm-central": + # Do not try to use this optimization on try-c-c builds + return False + + repository = params.get("comm_head_repository") + revision = params.get("comm_head_rev") + non_suite_changed_files = get_non_suite_changed_files(repository, revision) + # non_suite_changed_files will be an empty set (Falsy) for suite-only pushes + # so "skip" this task + if not non_suite_changed_files: + return True + return False + + +@register_strategy("skip-unless-mozlint") +class SkipUnlessMozlint(OptimizationStrategy): + schema = { + "root-path": Optional(str), + "mozlint-config": Required(str), + } + + def should_remove_task(self, task, params, args): + include = [] + exclude = [] + extensions = [] + support_files = [] + + root_path = join_path(GECKO, args.get("root-path", "")) + mozlint_root = join_path(root_path, "tools", "lint") + mozlint_yaml = join_path(mozlint_root, args["mozlint-config"]) + + logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.") + linter_config = load_yaml(mozlint_yaml) + for check, config in linter_config.items(): + include += config.get("include", []) + exclude += config.get("exclude", []) + extensions += [e.strip(".") for e in config.get("extensions", [])] + support_files += config.get("support-files", []) + + changed_files = files_changed.get_files_changed_extended(params) + + # Support files may not be part of "include" patterns, so check first + # Do not remove (return False) if any changed + for pattern in support_files: + for path in changed_files: + if match_path(path, pattern): + return False + + to_lint, to_exclude = filterpaths( + GECKO, + list(changed_files), + include=include, + exclude=exclude, + extensions=extensions, + ) + + # to_lint should be an empty list if there is nothing to check + if not to_lint: + return True + return False + + +register_strategy( + "skip-unless-backstop-no-suite", args=("skip-unless-backstop", "skip-suite-only") +)(Any) + +register_strategy( + "skip-unless-changed-no-suite", args=("comm-skip-unless-changed", "skip-suite-only") +)(Any) + +optimizations = ( + {"skip-suite-only": None}, + {"skip-unless-backstop-no-suite": None}, + {"skip-unless-changed-no-suite": [str]}, + {"skip-unless-mozlint": SkipUnlessMozlint.schema}, +) + +thunderbird_optimizations = default_optimizations + optimizations diff --git a/comm/taskcluster/comm_taskgraph/parameters.py b/comm/taskcluster/comm_taskgraph/parameters.py new file mode 100644 index 0000000000..a3cd3aebd5 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/parameters.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.parameters import extend_parameters_schema +from voluptuous import Required + +from gecko_taskgraph.parameters import gecko_parameters_schema as comm_parameters_schema +from gecko_taskgraph.parameters import get_app_version, get_version + +logger = logging.getLogger(__name__) + + +# Called at import time when comm_taskgraph:register is called +comm_parameters_schema.update( + { + Required("comm_base_repository"): str, + Required("comm_base_ref"): str, + Required("comm_base_rev"): str, + Required("comm_head_ref"): str, + Required("comm_head_repository"): str, + Required("comm_head_rev"): str, + Required("comm_src_path"): str, + } +) + + +def get_defaults(repo_root=None): + return { + "app_version": get_app_version(product_dir="comm/mail"), + "version": get_version("comm/mail"), + "comm_src_path": "comm/", + } + + +def register_parameters(): + """Register the additional comm_* parameters with taskgraph. Note that + defaults_fn is registered, but it does not actually run by design in the + decision task due to 'strict' being True in that case.""" + extend_parameters_schema(comm_parameters_schema, defaults_fn=get_defaults) diff --git a/comm/taskcluster/comm_taskgraph/target_tasks.py b/comm/taskcluster/comm_taskgraph/target_tasks.py new file mode 100644 index 0000000000..cfb758200d --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/target_tasks.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from gecko_taskgraph.target_tasks import ( + _target_task, + _try_task_config, + filter_on_platforms, + filter_out_shippable, + filter_out_shipping_phase, + standard_filter, +) + +from comm_taskgraph.try_option_syntax import _try_cc_option_syntax + +logger = logging.getLogger(__name__) + + +@_target_task("comm_searchfox_index") +def target_tasks_searchfox(full_task_graph, parameters, graph_config): + """Select tasks required for indexing Thunderbird for Searchfox web site each day""" + return [ + "searchfox-linux64-searchfox/debug", + "searchfox-macosx64-searchfox/debug", + "searchfox-win64-searchfox/debug", + ] + + +@_target_task("comm_central_tasks") +def target_tasks_default(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + return [ + l + for l, t in full_task_graph.tasks.items() + if standard_filter(t, parameters) and filter_out_shipping_phase(t, parameters) + ] + + +@_target_task("try_cc_tasks") +def target_tasks_try(full_task_graph, parameters, graph_config): + try_mode = parameters["try_mode"] + if try_mode == "try_task_config": + return _try_task_config(full_task_graph, parameters, graph_config) + elif try_mode == "try_option_syntax": + return _try_cc_option_syntax(full_task_graph, parameters, graph_config) + else: + # With no try mode, we schedule nothing, allowing the user to add tasks + # later via treeherder. + return [] + + +@_target_task("ash_tasks") +def target_tasks_ash(full_task_graph, parameters, graph_config): + run_for_platforms = ( + "linux64", + "macosx64", + "win64", + ) + + def filter_source_test(task): + return task.kind == "source-test" and task.attributes.get("always_target", False) + + def _filter(task, _parameters): + return all( + [ + filter_on_platforms(task, run_for_platforms) or filter_source_test(task), + standard_filter(task, _parameters), + filter_out_shippable(task), + ] + ) + + return [l for l, t in full_task_graph.tasks.items() if _filter(t, parameters)] diff --git a/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json new file mode 100644 index 0000000000..19751bad92 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json @@ -0,0 +1,154 @@ +{ + "changesets": [ + { + "author": "Magnus Melin <mkmelin+mozilla@iki.fi>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1829065", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1829065" + } + ], + "date": [1681974919, 0], + "desc": "Bug 1829065 - re-enable browser_messageFilters.js. r=leftmostcat\n\n\nReenable the test.\nRemove the customizaton sub test since the button it's about is now different + it's using the old customization which is on its way out.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175988", + "extra": { + "branch": "default" + }, + "files": [ + "mail/test/browser/folder-widget/browser.ini", + "mail/test/browser/folder-widget/browser_messageFilters.js" + ], + "node": "e6e51c25cd7162180080914b6bc46cb0620c4a87", + "parents": ["87bed05abb8f99147cf83ecb3bc0cec6ada7f39d"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38727, + "reviewers": [ + { + "name": "leftmostcat", + "revset": "reviewer(leftmostcat)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "John Bieling <john@thunderbird.net>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828924", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828924" + } + ], + "date": [1681911897, 0], + "desc": "Bug 1828924 - Fix imports of FileReader in the WebExtension API implementation files. r=mkmelin\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175899", + "extra": { + "amend_source": "83d21c8094985f1c8024d5966de1effa93b09b40", + "branch": "default" + }, + "files": [ + "mail/components/extensions/parent/ext-compose.js", + "mail/components/extensions/parent/ext-mail.js", + "mail/components/extensions/parent/ext-messages.js" + ], + "node": "04906b08eddb74d55b2fb4868290e4759a459158", + "parents": ["e6e51c25cd7162180080914b6bc46cb0620c4a87"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38728, + "reviewers": [ + { + "name": "mkmelin", + "revset": "reviewer(mkmelin)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "Magnus Melin <mkmelin+mozilla@iki.fi>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1826744", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1826744" + } + ], + "date": [1681879233, 0], + "desc": "Bug 1826744 - \"Copy to Folder\" should be enabled for .eml files opened from file - was broken again. r=john.bieling\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175841", + "extra": { + "amend_source": "9b3f446beaa7132a9bfcd5542df376adb2af1337", + "branch": "default" + }, + "files": ["mail/base/content/mailWindowOverlay.js"], + "node": "b72ae83322b6a8d878ad710cbe1f4d59201404e1", + "parents": ["04906b08eddb74d55b2fb4868290e4759a459158"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38729, + "reviewers": [ + { + "name": "john.bieling", + "revset": "reviewer(john.bieling)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "Geoff Lankow <geoff@darktrojan.net>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817367", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817367" + } + ], + "date": [1681948136, -43200], + "desc": "Bug 1817367 - Wrap loading of virtual folder views in batch notifications. r=leftmostcat\nThis functionality was added to the base `nsMsgDBView`, but not these subclasses.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175974", + "extra": { + "amend_source": "8722f310135f7c5f58cb1dcd4573189b3e225e6e", + "branch": "default", + "rebase_source": "4ca3cf372be8029d76bbfeecfcb406d197f0dbb6" + }, + "files": [ + "mailnews/base/src/nsMsgGroupView.cpp", + "mailnews/base/src/nsMsgQuickSearchDBView.cpp", + "mailnews/base/src/nsMsgSearchDBView.cpp", + "mailnews/base/src/nsMsgThreadedDBView.cpp", + "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp" + ], + "node": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "parents": ["b72ae83322b6a8d878ad710cbe1f4d59201404e1"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38730, + "reviewers": [ + { + "name": "leftmostcat", + "revset": "reviewer(leftmostcat)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + } + ], + "visible": true +} diff --git a/comm/taskcluster/comm_taskgraph/test/conftest.py b/comm/taskcluster/comm_taskgraph/test/conftest.py new file mode 100644 index 0000000000..83ed5343f4 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +HERE = os.path.dirname(__file__) +EXT_PATH = os.path.abspath(os.path.join(HERE, "..", "..")) + +sys.path.insert(0, EXT_PATH) diff --git a/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json new file mode 100644 index 0000000000..8e90731d92 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json @@ -0,0 +1,309 @@ +{ + "changesets": [ + { + "author": "Noah <osuolale49@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1825749", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825749" + } + ], + "date": [1681922538, 0], + "desc": "Bug 1825749 - Only run IdentityCredentialStorageCleaner if FedCM is enabled. r=hpeuckmann\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174757", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/cleardata/ClearDataService.sys.mjs", + "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js" + ], + "landingsystem": "lando", + "node": "e35b8568fd3f9d49b77e791b44159589b6bcb309", + "parents": ["680421823d4d4a444a0045996d16299691565aa7"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661160, + "reviewers": [ + { + "name": "hpeuckmann", + "revset": "reviewer(hpeuckmann)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Denis Palmeiro <dpalmeiro@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1824772", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1824772" + } + ], + "date": [1681925585, 0], + "desc": "Bug 1824772: part 1 - Add jit option and static pref to toggle eager baseline hints. r=iain\n\nAdd a pref, javascript.options.jitHints, to toggle eager baseline hints.\n\nDepends on D175520\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175521", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": ["modules/libpref/init/StaticPrefList.yaml"], + "landingsystem": "lando", + "node": "3740c123c6b87c6712dda72f53b0a2a8feac5110", + "parents": ["e35b8568fd3f9d49b77e791b44159589b6bcb309"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661161, + "reviewers": [ + { + "name": "iain", + "revset": "reviewer(iain)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Tim Huang <tihuang@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817463", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463" + } + ], + "date": [1681925941, 0], + "desc": "Bug 1817463 - Deferring SafeBrowsing updates if the browser is in idle mode. r=dimi\n\nThis patch implements the behavior for deferring SafeBrowsing updates\nwhen the browser is in idle mode. The update will be deferred until the\nnext user interaction active.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175763", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/url-classifier/UrlClassifierListManager.jsm" + ], + "landingsystem": "lando", + "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce", + "parents": ["6895dba1dca74eee9f56b4ce8656b4696f7c386d"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661165, + "reviewers": [ + { + "name": "dimi", + "revset": "reviewer(dimi)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Sergey Galich <sgalich@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828524", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828524" + } + ], + "date": [1681926293, 0], + "desc": "Bug 1828524 - [Relay] Manage masks button leads to SUMO instead of relay.firefox.com r=credential-management-reviewers,dimi\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175705", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "modules/libpref/init/all.js", + "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs" + ], + "landingsystem": "lando", + "node": "091fb26cb5df9e5a61d9a43883b33bdf1736ea6e", + "parents": ["ea52cdb6d093a5eb233b10a207b64d02b7489c23"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661167, + "reviewers": [ + { + "name": "credential-management-reviewers", + "revset": "reviewer(credential-management-reviewers)" + }, + { + "name": "dimi", + "revset": "reviewer(dimi)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Randell Jesup <rjesup@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828973", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828973" + } + ], + "date": [1681928684, 0], + "desc": "Bug 1828973: Disable OCSP for web-platform tests r=jgraham\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175936", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "testing/profiles/web-platform/user.js", + "testing/web-platform/meta/webtransport/__dir__.ini" + ], + "landingsystem": "lando", + "node": "ef5c4d384d2819bb857e1812ddca469fb0b87b86", + "parents": ["ccc54cebee26a52410f7c0cbd8ff8693078a620f"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661170, + "reviewers": [ + { + "name": "jgraham", + "revset": "reviewer(jgraham)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Stanca Serban <sstanca@mozilla.com>", + "backsoutnodes": [ + { + "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce" + } + ], + "bugs": [ + { + "no": "1817463", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463" + } + ], + "date": [1681930381, -10800], + "desc": "Backed out changeset 80b29d7af431 (bug 1817463) for failures in toolkit/components/url-classifier/tests.", + "extra": { + "branch": "default", + "rebase_source": "9fc0fd23fdbaa4c30568ca895af3e1c9e3c08aea" + }, + "files": [ + "toolkit/components/url-classifier/UrlClassifierListManager.jsm" + ], + "node": "ae17e25d0f24bd1b6dee872c77c9770ad0ec7d87", + "parents": ["6ec708eb48569fcfb9bf708d91a8876383865a65"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661172, + "reviewers": [], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Bilal <bnasar@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817002", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817002" + } + ], + "date": [1681931937, 0], + "desc": "Bug 1817002 - ESMified browser/components/pocket/content. r=kpatenio\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175700", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/normandy/content/AboutPages.sys.mjs", + "toolkit/modules/NewTabUtils.sys.mjs" + ], + "landingsystem": "lando", + "node": "eff25b124b5e2c29c10442e902e9c4a9c488672c", + "parents": ["dcb12f0cfb12a4d3ae666f1660276b9cea2419ba"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661174, + "reviewers": [ + { + "name": "kpatenio", + "revset": "reviewer(kpatenio)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Sammy Khamis <skhamis@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1825905", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825905" + } + ], + "date": [1681937295, 0], + "desc": "Bug 1825905: Autofill sync engine should roundtrip data it doesn't know about r=sgalich,lina\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174834", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs" + ], + "landingsystem": "lando", + "node": "2d1f87aeb76249a22f8db78ac031305816544514", + "parents": ["5f3904256e0d38a7cffed5ef443b7b259d43ad49"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661182, + "reviewers": [ + { + "name": "sgalich", + "revset": "reviewer(sgalich)" + }, + { + "name": "lina", + "revset": "reviewer(lina)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + } + ], + "visible": true +} diff --git a/comm/taskcluster/comm_taskgraph/test/python.ini b/comm/taskcluster/comm_taskgraph/test/python.ini new file mode 100644 index 0000000000..84548afeb0 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/python.ini @@ -0,0 +1,6 @@ +[DEFAULT] +subsuite = comm_taskgraph + +[test_files_changed.py] +[test_optimization_strategies.py] +[test_parameters.py] diff --git a/comm/taskcluster/comm_taskgraph/test/test_files_changed.py b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py new file mode 100644 index 0000000000..5759520de6 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py @@ -0,0 +1,144 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import json +import os +import unittest + +import conftest # noqa: F401 +from mozunit import main + +from gecko_taskgraph.util import hg + +from comm_taskgraph import files_changed + +PARAMS = { + "comm_head_repository": "https://hg.mozilla.org/comm-central", + "comm_head_rev": "d0c328a2cc33", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "6ca54f5a4a1b", + "comm_src_path": "comm/", +} + +FILES_CHANGED_CC = [ + "mail/base/content/mailWindowOverlay.js", + "mail/components/extensions/parent/ext-compose.js", + "mail/components/extensions/parent/ext-mail.js", + "mail/components/extensions/parent/ext-messages.js", + "mail/test/browser/folder-widget/browser.ini", + "mail/test/browser/folder-widget/browser_messageFilters.js", + "mailnews/base/src/nsMsgGroupView.cpp", + "mailnews/base/src/nsMsgQuickSearchDBView.cpp", + "mailnews/base/src/nsMsgSearchDBView.cpp", + "mailnews/base/src/nsMsgThreadedDBView.cpp", + "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp", +] + +FILES_CHANGED_MC = [ + "modules/libpref/init/StaticPrefList.yaml", + "modules/libpref/init/all.js", + "testing/profiles/web-platform/user.js", + "testing/web-platform/meta/webtransport/__dir__.ini", + "toolkit/components/cleardata/ClearDataService.sys.mjs", + "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js", + "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs", + "toolkit/components/normandy/content/AboutPages.sys.mjs", + "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs", + "toolkit/components/url-classifier/UrlClassifierListManager.jsm", + "toolkit/modules/NewTabUtils.sys.mjs", +] + +FILES_CHANGED = sorted( + FILES_CHANGED_MC + [os.path.join("comm", file) for file in FILES_CHANGED_CC] +) + + +class FakeResponse: + def __init__(self, url, **kwargs): + if "comm-central" in url: + self.filename = "cc_automationrelevance.json" + elif "mozilla-central" in url: + self.filename = "mc_automationrelevance.json" + else: + raise Exception(f"Invalid automation URL: {url}") + + def json(self): + with open(os.path.join(os.path.dirname(__file__), self.filename)) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files.clear() + self.old_get = hg.requests.get + + def fake_get(url, **kwargs): + return FakeResponse(url) + + hg.requests.get = fake_get + + def tearDown(self): + hg.requests.get = self.old_get + files_changed.get_changed_files.clear() + + def test_get_changed_files_mc(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_changed_files(PARAMS["head_repository"], PARAMS["head_rev"])), + FILES_CHANGED_MC, + ) + + def test_get_changed_files_cc(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted( + files_changed.get_changed_files( + PARAMS["comm_head_repository"], PARAMS["comm_head_rev"] + ) + ), + FILES_CHANGED_CC, + ) + + def test_get_changed_files_extended(self): + """Get_changed_files_extended correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_files_changed_extended(PARAMS)), + FILES_CHANGED, + ) + + +class TestCheck(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files[PARAMS["head_repository"], PARAMS["head_rev"]] = set( + FILES_CHANGED_MC + ) + files_changed.get_changed_files[ + PARAMS["comm_head_repository"], PARAMS["comm_head_rev"] + ] = set(FILES_CHANGED_CC) + + def tearDown(self): + files_changed.get_changed_files.clear() + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + self.assertFalse(files_changed.check(PARAMS, ["comm/nosuch/**"])) + + def test_check_match_mc(self): + self.assertTrue(files_changed.check(PARAMS, ["toolkit/**"])) + + def test_check_match_cc(self): + self.assertTrue(files_changed.check(PARAMS, ["comm/mail/**"])) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py new file mode 100644 index 0000000000..8e43760714 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py @@ -0,0 +1,143 @@ +# Any copyright is dedicated to the public domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import hashlib +from datetime import datetime +from time import mktime + +import conftest # noqa: F401 +import pytest +import responses +from mozunit import main +from taskgraph.task import Task + +from gecko_taskgraph.optimize import registry + +from comm_taskgraph.optimize import SkipSuiteOnly + + +def generate_task(): + task = {} + task.setdefault("label", "task-label") + task.setdefault("kind", "build") + task.setdefault("task", {}) + task.setdefault("attributes", {}) + + for attr in ( + "dependencies", + "optimization", + "soft_dependencies", + "release_artifacts", + ): + task.setdefault(attr, None) + + task["task"].setdefault("label", task["label"]) + return Task.from_json(task) + + +def idfn(param): + if isinstance(param, tuple): + return param[0].__name__ + return None + + +def generate_json_push_data(files_changed): + return {"changesets": [{"desc": "commit comment", "files": files_changed, "node": "cdefgh"}]} + + +@pytest.fixture +def params(): + return { + "branch": "comm-central", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "zyxwvu", + "comm_head_repository": "https://hg.mozilla.org/comm-central", + "comm_head_rev": "abcdef", + "comm_src_path": "comm/", + "project": "comm-central", + "pushlog_id": 1, + "pushdate": mktime(datetime.now().timetuple()), + } + + +def mk_rev(strings): + data = "".join(strings).encode("utf-8") + h = hashlib.new("sha1") + h.update(data) + return h.hexdigest() + + +@responses.activate +@pytest.mark.parametrize( + "pushed_files,expected", + [ + # suite-only push + pytest.param(["suite/a/b/c.txt", "suite/b/c/d.txt"], True), + # non-suite push + pytest.param(["mail/a/b/c.txt", "mailnews/b/c/d.txt"], False), + # mixed push + pytest.param(["suite/a/b/c.txt", "calendar/b/c/d.txt"], False), + ], + ids=idfn, +) +def test_suite_only_strategy(params, pushed_files, expected): + rev = mk_rev(pushed_files) + params["comm_head_rev"] = rev + + responses.add( + responses.GET, + "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev), + json=generate_json_push_data(pushed_files), + status=200, + ) + task = generate_task() + + opt = SkipSuiteOnly() + remove = opt.should_remove_task(task, params, None) + + assert remove == expected + + +@responses.activate +@pytest.mark.parametrize( + "file_patterns,pushed_files,expected", + [ + # suite-only push, matches files-changed + pytest.param(["comm/**/*.txt"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True), + # suite-only push, does not match files-changed + pytest.param(["comm/**/*.cpp"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True), + # non-suite push, matches files changed + pytest.param(["comm/**/*.txt"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], False), + # non-suite push, does not match files changed + pytest.param(["comm/**/*.cpp"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], True), + ], + ids=idfn, +) +def test_suite_files_changed_strategy(params, file_patterns, pushed_files, expected): + rev = mk_rev(pushed_files) + params["comm_head_rev"] = rev + + # Fake the m-c json data + responses.add( + responses.GET, + "https://hg.mozilla.org/mozilla-central/json-automationrelevance/zyxwvu", + json=generate_json_push_data([]), + status=200, + ) + + responses.add( + responses.GET, + "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev), + json=generate_json_push_data(pushed_files), + status=200, + ) + task = generate_task() + + opt = registry["skip-unless-changed-no-suite"] + remove = opt.should_remove_task(task, params, file_patterns) + + assert remove == expected + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/test/test_parameters.py b/comm/taskcluster/comm_taskgraph/test/test_parameters.py new file mode 100644 index 0000000000..8a9d78701b --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_parameters.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +import conftest # noqa: F401 +from mozunit import main +from taskgraph.parameters import Parameters + +from comm_taskgraph.parameters import register_parameters + + +class TestCommParameters(unittest.TestCase): + + vals = { + "app_version": "app_version", + "backstop": False, + "base_repository": "base_repository", + "base_ref": "base_ref", + "base_rev": "base_rev", + "build_date": 0, + "build_number": 0, + "comm_base_repository": "comm_base_repository", + "comm_base_ref": "comm_base_ref", + "comm_base_rev": "comm_base_rev", + "comm_head_ref": "comm_head_ref", + "comm_head_repository": "comm_head_repository", + "comm_head_rev": "comm_head_rev", + "comm_src_path": "comm/", + "do_not_optimize": [], + "enable_always_target": False, + "existing_tasks": {}, + "filters": [], + "head_ref": "head_ref", + "head_repository": "head_repository", + "head_rev": "head_rev", + "head_tag": "", + "hg_branch": "hg_branch", + "level": "level", + "message": "message", + "moz_build_date": "moz_build_date", + "next_version": "next_version", + "optimize_strategies": None, + "optimize_target_tasks": False, + "owner": "owner", + "phabricator_diff": "phabricator_diff", + "project": "project", + "pushdate": 0, + "pushlog_id": "pushlog_id", + "release_enable_emefree": False, + "release_enable_partner_repack": False, + "release_enable_partner_attribution": False, + "release_eta": None, + "release_history": {}, + "release_partners": [], + "release_partner_config": None, + "release_partner_build_number": 1, + "release_type": "release_type", + "release_product": None, + "repository_type": "hg", + "required_signoffs": [], + "signoff_urls": {}, + "target_tasks_method": "target_tasks_method", + "test_manifest_loader": "default", + "tasks_for": "tasks_for", + "try_mode": "try_mode", + "try_options": None, + "try_task_config": {}, + "version": "version", + } + + def setUp(self): + register_parameters() + + def test_Parameters_check(self): + """ + Specifying all of the gecko and comm parameters doesn't result in an error. + """ + p = Parameters(**self.vals) + p.check() # should not raise + + def test_Parameters_check_missing(self): + """ + If any of the comm parameters are specified, all of them must be specified. + """ + vals = self.vals.copy() + del vals["comm_base_repository"] + p = Parameters(**vals) + self.assertRaises(Exception, p.check) + + def test_Parameters_check_extra(self): + """ + If parameters other than the global and comm parameters are specified, + an error is reported. + """ + p = Parameters(extra="data", **self.vals) + self.assertRaises(Exception, p.check) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/transforms/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py new file mode 100644 index 0000000000..a80c058a2b --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running toolchain-building jobs via dedicated scripts in comm-central +""" + +import os.path + +import taskgraph +import taskgraph.util.path as util_path +from taskgraph.util.schema import resolve_keyed_by +from voluptuous import Any, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import docker_worker_add_artifacts +from gecko_taskgraph.transforms.job.toolchain import toolchain_defaults, toolchain_run_schema +from gecko_taskgraph.util.attributes import RELEASE_PROJECTS +from gecko_taskgraph.util.hash import hash_paths as hash_paths_gecko_root + +from comm_taskgraph.util.hash import hash_paths_extended + +CACHE_TYPE = "toolchains.v3" + +TOOLCHAIN_SCRIPT_PATH = "comm/taskcluster/scripts" + + +comm_toolchain_run_schema = toolchain_run_schema.extend( + { + Required("using"): Any("comm-toolchain-script"), + Optional("script"): str, + } +) + + +def hash_paths(*args): + """ + Helper function while the single repository project is in development. + The extended version of hash_paths found in comm_taskgraph.util.hash is + not necessary (and does not work) with single-repo. This is a wrapper + function to pick the right function based on the presence of a comm/.hg + directory. + """ + comm_hg_path = util_path.join(GECKO, "comm", ".hg") + if os.path.exists(comm_hg_path): + return hash_paths_extended(*args) + else: + return hash_paths_gecko_root(*args) + + +def get_digest_data(config, run, taskdesc): + """ + Copied from gecko_taskgraph.transforms.job.toolchain, with minor + modifications to support the required script path. + """ + files = list(run.pop("resources", [])) + # This file + files.append("comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py") + # The script + if "script" in run: + files.append("{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run["script"])) + # Tooltool manifest if any is defined: + tooltool_manifest = taskdesc["worker"]["env"].get("TOOLTOOL_MANIFEST") + if tooltool_manifest: + files.append(tooltool_manifest) + + # Accumulate dependency hashes for index generation. + data = [hash_paths(GECKO, files)] + + data.append(taskdesc["attributes"]["toolchain-artifact"]) + + # If the task uses an in-tree docker image, we want it to influence + # the index path as well. Ideally, the content of the docker image itself + # should have an influence, but at the moment, we can't get that + # information here. So use the docker image name as a proxy. Not a lot of + # changes to docker images actually have an impact on the resulting + # toolchain artifact, so we'll just rely on such important changes to be + # accompanied with a docker image name change. + image = taskdesc["worker"].get("docker-image", {}).get("in-tree") + if image: + data.append(image) + + # Likewise script arguments should influence the index. + args = run.get("arguments") + if args: + data.extend(args) + + if taskdesc["attributes"].get("rebuild-on-release"): + # Add whether this is a release branch or not + data.append(str(config.params["project"] in RELEASE_PROJECTS)) + return data + + +@run_job_using( + "docker-worker", + "comm-toolchain-script", + schema=comm_toolchain_run_schema, + defaults=toolchain_defaults, +) +def docker_worker_toolchain(config, job, taskdesc): + run = job["run"] + run["comm-checkout"] = True + + worker = taskdesc["worker"] = job["worker"] + worker["chain-of-trust"] = True + + # If the task doesn't have a docker-image, set a default + worker.setdefault("docker-image", {"in-tree": "deb11-toolchain-build"}) + + # Toolchain checkouts don't live under {workdir}/checkouts + workspace = "{workdir}/workspace/build".format(**run) + gecko_path = "{}/src".format(workspace) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "GECKO_PATH": gecko_path, + "TOOLCHAIN_ARTIFACT": run["toolchain-artifact"], + } + ) + + attributes = taskdesc.setdefault("attributes", {}) + attributes["toolchain-artifact"] = run.pop("toolchain-artifact") + toolchain_artifact = attributes["toolchain-artifact"] + if not toolchain_artifact.startswith("public/build/"): + attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact) + + resolve_keyed_by( + run, + "toolchain-alias", + item_name=taskdesc["label"], + project=config.params["project"], + ) + alias = run.pop("toolchain-alias", None) + if alias: + attributes["toolchain-alias"] = alias + if "toolchain-env" in run: + attributes["toolchain-env"] = run.pop("toolchain-env") + + # Allow the job to specify where artifacts come from, but add + # public/build if it's not there already. + artifacts = worker.setdefault("artifacts", []) + if not artifacts: + docker_worker_add_artifacts(config, job, taskdesc) + + digest_data = get_digest_data(config, run, taskdesc) + + if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast: + name = taskdesc["label"].replace(f"{config.kind}-", "", 1) + taskdesc["cache"] = { + "type": CACHE_TYPE, + "name": name, + "digest-data": digest_data, + } + + run["using"] = "run-task" + run["cwd"] = run["workdir"] + run["command"] = [ + "workspace/build/src/{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run.pop("script")) + ] + run.pop("arguments", []) + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n.py b/comm/taskcluster/comm_taskgraph/transforms/l10n.py new file mode 100644 index 0000000000..51246ad888 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Do transforms specific to l10n kind +""" + +from taskgraph.transforms.base import TransformSequence, ValidateSchema +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.l10n import ( + all_locales_attribute, + chunk_locales, + copy_in_useful_magic, + handle_artifact_prefix, + handle_keyed_by, + l10n_description_schema, + make_job_description, + set_extra_config, + setup_name, +) + + +def setup_signing_dependency(config, jobs): + """Sets up a task dependency to the signing job this relates to""" + for job in jobs: + job["dependencies"].update( + { + "build": job["dependent-tasks"]["build"].label, + } + ) + + if job["attributes"]["build_platform"].startswith("win"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + + if "shippable" in job["attributes"]["build_platform"]: + if job["attributes"]["build_platform"].startswith("macosx"): + job["dependencies"].update( + {"repackage": job["dependent-tasks"]["repackage"].label} + ) + if job["attributes"]["build_platform"].startswith("linux"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + yield job + + +def handle_keyed_by_local(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locales-file", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +transforms = TransformSequence() + + +for transform_func in ( + setup_name, + copy_in_useful_magic, + handle_keyed_by_local, + ValidateSchema(l10n_description_schema), + setup_signing_dependency, + handle_keyed_by, + handle_artifact_prefix, + all_locales_attribute, + chunk_locales, + ValidateSchema(l10n_description_schema), + set_extra_config, + make_job_description, +): + transforms.add(transform_func) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py new file mode 100644 index 0000000000..a642b0e592 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Create a strings build artifact to be consumed by shippable-l10n. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locale-list", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + locale_list = job.pop("locale-list") + comm_locales_file = job.pop("comm-locales-file") + browser_locales_file = job.pop("browser-locales-file") + job["run"].update( + { + "job-script": "comm/taskcluster/scripts/build-l10n-pre.sh", + "options": [ + f"locale-list={locale_list}", + f"comm-locales-file={comm_locales_file}", + f"browser-locales-file={browser_locales_file}", + ], + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py new file mode 100644 index 0000000000..92a95cb612 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_path + +from gecko_taskgraph.transforms.build_signing import add_signed_routes +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +transforms = TransformSequence() + +transforms.add(add_signed_routes) + + +@transforms.add +def define_upstream_artifacts(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) + + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, "strings_all.tar.zst"), + get_artifact_path(job, "l10n-changesets.json"), + ], + "formats": ["autograph_gpg"], + } + ] + + task_ref = f"<{upstream_artifact_task.kind}>" + task_type = "build" + if "notarization" in upstream_artifact_task.kind: + task_type = "scriptworker" + + job["upstream-artifacts"] = [ + { + "taskId": {"task-reference": task_ref}, + "taskType": task_type, + "paths": spec["artifacts"], + "formats": spec["formats"], + } + for spec in artifacts_specifications + ] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py new file mode 100644 index 0000000000..75ced147bb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Fix-ups for comm-central merge automation +""" + +import os +import re + +from taskgraph.transforms.base import TransformSequence + +from comm_taskgraph import COMM + +transforms = TransformSequence() + + +def do_suite_verbump(replacements): + """Bump the minor version of suite version files.""" + allowed_files = ("suite/config/version.txt", "suite/config/version_display.txt") + old_version, new_version = None, None + + new_replacements = [] + for file, old, new in replacements: + if file not in allowed_files: + break + if old_version is None or new_version is None: + path = os.path.join(COMM, file) + data = open(path).read() + match = re.match(r"^(2)\.(\d+)(a1)$", data) + if match: + old_version = match.group(0) + + old_minor = match.group(2) + new_minor = str(int(old_minor) + 1) + + new_version = f"{match.group(1)}.{new_minor}{match.group(3)}" + + new_replacements.append([file, old_version, new_version]) + + if len(new_replacements) == len(replacements): + return new_replacements + else: + raise Exception(f"do_suite_version failed: {replacements}, {new_replacements}") + + +@transforms.add +def update_suite_versions(config, tasks): + for task in tasks: + if "merge_config" not in config.params: + break + behavior = config.params["merge_config"]["behavior"] + if behavior == "comm-bump-central": + merge_config = task["worker"]["merge-info"] + replacements = merge_config["replacements"] + merge_config["replacements"] = do_suite_verbump(replacements) + + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/partials.py b/comm/taskcluster/comm_taskgraph/transforms/partials.py new file mode 100644 index 0000000000..39d6cf95a7 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/partials.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to partial update building +""" +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def update_scopes(config, jobs): + """ + Firefox does some caching when building partial updates, but there's no bucket for Thunderbird + at the moment. In the meantime, remove the scope from the task to avoid an error. + """ + # If no balrog release history, then don't run + if not config.params.get("release_history"): + return + + MBSDIFF_SCOPE = "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" + + for job in jobs: + task = job["task"] + if MBSDIFF_SCOPE in task["scopes"]: + task["scopes"].remove(MBSDIFF_SCOPE) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py new file mode 100644 index 0000000000..8f6c7a5d5f --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-push-langpacks task into an actual task description. +""" + +import json +import os +from contextlib import contextmanager + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, taskref_or_string +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from mozbuild.action.langpack_manifest import get_version_maybe_buildid + +transforms = TransformSequence() + +langpack_push_description_schema = schema.extend( + { + Required("label"): str, + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): { + Required("docker-image"): {"in-tree": str}, + Required("implementation"): "docker-worker", + Required("os"): "linux", + Optional("max-run-time"): int, + Required("env"): {str: taskref_or_string}, + Required("channel"): optionally_keyed_by( + "project", "platform", Any("listed", "unlisted") + ), + Required("command"): [taskref_or_string], + }, + Required("run-on-projects"): [], + Required("scopes"): optionally_keyed_by("release-level", [str]), + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + } +) + + +@transforms.add +def set_label(config, jobs): + for job in jobs: + label = "push-langpacks-{}".format(job["primary-dependency"].label) + job["label"] = label + + yield job + + +transforms.add_validate(langpack_push_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "worker.channel", + item_name=job["label"], + project=config.params["project"], + platform=job["primary-dependency"].attributes["build_platform"], + ) + + yield job + + +@transforms.add +def copy_attributes(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + job["attributes"]["chunk_locales"] = dep_job.attributes.get("chunk_locales", ["en-US"]) + + yield job + + +@transforms.add +def filter_out_macos_jobs_but_mac_only_locales(config, jobs): + for job in jobs: + build_platform = job["primary-dependency"].attributes.get("build_platform") + + if build_platform == "linux64-shippable": + yield job + elif ( + build_platform == "macosx64-shippable" + and "ja-JP-mac" in job["attributes"]["chunk_locales"] + ): + # Other locales of the same job shouldn't be processed + job["attributes"]["chunk_locales"] = ["ja-JP-mac"] + job["label"] = job["label"].replace( + # Guard against a chunk 10 or chunk 1 (latter on try) weird munging + "-{}/".format(job["attributes"]["l10n_chunk"]), + "-ja-JP-mac/", + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(P{})".format(job["attributes"].get("l10n_chunk", "")) + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + ) + + job["dependencies"] = {dep_job.kind: dep_job.label} + job["treeherder"] = treeherder + + yield job + + +def generate_upstream_artifacts(upstream_task_ref, locales): + return [ + { + "task": upstream_task_ref, + "extract": False, + "dest": f"{locale}", + "artifact": "public/build{locale}/target.langpack.xpi".format( + locale="" if locale == "en-US" else "/" + locale + ), + } + for locale in locales + ] + + +@transforms.add +def make_fetches(config, jobs): + for job in jobs: + upstream_task_ref = get_upstream_task_ref(job, expected_kinds=("build", "shippable-l10n")) + + worker = job.setdefault("worker", {}) + worker["taskcluster-proxy"] = True + + env = worker.setdefault("env", {}) + + job_fetches = generate_upstream_artifacts( + upstream_task_ref, job["attributes"]["chunk_locales"] + ) + env["MOZ_FETCHES"] = { + "task-reference": json.dumps( + sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True + ) + } + env["MOZ_SCM_LEVEL"] = config.params["level"] + + yield job + + +def get_upstream_task_ref(job, expected_kinds): + upstream_tasks = [ + job_kind for job_kind in job["dependencies"].keys() if job_kind in expected_kinds + ] + + if len(upstream_tasks) > 1: + raise Exception("Only one dependency expected") + + return f"<{upstream_tasks[0]}>" + + +@contextmanager +def environment(key, value): + """Set an environment variable in a context""" + old_value = None + if key in os.environ: + old_value = os.environ[key] + os.environ[key] = value + try: + yield True + finally: + if old_value is None: + del os.environ[key] + else: + os.environ[key] = old_value + + +@transforms.add +def set_env(config, jobs): + buildid = config.params["moz_build_date"] + app_version = config.params.get("app_version") + + with environment("MOZ_BUILD_DATE", buildid): + langpack_version = get_version_maybe_buildid(app_version) + + for job in jobs: + job["worker"].get("env", {}).update( + { + "LANGPACK_VERSION": langpack_version, + "LOCALES": json.dumps(job["attributes"]["chunk_locales"]), + "MOZ_FETCHES_DIR": "fetches", + "ATN_CHANNEL": job["worker"].get("channel"), + } + ) + + yield job + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + del job["worker"]["channel"] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py new file mode 100644 index 0000000000..c4f39fc4eb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-flatpak-push kind into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +push_flatpak_description_schema = Schema( + { + Required("name"): str, + Required("job-from"): task_description_schema["job-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Required("flathub-scope"): str, + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(push_flatpak_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + if len(job["dependencies"]) != 1: + raise Exception("Exactly 1 dependency is required") + + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(job["dependencies"]) + + resolve_keyed_by( + job, + "worker.channel", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + if release_level(config.params["project"]) == "production": + job.setdefault("scopes", []).append( + add_scope_prefix( + config, + "{}:{}".format(job["flathub-scope"], job["worker"]["channel"]), + ) + ) + del job["flathub-scope"] + + yield job + + +def generate_upstream_artifacts(dependencies): + return [ + { + "taskId": {"task-reference": f"<{task_kind}>"}, + "taskType": "build", + "paths": ["public/build/target.flatpak.tar.xz"], + } + for task_kind in dependencies.keys() + ] diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_started.py b/comm/taskcluster/comm_taskgraph/transforms/release_started.py new file mode 100644 index 0000000000..ce0cfa9fab --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_started.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add notifications via taskcluster-notify for release tasks +""" + +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def add_notifications(config, jobs): + for job in jobs: + label = "{}-{}".format(config.kind, job["name"]) + + resolve_keyed_by(job, "emails", label, project=config.params["project"]) + emails = [email.format(config=config.__dict__) for email in job.pop("emails")] + + command = [ + "release", + "send-buglist-email", + "--version", + config.params["version"], + "--product", + job["shipping-product"], + "--revision", + config.params["comm_head_rev"], + "--build-number", + str(config.params["build_number"]), + "--repo", + config.params["comm_head_repository"], + ] + for address in emails: + command += ["--address", address] + command += [ + # We wrap this in `{'task-reference': ...}` below + "--task-group-id", + "<decision>", + ] + + job["scopes"] = ["notify:email:{}".format(address) for address in emails] + job["run"] = { + "using": "mach", + "sparse-profile": "mach", + "mach": {"task-reference": " ".join(map(shell_quote, command))}, + } + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py new file mode 100644 index 0000000000..793519f6ff --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def add_langpack_fetches(config, jobs): + """Adds the fetch configuration for the langpacks. This is done here + because Thunderbird langpacks are not signed and therefore not found as + artifacts of "shippable-l10n-signing" like they are for Firefox. Need to + use "shippable-l10n". + """ + + def depends_filter(dep_task): + return ( + dep_task.kind == "shippable-l10n" + and dep_task.attributes["build_platform"] == "linux64-shippable" + and dep_task.attributes["build_type"] == "opt" + ) + + for job in jobs: + dependencies = job.get("dependencies", {}) + fetches = job.setdefault("fetches", {}) + + # The keys are unique, like `shippable-l10n-linux64-shippable-1/opt`, so we + # can't ask for the tasks directly, we must filter for them. + for t in filter(depends_filter, config.kind_dependencies_tasks.values()): + dependencies.update({t.label: t.label}) + + fetches.update( + { + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"distribution/extensions/{loc}", + } + for loc in t.attributes["chunk_locales"] + ] + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/signing.py b/comm/taskcluster/comm_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..297fec0d2e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/signing.py @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.signed_artifacts import is_notarization_kind + +transforms = TransformSequence() + + +def check_notarization(dependencies): + """ + Determine whether a signing job is the last step of a notarization + by looking at its dependencies. + """ + for dep in dependencies: + if is_notarization_kind(dep): + return True + + +@transforms.add +def remove_widevine(config, jobs): + """ + Remove references to widevine signing. + + This is to avoid adding special cases for handling signed artifacts + in mozilla-central code. Artifact signature formats are determined in + gecko_taskgraph.util.signed_artifacts. There's no override mechanism so we + remove the autograph_widevine format here. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + widevine_scope = "project:comm:thunderbird:releng:signing:format:autograph_widevine" + if widevine_scope in task["scopes"]: + task["scopes"].remove(widevine_scope) + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_widevine" in artifact.get("formats", []): + artifact["formats"].remove("autograph_widevine") + + yield job + + +@transforms.add +def no_sign_langpacks(config, jobs): + """ + Remove langpacks from signing jobs after they are automatically added. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_langpack" in artifact.get("formats", []): + artifact["formats"].remove("autograph_langpack") + + # Make sure that there are no .xpi files in the artifact list + if all([p.endswith("target.langpack.xpi") for p in artifact["paths"]]): + payload["upstreamArtifacts"].remove(artifact) + + yield job + + +@transforms.add +def check_for_no_formats(config, jobs): + """ + Check for signed artifacts without signature formats and remove them to + avoid scriptworker errors. + Signing jobs that use macOS notarization do not have formats, so keep + those. + """ + for job in jobs: + if not check_notarization(job["dependencies"]): + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "formats" in artifact and not artifact["formats"]: + for remove_path in artifact["paths"]: + job["release-artifacts"].remove(remove_path) + + payload["upstreamArtifacts"].remove(artifact) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/source_test.py b/comm/taskcluster/comm_taskgraph/transforms/source_test.py new file mode 100644 index 0000000000..d4ae39c134 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/source_test.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import shlex + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path + +from gecko_taskgraph.files_changed import get_changed_files + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def get_patterns(job): + """Get the "run on-changed" file patterns.""" + optimization = job.get("optimization", {}) + if optimization: + return optimization.copy().popitem()[1] + return [] + + +def shlex_join(split_command): + """shlex.join from Python 3.8+""" + return " ".join(shlex.quote(arg) for arg in split_command) + + +@transforms.add +def changed_clang_format(config, jobs): + """ + Transform for clang-format job to set the commandline to only check + C++ files that were changed in the current push rather than running on + the entire repository. + """ + for job in jobs: + if job.get("name", "") == "clang-format": + repository = config.params.get("comm_head_repository") + revision = config.params.get("comm_head_rev") + + match_patterns = get_patterns(job) + changed_files = { + join_path("comm", file) for file in get_changed_files(repository, revision) + } + + cpp_files = [] + for pattern in match_patterns: + for path in changed_files: + if match_path(path, pattern): + cpp_files.append(path) + + # In the event that no C/C++ files were changed in the current push, + # the commandline will end up being invalid. But, the clang-format + # job will get dropped by optimization, so it doesn't really matter. + if cpp_files: + job["run"]["command"] = job["run"]["command"].format( + changed_files=shlex_join(cpp_files) + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_build.py b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py new file mode 100644 index 0000000000..eff16d8e69 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def munge_environment(config, jobs): + for job in jobs: + env = job["worker"]["env"] + # Remove MOZ_SOURCE_CHANGESET/REPO from the job environment and discard + # if present. Having these variables set in the environment causes problems + # with generating debug sym files. Bug 1747879. + env.pop("MOZ_SOURCE_CHANGESET", None) + env.pop("MOZ_SOURCE_REPO", None) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py new file mode 100644 index 0000000000..ecdfd577af --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build a command to run `mach tb-l10n-x-channel`. +""" + +from shlex import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from rocbuild.notify import TB_BUILD_ADDR + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for item in ["ssh-key-secret", "run.actions"]: + resolve_keyed_by(job, item, item, **{"level": str(config.params["level"])}) + yield job + + +@transforms.add +def build_command(config, jobs): + for job in jobs: + command = [ + "tb-l10n-x-channel", + "-o", + "/builds/worker/artifacts/outgoing.diff", + "--attempts", + "5", + ] + ssh_key_secret = job.pop("ssh-key-secret") + if ssh_key_secret: + command.extend(["--ssh-secret", ssh_key_secret]) + job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") + job["scopes"].append(f"notify:email:{TB_BUILD_ADDR}") + + command.extend(job["run"].pop("actions", [])) + job.setdefault("run", {}).update( + {"using": "mach", "mach": " ".join(map(shell_quote, command))} + ) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tests.py b/comm/taskcluster/comm_taskgraph/transforms/tests.py new file mode 100644 index 0000000000..9c1df4663e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tests.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to test jobs +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def optimization_keyed_by(config, tasks): + """Used to set the optimization strategy""" + for task in tasks: + resolve_keyed_by(task, "optimization", item_name=task["test-name"]) + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py new file mode 100644 index 0000000000..2f9ee143fa --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from urllib.parse import urlsplit + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev +from gecko_taskgraph.transforms.update_verify_config import ensure_wrapped_singlequote +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +# The beta regexes do not match point releases. +# In the rare event that we do ship a point +# release to beta, we need to either: +# 1) update these regexes to match that specific version +# 2) pass a second include version that matches that specific version +INCLUDE_VERSION_REGEXES = { + "beta": r"'^(\d+\.\d+b\d+)$'", + "nonbeta": r"'^\d+\.\d+(\.\d+)?$'", + # Previous major versions, for update testing before we update users to a new esr + "release-next": r"'^(91|102)\.\d+(\.\d+)?$'", +} + +MAR_CHANNEL_ID_OVERRIDE_REGEXES = { + "beta": r"'^\d+\.\d+(\.\d+)?$$,thunderbird-comm-beta,thunderbird-comm-release'", +} + + +ensure_wrapped_singlequote(INCLUDE_VERSION_REGEXES) +ensure_wrapped_singlequote(MAR_CHANNEL_ID_OVERRIDE_REGEXES) + + +@transforms.add +def add_command(config, tasks): + keyed_by_args = [ + "channel", + "archive-prefix", + "previous-archive-prefix", + "aus-server", + "override-certs", + "include-version", + "mar-channel-id-override", + "last-watershed", + ] + optional_args = [ + "updater-platform", + ] + + release_config = get_release_config(config) + + for task in tasks: + task["description"] = "generate update verify config for {}".format( + task["attributes"]["build_platform"] + ) + + command = [ + "python", + "testing/mozharness/scripts/release/update-verify-config-creator.py", + "--product", + task["extra"]["product"], + "--stage-product", + task["shipping-product"], + "--app-name", + task["extra"]["app-name"], + "--branch-prefix", + task["extra"]["branch-prefix"], + "--platform", + task["extra"]["platform"], + "--to-version", + release_config["version"], + "--to-app-version", + release_config["appVersion"], + "--to-build-number", + str(release_config["build_number"]), + "--to-buildid", + config.params["moz_build_date"], + "--to-revision", + get_branch_rev(config), + "--output-file", + "update-verify.cfg", + ] + + repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/") + command.extend(["--repo-path", repo_path]) + + if release_config.get("partial_versions"): + for partial in release_config["partial_versions"].split(","): + command.extend(["--partial-version", partial.split("build")[0]]) + + for arg in optional_args: + if task["extra"].get(arg): + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + for arg in keyed_by_args: + thing = f"extra.{arg}" + resolve_keyed_by( + task, + thing, + item_name=task["name"], + platform=task["attributes"]["build_platform"], + **{ + "release-type": config.params["release_type"], + "release-level": release_level(config.params["project"]), + }, + ) + # ignore things that resolved to null + if not task["extra"].get(arg): + continue + if arg == "include-version": + task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]] + if arg == "mar-channel-id-override": + task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]] + + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + task["run"].update( + { + "using": "mach", + "mach": " ".join(command), + } + ) + + yield task diff --git a/comm/taskcluster/comm_taskgraph/try_option_syntax.py b/comm/taskcluster/comm_taskgraph/try_option_syntax.py new file mode 100644 index 0000000000..b401bfa7a2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/try_option_syntax.py @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from gecko_taskgraph.target_tasks import ( + filter_by_uncommon_try_tasks, + filter_unsupported_artifact_builds, +) +from gecko_taskgraph.try_option_syntax import TryOptionSyntax + +logger = logging.getLogger(__name__) + + +class TryCCOptionSyntax(TryOptionSyntax): + """ + Override parse_platforms in the superclass. Removed the attempt to replace + macosx64 jobs with macosx64-shippable. + """ + + def parse_platforms(self, options, full_task_graph): + platform_arg = options["platforms"] + if platform_arg == "all": + return None + + results = [] + for build in platform_arg.split(","): + results.append(build) + + test_platforms = { + t.attributes["test_platform"] + for t in full_task_graph.tasks.values() + if "test_platform" in t.attributes + } + build_platforms = { + t.attributes["build_platform"] + for t in full_task_graph.tasks.values() + if "build_platform" in t.attributes + } + all_platforms = test_platforms | build_platforms + bad_platforms = set(results) - all_platforms + if bad_platforms: + raise Exception("Unknown platform(s) [%s] specified for try" % ",".join(bad_platforms)) + + return results + + +def _try_cc_option_syntax(full_task_graph, parameters, graph_config): + """Generate a list of target tasks based on try syntax in + parameters['message'] and, for context, the full task graph. + + Based on gecko_taskgraph.target_tasks._try_option_syntax. Removed talos + and raptor references and use TryCCOptionSyntax. + """ + options = TryCCOptionSyntax(parameters, full_task_graph, graph_config) + target_tasks_labels = [ + t.label + for t in full_task_graph.tasks.values() + if options.task_matches(t) + and filter_by_uncommon_try_tasks(t.label) + and filter_unsupported_artifact_builds(t, parameters) + ] + + attributes = { + k: getattr(options, k) + for k in [ + "no_retry", + "tag", + ] + } + + for l in target_tasks_labels: + task = full_task_graph[l] + if "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + for l in target_tasks_labels: + task = full_task_graph[l] + # If the developer wants test jobs to be rebuilt N times we add that value here + if options.trigger_tests > 1 and "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + task.attributes.update(attributes) + + # Add notifications here as well + if options.notifications: + for task in full_task_graph: + owner = parameters.get("owner") + routes = task.task.setdefault("routes", []) + if options.notifications == "all": + routes.append(f"notify.email.{owner}.on-any") + elif options.notifications == "failure": + routes.append(f"notify.email.{owner}.on-failed") + routes.append(f"notify.email.{owner}.on-exception") + + return target_tasks_labels diff --git a/comm/taskcluster/comm_taskgraph/util/__init__.py b/comm/taskcluster/comm_taskgraph/util/__init__.py new file mode 100644 index 0000000000..71f5dc9cd0 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/__init__.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def strip_comm_prefix(relpath): + """ + Returns relpath with 'comm/' prefix removed. + :param string relpath: relative path + :return string: stripped path + """ + if relpath[:5] == "comm/": + return relpath[5:] + else: + return relpath diff --git a/comm/taskcluster/comm_taskgraph/util/docker.py b/comm/taskcluster/comm_taskgraph/util/docker.py new file mode 100644 index 0000000000..07971f3f28 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/docker.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from gecko_taskgraph.util import docker as utildocker + +from .. import COMM + +logger = logging.getLogger(__name__) + +COMM_IMAGE_DIR = os.path.join(COMM, "taskcluster", "docker") + + +def register(): + logger.info("Registering comm docker image definition path.") + utildocker.image_paths.register( + "comm/taskcluster/ci/docker-image/docker-image.yml", COMM_IMAGE_DIR + ) + + +register() diff --git a/comm/taskcluster/comm_taskgraph/util/hash.py b/comm/taskcluster/comm_taskgraph/util/hash.py new file mode 100644 index 0000000000..d372a0192a --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/hash.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib + +import taskgraph.util.path as util_path + +from gecko_taskgraph.util.hash import get_file_finder, hash_path + + +def split_patterns_list(patterns): + """ + Give a list of path patterns and return two lists. rv[0] corresponds to files from the + GECKO repository, rv[1] corresponds to COMM. + The pattern list for the COMM repository will have *not* the 'comm/' prefix stripped. + """ + return [ + [p for p in patterns if not p.startswith("comm/")], + [p for p in patterns if p.startswith("comm/")], + ] + + +def prefix_paths(prefix, paths): + """ + Prepend a prefix to a list of paths. + """ + return [util_path.join(prefix, p) for p in paths] + + +def process_found(found_gen, prefix=None): + """ + Transform the results from finder.find(pattern) into a list of files. + If prefix is given, prepend it to results. + """ + for filename, fileobj in found_gen: + if prefix: + yield util_path.join(prefix, filename) + else: + yield filename + + +def hash_paths_extended(base_path, patterns): + """ + Works like gecko_taskgraph.util.hash.hash_paths, except it is able to account for Thunderbird + source code being part of a separate repository. + Two file finders are created if necessary. + """ + gecko_patterns, comm_patterns = split_patterns_list(patterns) + gecko_finder = get_file_finder(base_path) + comm_finder = get_file_finder(util_path.join(base_path, "comm")) + + h = hashlib.sha256() + files = [] + for (patterns, finder, prefix) in [ + (gecko_patterns, gecko_finder, None), + (comm_patterns, comm_finder, "comm/"), + ]: + for pattern in patterns: + if prefix: + pattern = pattern.lstrip(prefix) + found = list(process_found(finder.find(pattern), prefix)) + if found: + files.extend(found) + else: + raise Exception("%s did not match anything" % pattern) + for path in sorted(files): + if path.endswith((".pyc", ".pyd", ".pyo")): + continue + h.update( + "{} {}\n".format( + hash_path(util_path.abspath(util_path.join(base_path, path))), + util_path.normsep(path), + ).encode("utf-8") + ) + return h.hexdigest() diff --git a/comm/taskcluster/docker/tb-atn/Dockerfile b/comm/taskcluster/docker/tb-atn/Dockerfile new file mode 100644 index 0000000000..dcf66f1f83 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Thunderbird Releng <tb-builds@thunderbird.net> + +VOLUME /builds/worker/workspace + +COPY atn_langpack.py /builds/worker/bin/atn_langpack.py +COPY make_venv.sh /builds/worker/bin/make_venv.sh +COPY runme.sh /builds/worker/bin/runme.sh +COPY requirements.txt /builds/worker/requirements.txt + +RUN /builds/worker/bin/make_venv.sh + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + diff --git a/comm/taskcluster/docker/tb-atn/atn_langpack.py b/comm/taskcluster/docker/tb-atn/atn_langpack.py new file mode 100644 index 0000000000..fc66c09b2f --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/atn_langpack.py @@ -0,0 +1,171 @@ +#!python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import datetime +import json +import logging +import os +import random +import string +import sys +from enum import Enum +from pathlib import Path +from pprint import pprint as pp +from typing import List, Literal, NamedTuple, Tuple, Union + +import jwt +import requests +from redo import retry + +logging.getLogger("requests").setLevel(logging.DEBUG) + +ATN_UPLOAD_URL = "https://addons.thunderbird.net/api/v3/addons/langpack-{langcode}@thunderbird.mozilla.org/versions/{version}/" +CHUNK_SIZE = 128 * 1024 + + +class ATNChannel(Enum): + LISTED = "listed" + UNLISTED = "unlisted" + + +Locales = List[str] +Version = str +ApiParam = str +EnvVars = NamedTuple( + "EnvVars", + [ + ("langpack_version", Version), + ("locales", Locales), + ("langpack_dir", Path), + ("langpack_channel", Literal[ATNChannel.LISTED, ATNChannel.UNLISTED]), + ("api_key", ApiParam), + ("api_secret", ApiParam), + ], +) +Result = Tuple[str, Union[object, None]] + + +def print_line(message): + msg_bytes = message.encode("utf-8") + written = 0 + while written < len(msg_bytes): + written += sys.stdout.buffer.write(msg_bytes[written:]) or 0 + sys.stdout.buffer.flush() + + +class ATNUploader: + def __init__(self, options: EnvVars): + self.api_key = options.api_key + self.api_secret = options.api_secret + self.langpack_dir = options.langpack_dir + self.langpack_version = options.langpack_version + self.langpack_channel = options.langpack_channel + self.locales = options.locales + + def mk_headers(self) -> dict: + now = datetime.datetime.utcnow() + payload = { + "iss": self.api_key, + "jti": "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(64) + ), + "exp": now + datetime.timedelta(seconds=60), + "iat": now, + } + headers = { + "Authorization": "JWT {0}".format( + jwt.encode(payload, self.api_secret, algorithm="HS256") + ) + } + return headers + + def upload_langpack(self, locale: str) -> Result: + langpack_path = self.langpack_dir / locale / "target.langpack.xpi" + headers = self.mk_headers() + langpack_fd = open(langpack_path, "rb") + file = {"upload": ("upload", langpack_fd)} + data = {"channel": self.langpack_channel} + + url = ATN_UPLOAD_URL.format(version=self.langpack_version, langcode=locale) + with requests.put(url, files=file, data=data, headers=headers, verify=False) as resp: + if not resp.ok: + print_line(f"Failed {locale}") + return resp.json() + else: + return resp.json() + + def upload_all_locales(self) -> Tuple[List[Result], List[Result]]: + failed = [] + success = [] + for locale in self.locales: + try: + rv = retry(self.upload_langpack, args=(locale,), attempts=3, sleeptime=10) + if "error" not in rv: + success.append((locale, rv)) + else: + failed.append((locale, rv)) + except requests.HTTPError as e: + print_line(e) + failed.append((locale, None)) + return success, failed + + +def get_secret(name: str) -> Tuple[ApiParam, ApiParam]: + secret = {} + if "MOZ_SCM_LEVEL" in os.environ: + level = os.environ.get("MOZ_SCM_LEVEL", "1") + taskcluster_url = os.environ.get("TASKCLUSTER_PROXY_URL") or os.environ.get( + "TASKCLUSTER_ROOT_URL", "" + ) + secrets_url = ( + f"{taskcluster_url}/secrets/v1/secret/project/comm/thunderbird/releng" + f"/build/level-{level}/{name}" + ) + res = requests.get(secrets_url) + res.raise_for_status() + secret = res.json() + elif "SECRET_FILE" in os.environ: # For local dev/debug + with open(os.environ["SECRET_FILE"]) as fp: + secret = json.load(fp)["secret"] + secret = secret.get("secret") + api_key = secret["api_key"] if "api_key" in secret else None + api_secret = secret["api_secret"] if "api_secret" in secret else None + if api_key is None or api_secret is None: + raise Exception(f"Unable to get secret. {secret.keys()}") + + return api_key, api_secret + + +def read_env_vars() -> EnvVars: + try: + langpack_version = os.environ["LANGPACK_VERSION"] + locales_json = os.environ["LOCALES"] + langpack_dir = Path(os.environ["MOZ_FETCHES_DIR"]).resolve() + langpack_channel = os.environ["ATN_CHANNEL"] + except KeyError: + raise Exception("Missing environment variable(s)") + + locales = json.loads(locales_json) + api_key, api_secret = get_secret("atn_langpack") + + return EnvVars( + langpack_version, locales, langpack_dir, ATNChannel(langpack_channel), api_key, api_secret + ) + + +def main(): + options = read_env_vars() + + atn_uploader = ATNUploader(options) + success, failed = atn_uploader.upload_all_locales() + + pp(success) + if failed: + pp(failed) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/docker/tb-atn/make_venv.sh b/comm/taskcluster/docker/tb-atn/make_venv.sh new file mode 100755 index 0000000000..1ae7b1f881 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/make_venv.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /builds/worker || exit 1 + +python3 -m venv --system-site-packages venv +source ./venv/bin/activate +python3 -m pip install -r /builds/worker/requirements.txt diff --git a/comm/taskcluster/docker/tb-atn/requirements.in b/comm/taskcluster/docker/tb-atn/requirements.in new file mode 100644 index 0000000000..d6bcfa8d38 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/requirements.in @@ -0,0 +1,3 @@ +PyJWT==2.7.0 +requests==2.30.0 +redo==2.0.4 diff --git a/comm/taskcluster/docker/tb-atn/requirements.txt b/comm/taskcluster/docker/tb-atn/requirements.txt new file mode 100644 index 0000000000..d223006e6c --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/requirements.txt @@ -0,0 +1,107 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements.txt - +# +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via requests +charset-normalizer==3.2.0 \ + --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \ + --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \ + --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \ + --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \ + --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \ + --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \ + --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \ + --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \ + --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \ + --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \ + --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \ + --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \ + --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \ + --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \ + --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \ + --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \ + --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \ + --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \ + --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \ + --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \ + --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \ + --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \ + --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \ + --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \ + --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \ + --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \ + --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \ + --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \ + --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \ + --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \ + --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \ + --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \ + --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \ + --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \ + --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \ + --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \ + --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \ + --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \ + --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \ + --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \ + --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \ + --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \ + --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \ + --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \ + --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \ + --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \ + --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \ + --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \ + --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \ + --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \ + --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \ + --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \ + --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \ + --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \ + --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \ + --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \ + --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \ + --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \ + --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \ + --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \ + --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \ + --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \ + --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \ + --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \ + --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \ + --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \ + --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \ + --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \ + --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \ + --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \ + --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \ + --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \ + --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \ + --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \ + --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa + # via requests +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 + # via requests +pyjwt==2.7.0 \ + --hash=sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1 \ + --hash=sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074 + # via -r - +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 \ + --hash=sha256:c76e4c23ab2f8840261736a851323cd98493710e7a9d36a1058535dca501f293 + # via -r - +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 + # via -r - +urllib3==2.0.4 \ + --hash=sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11 \ + --hash=sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4 + # via requests diff --git a/comm/taskcluster/docker/tb-atn/runme.sh b/comm/taskcluster/docker/tb-atn/runme.sh new file mode 100755 index 0000000000..bf504d3735 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/runme.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /builds/worker || exit 1 + +source venv/bin/activate + +exec python3 bin/atn_langpack.py diff --git a/comm/taskcluster/docker/tb-debian-mingw/Dockerfile b/comm/taskcluster/docker/tb-debian-mingw/Dockerfile new file mode 100644 index 0000000000..ab5a7dbc2c --- /dev/null +++ b/comm/taskcluster/docker/tb-debian-mingw/Dockerfile @@ -0,0 +1,19 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Rob Lemley <rob@thunderbird.net> +# Used by Thunderbird to build third party libraries for OTR messaging. + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +RUN apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + autoconf \ + automake \ + binutils-mingw-w64 \ + gcc-mingw-w64 \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libtool \ + mingw-w64-tools diff --git a/comm/taskcluster/docker/tb-flatpak/Dockerfile b/comm/taskcluster/docker/tb-flatpak/Dockerfile new file mode 100644 index 0000000000..0ec129c9f3 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/Dockerfile @@ -0,0 +1,18 @@ +FROM freedesktopsdk/flatpak:22.08-x86_64 +MAINTAINER tb-builds@thunderbird.net + +RUN mkdir /scripts/ +WORKDIR /scripts/ + +# Copy everything in the docker/tb-flatpak folder but the Dockerfile +COPY [^D]* /scripts/ + +# Set up Python virtual environment +RUN /scripts/make_venv.sh + +# Manually add close_range syscall to image +RUN ["gcc", "-Wall", "-shared", "-o", "/scripts/close_range.so", "/scripts/close_range.c"] +ENV LD_PRELOAD /scripts/close_range.so + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py b/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py new file mode 100644 index 0000000000..65ed76eeb8 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py @@ -0,0 +1,159 @@ +#!/usr/bin/python3 -u +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +""" +Build the Flatpak .desktop file. Needs to run in the Python virtualenv +due to dependencies. + +python3 /scripts/build_desktop_file.py -o "$WORKSPACE/org.mozilla.Thunderbird.desktop" \ + -t "/scripts/org.mozilla.Thunderbird.desktop.jinja2" \ + -l "$WORKSPACE/l10n-central" \ + -L "$WORKSPACE/shipped-locales" \ + -f "mail/branding/thunderbird/brand.ftl" \ + -f "mail/messenger/flatpak.ftl" +""" + +import argparse +import json +import os +import urllib.request +import zipfile +from pathlib import Path +from typing import List, Union + +import jinja2 +from fluent.runtime.fallback import FluentLocalization, FluentResourceLoader + +COMM_L10N_ZIP = "https://hg.mozilla.org/projects/comm-l10n/archive/{rev}.zip" +COMM_L10N_ZIP_PREFIX = "comm-l10n-{rev}" + + +class FluentTranslator: + """ + FluentTranslator is an enhanced FluentLocalization + """ + + def __init__(self, l10n_base: Path, locales: List[str], resource_ids: List[str]): + self._locales = locales + self._localizations = self._populate(l10n_base, resource_ids) + + @property + def locales(self): + return sorted([l for l in self._locales if l != "en-US"]) + + def _populate(self, l10n_path, resource_ids): + loader = FluentResourceLoader(str(l10n_path / "{locale}")) + + rv = {} + for locale in self._locales: + rv[locale] = FluentLocalization([locale], resource_ids, loader) + + return rv + + def get_message(self, locale, message_id) -> Union[str, None]: + rv = self._localizations[locale].format_value(message_id) + if rv == message_id: + return None + return rv + + +def get_multi_translate(l10n_strings: FluentTranslator): + def translate_multi(key: str, fluent_id: str): + for locale in l10n_strings.locales: + translated = l10n_strings.get_message(locale, fluent_id) + if translated is not None: + yield f"{key}[{locale}]={translated}" + + return translate_multi + + +def build_template( + output: Path, + template: Path, + l10n_base: Path, + locales: List[str], + fluent_resources: List[str], + is_beta: bool, +): + wmclass = "thunderbird" + if is_beta: + wmclass = wmclass + "-beta" + locales_plus = locales + ["en-US"] + l10n_strings = FluentTranslator(l10n_base.resolve(), locales_plus, fluent_resources) + + with open(template) as fp: + jinja_template = jinja2.Template(fp.read()) + + translate_multi = get_multi_translate(l10n_strings) + result = jinja_template.render( + strings=l10n_strings, translate=translate_multi, wmclass=wmclass + ) + + with open(output, "w") as fp: + fp.write(result) + + +def get_extract_members( + zip_file: zipfile.ZipFile, file_pats: List[str], prefix: str +) -> List[zipfile.ZipInfo]: + for m in zip_file.infolist(): + for pat in file_pats: + if m.filename.endswith(pat): + m.filename = os.path.relpath(m.filename, prefix) + print(f"Found {m.filename} in strings repo.") + yield m + + +def get_strings(l10n_base, rev, fluent_files): + url = COMM_L10N_ZIP.format(rev=rev) + temp_file, headers = urllib.request.urlretrieve(url) + with zipfile.ZipFile(temp_file, "r") as strings_zip: + to_extract = get_extract_members( + strings_zip, fluent_files, COMM_L10N_ZIP_PREFIX.format(rev=rev) + ) + + strings_zip.extractall(path=l10n_base, members=to_extract) + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("-o", dest="output", type=Path, required=True, help="Output file") + parser.add_argument( + "-t", dest="template", type=Path, required=True, help="Jinja2 template file" + ) + parser.add_argument( + "-l", dest="l10n_base", type=Path, required=True, help="l10n-central root path" + ) + parser.add_argument( + "-L", dest="locales_file", type=Path, required=True, help="List of supported locales" + ) + parser.add_argument( + "-f", dest="fluent_files", type=str, required=True, action="extend", nargs="+" + ) + parser.add_argument( + "--beta", + dest="is_beta", + action="store_true", + default=False, + help="Mark this build a beta version", + ) + + args = parser.parse_args() + + with open(args.locales_file) as fp: + locale_data = json.load(fp) + locales = [l for l in locale_data.keys() if l != "ja-JP-mac"] + comm_l10n_rev = locale_data.get("en-GB", {}).get("revision") + + get_strings(args.l10n_base, comm_l10n_rev, args.fluent_files) + + build_template( + args.output, args.template, args.l10n_base, locales, args.fluent_files, args.is_beta + ) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/docker/tb-flatpak/close_range.c b/comm/taskcluster/docker/tb-flatpak/close_range.c new file mode 100644 index 0000000000..d786e78e3b --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/close_range.c @@ -0,0 +1,12 @@ +/* + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +#include <errno.h> + +int close_range(unsigned int first, unsigned int last, unsigned int flags) { + errno = ENOSYS; + return -1; +} diff --git a/comm/taskcluster/docker/tb-flatpak/distribution.ini b/comm/taskcluster/docker/tb-flatpak/distribution.ini new file mode 100644 index 0000000000..d7793363a1 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/distribution.ini @@ -0,0 +1,13 @@ +[Global] +id=thunderbird-flatpak +version=1.0 +about=Mozilla Thunderbird Flatpak +about.en-US=Mozilla Thunderbird Flatpak en-US + +[Preferences] +intl.locale.requested="" +app.update.auto=false +app.update.enabled=false +app.update.autoInstallEnabled=false +mail.shell.checkDefaultClient=false +spellchecker.dictionary_path=/usr/share/hunspell diff --git a/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py b/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..b1eb745d7f --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +""" +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +""" + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt b/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt new file mode 100644 index 0000000000..4b9fde4a30 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt @@ -0,0 +1,2 @@ +fluent.runtime==0.4.0 +jinja2==3.1.2 diff --git a/comm/taskcluster/docker/tb-flatpak/launch_script.sh b/comm/taskcluster/docker/tb-flatpak/launch_script.sh new file mode 100755 index 0000000000..27875568b9 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/launch_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export TMPDIR="$XDG_RUNTIME_DIR/app/$FLATPAK_ID" +exec /app/lib/thunderbird/thunderbird --name org.mozilla.Thunderbird "$@" diff --git a/comm/taskcluster/docker/tb-flatpak/make_venv.sh b/comm/taskcluster/docker/tb-flatpak/make_venv.sh new file mode 100755 index 0000000000..6b807e64f4 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/make_venv.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /scripts || exit 1 + +python -m venv --system-site-packages venv +source ./venv/bin/activate +python -m pip install -r fluent_requirements.txt diff --git a/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in new file mode 100644 index 0000000000..5a103650ce --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in @@ -0,0 +1,83 @@ +<?xml version="1.0" encoding="UTF-8"?> +<component type="desktop-application"> + <id>org.mozilla.Thunderbird</id> + <launchable type="desktop-id">org.mozilla.Thunderbird.desktop</launchable> + <metadata_license>CC0-1.0</metadata_license> + <name>Thunderbird</name> + <summary>Thunderbird is a free and open source email, newsfeed, chat, and calendaring client</summary> + + <description> + <!-- From https://www.thunderbird.net/en-US/about/ --> + <p> + Thunderbird is a free and open source email, newsfeed, chat, and + calendaring client, that’s easy to set up and customize. One of the core + principles of Thunderbird is the use and promotion of open standards - + this focus is a rejection of our world of closed platforms and services + that can’t communicate with each other. We want our users to have freedom + and choice in how they communicate. + </p> + <p> + Thunderbird is an open source project, which means anyone can contribute + ideas, designs, code, and time helping fellow users. + </p> + </description> + + <releases> + <release version="$VERSION" date="$DATE"> + <url>$RELEASE_NOTES_URL</url> + </release> + </releases> + + <keywords> + <keyword>mozilla</keyword> + <keyword>mail</keyword> + <keyword>email</keyword> + <keyword>calendar</keyword> + </keywords> + + <categories> + <category>Calendar</category> + <category>Chat</category> + <category>ContactManagement</category> + <category>Email</category> + <category>Feed</category> + <category>InstantMessaging</category> + <category>IRCClient</category> + <category>Network</category> + <category>News</category> + <category>Office</category> + </categories> + + <provides> + <mediatype>message/rfc822</mediatype> + <mediatype>x-scheme-handler/mailto</mediatype> + <mediatype>text/calendar</mediatype> + <mediatype>text/vcard</mediatype> + <mediatype>text/x-vcard</mediatype> + <mediatype>x-scheme-handler/webcal</mediatype> + <mediatype>x-scheme-handler/webcals</mediatype> + <mediatype>x-scheme-handler/mid</mediatype> + </provides> + + <content_rating type="oars-1.1" /> + <url type="homepage">https://www.thunderbird.net/</url> + <url type="bugtracker">https://bugzilla.mozilla.org/</url> + <url type="faq">https://support.mozilla.org/kb/thunderbird-faq/</url> + <url type="help">https://support.mozilla.org/products/thunderbird/</url> + <url type="donation">https://give.thunderbird.net/</url> + <url type="translate">https://www.thunderbird.net/en-US/get-involved/#translation</url> + <url type="contact">https://www.thunderbird.net/contact/</url> + + <screenshots> + <screenshot type="default">https://raw.githubusercontent.com/thunderbird/flatpak-screenshots/main/image_1.png</screenshot> + <screenshot>https://raw.githubusercontent.com/thunderbird/flatpak-screenshots/main/image_2.png</screenshot> + </screenshots> + + <custom> + <value key="flathub::manifest">$MANIFEST_URL</value> + </custom> + + <project_group>Mozilla</project_group> + <project_license>MPL-2.0</project_license> + <developer_name>MZLA Technologies, part of the Mozilla Foundation</developer_name> +</component> diff --git a/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 new file mode 100644 index 0000000000..3623c55302 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 @@ -0,0 +1,33 @@ +[Desktop Entry] +Name={{ strings.get_message("en-US", "flatpak-desktop-name") }} +Comment={{ strings.get_message("en-US", "flatpak-desktop-comment") }} +{%- for line in translate("Comment", "flatpak-desktop-comment") %} +{{ line }} +{%- endfor %} +GenericName={{ strings.get_message("en-US", "flatpak-desktop-generic-name") }} +{%- for line in translate("GenericName", "flatpak-desktop-generic-name") %} +{{ line }} +{%- endfor %} +Exec=thunderbird %u +Terminal=false +Type=Application +Icon=org.mozilla.Thunderbird +Categories=Network;Email; +MimeType=message/rfc822;x-scheme-handler/mailto;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid; +StartupNotify=true +StartupWMClass={{ wmclass }} +Actions=ComposeMessage;OpenAddressBook; + +[Desktop Action ComposeMessage] +Name={{ strings.get_message("en-US", "flatpak-desktop-action-compose") }} +{%- for line in translate("Name", "flatpak-desktop-action-compose") %} +{{ line }} +{%- endfor %} +Exec=thunderbird -compose + +[Desktop Action OpenAddressBook] +Name={{ strings.get_message("en-US", "flatpak-desktop-action-addressbook") }} +{%- for line in translate("Name", "flatpak-desktop-action-addressbook") %} +{{ line }} +{%- endfor %} +Exec=thunderbird -addressbook diff --git a/comm/taskcluster/docker/tb-flatpak/runme.sh b/comm/taskcluster/docker/tb-flatpak/runme.sh new file mode 100755 index 0000000000..389929a124 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/runme.sh @@ -0,0 +1,198 @@ +#!/bin/bash +set -xe + +# Future products supporting Flatpaks will set this accordingly +: PRODUCT "${PRODUCT:=thunderbird}" + +# Required environment variables +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" +test "$FLATPAK_BRANCH" +test "$MANIFEST_URL" +test "$RELEASE_NOTES_URL" + +# Optional environment variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" + +# Populate remaining environment variables +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +TARGET_TAR_XZ_FULL_PATH="$ARTIFACTS_DIR/target.flatpak.tar.xz" +SOURCE_DEST="${WORKSPACE}/source" +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +FREEDESKTOP_VERSION="23.08" +FIREFOX_BASEAPP_CHANNEL="23.08" + +# Create alias for ideal curl command +CURL="curl --location --retry 10 --retry-delay 10" + +# Get current date +# +# This is used to populate the datetime in org.mozilla.Thunderbird.appdata.xml +DATE=$(date +%Y-%m-%d) +export DATE + +# Prepare directories +# +# This command is temporary, there's an upcoming fix in the upstream +# Docker image that we work on top of, from 'freedesktopsdk', that will +# make these two lines go away eventually. +mkdir -p /root /tmp /var/tmp +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +# Ensure a clean slate in the local Flatpak repo +rm -rf ~/.local/share/flatpak/ + +# Download en-US linux64 (English, 64-bit Linux) Thunderbird binary +$CURL -o "${WORKSPACE}/thunderbird.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/thunderbird-${VERSION}.tar.bz2" + +# Fetch list of Thunderbird locales +$CURL -o "${WORKSPACE}/l10n-changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n-changesets.json") + +# Fetch langpack extension for each locale +mkdir -p "$DISTRIBUTION_DIR" +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$DISTRIBUTION_DIR/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +# Download artifacts from dependencies and build the .desktop file. +( +source /scripts/venv/bin/activate +python3 /scripts/build_desktop_file.py -o "$WORKSPACE/org.mozilla.Thunderbird.desktop" \ + -t "/scripts/org.mozilla.Thunderbird.desktop.jinja2" \ + -l "$WORKSPACE/l10n-central" \ + -L "$WORKSPACE/l10n-changesets.json" \ + -f "mail/branding/thunderbird/brand.ftl" \ + -f "mail/messenger/flatpak.ftl" +) + +# Generate AppData XML from template, add various +envsubst < "$SCRIPT_DIRECTORY/org.mozilla.Thunderbird.appdata.xml.in" > "${WORKSPACE}/org.mozilla.Thunderbird.appdata.xml" +cp -v "$SCRIPT_DIRECTORY/distribution.ini" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/launch_script.sh" "$WORKSPACE" +cd "${WORKSPACE}" + +# Fetch and install Firefox base app (as user, not system-wide) +flatpak remote-add --user --if-not-exists --from flathub https://dl.flathub.org/repo/flathub.flatpakrepo +flatpak install --user -y flathub org.mozilla.firefox.BaseApp//${FIREFOX_BASEAPP_CHANNEL} --no-deps + +# Create build directory and add Firefox base app files +# +# This command is temporary, there's an upcoming fix in the upstream +# Docker image that we work on top of, from 'freedesktopsdk', that will +# make these two lines go away eventually. +mkdir -p build +cp -r ~/.local/share/flatpak/app/org.mozilla.firefox.BaseApp/current/active/files build/files + +# Create Flatpak build metadata file for Thunderbird +ARCH=$(flatpak --default-arch) +cat <<EOF > build/metadata +[Application] +name=org.mozilla.Thunderbird +runtime=org.freedesktop.Platform/${ARCH}/${FREEDESKTOP_VERSION} +sdk=org.freedesktop.Sdk/${ARCH}/${FREEDESKTOP_VERSION} +base=app/org.mozilla.firefox.BaseApp/${ARCH}/${FIREFOX_BASEAPP_CHANNEL} +[Extension org.mozilla.Thunderbird.Locale] +directory=share/runtime/langpack +autodelete=true +locale-subset=true +EOF + +# Create Flatpak build metadata file for locales +cat <<EOF > build/metadata.locale +[Runtime] +name=org.mozilla.Thunderbird.Locale + +[ExtensionOf] +ref=app/org.mozilla.Thunderbird/${ARCH}/${FLATPAK_BRANCH} +EOF + +# Install Thunderbird files into appdir +appdir=build/files +install -d "${appdir}/lib/" +(cd "${appdir}/lib/" && tar jxf "${WORKSPACE}/thunderbird.tar.bz2") +install -D -m644 -t "${appdir}/share/appdata" org.mozilla.Thunderbird.appdata.xml +install -D -m644 -t "${appdir}/share/applications" org.mozilla.Thunderbird.desktop +for size in 16 32 48 64 128; do + install -D -m644 "${appdir}/lib/thunderbird/chrome/icons/default/default${size}.png" "${appdir}/share/icons/hicolor/${size}x${size}/apps/org.mozilla.Thunderbird.png" +done + +# Generate AppStream metadata and add screenshots from Flathub +appstream-compose --prefix="${appdir}" --origin=flatpak --basename=org.mozilla.Thunderbird org.mozilla.Thunderbird +appstream-util mirror-screenshots "${appdir}"/share/app-info/xmls/org.mozilla.Thunderbird.xml.gz "https://dl.flathub.org/repo/screenshots/org.mozilla.Thunderbird-${FLATPAK_BRANCH}" build/screenshots "build/screenshots/org.mozilla.Thunderbird-${FLATPAK_BRANCH}" + +# Install locales, distribution, and launch_script.sh into appdir +# +# We must install each locale individually, since we're symlinking +# each one. +# +# We put the langpacks in /app/share/locale/$LANG_CODE and symlink that +# directory to where Thunderbird looks them up; this way only the subset +# of locales configured on the user's system are downloaded, instead of +# all locales. +mkdir -p "${appdir}/lib/thunderbird/distribution/extensions" +for locale in $locales; do + install -D -m644 -t "${appdir}/share/runtime/langpack/${locale%%-*}/" "${DISTRIBUTION_DIR}/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" + ln -sf "/app/share/runtime/langpack/${locale%%-*}/langpack-${locale}@thunderbird.mozilla.org.xpi" "${appdir}/lib/thunderbird/distribution/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" +done +install -D -m644 -t "${appdir}/lib/thunderbird/distribution" distribution.ini +install -D -m755 launch_script.sh "${appdir}/bin/thunderbird" + +# Build Flatpak +# +# We use features=devel to enable ptrace, which we need for the crash +# reporter. The application is still confined in a pid namespace, so +# that won't let us escape the flatpak sandbox. See bug 1653852. +# +# We use own-name to ensure Thunderbird has access to DBus, as app ID +# (org.mozilla.Thunderbird) does not match bus names +# (org.mozilla.thunderbird, lowercase "t"). The app ID may be updated +# in the future to match the default bus names. +flatpak build-finish build \ + --allow=devel \ + --share=ipc \ + --share=network \ + --socket=pulseaudio \ + --socket=wayland \ + --socket=x11 \ + --socket=pcsc \ + --socket=cups \ + --require-version=0.10.3 \ + --persist=.thunderbird \ + --filesystem=xdg-download:rw \ + --filesystem=~/.gnupg \ + --filesystem=xdg-run/gnupg:ro \ + --filesystem=xdg-run/speech-dispatcher:ro \ + --filesystem=/run/.heim_org.h5l.kcm-socket \ + --device=dri \ + --own-name="org.mozilla.thunderbird.*" \ + --own-name="org.mozilla.thunderbird_beta.*" \ + --talk-name="org.gtk.vfs.*" \ + --talk-name=org.a11y.Bus \ + --system-talk-name=org.freedesktop.NetworkManager \ + --command=thunderbird + +# Export Flatpak build into repo +flatpak build-export --disable-sandbox --no-update-summary --exclude='/share/runtime/langpack/*/*' repo build "$FLATPAK_BRANCH" +flatpak build-export --disable-sandbox --no-update-summary --metadata=metadata.locale --files=files/share/runtime/langpack repo build "$FLATPAK_BRANCH" + +# Commit screenshots to repo +ostree commit --repo=repo --canonical-permissions --branch=screenshots/x86_64 build/screenshots +flatpak build-update-repo --generate-static-deltas repo + +# Package Flatpak repo as tar +tar cvfJ flatpak.tar.xz repo +mv -- flatpak.tar.xz "$TARGET_TAR_XZ_FULL_PATH" + +# Build Flatpak bundle (.flatpak) from repo +flatpak build-bundle "$WORKSPACE"/repo org.mozilla.Thunderbird.flatpak org.mozilla.Thunderbird "$FLATPAK_BRANCH" + +# Move bundle to artifacts +mv org.mozilla.Thunderbird.flatpak "$ARTIFACTS_DIR/" diff --git a/comm/taskcluster/docs/attributes.rst b/comm/taskcluster/docs/attributes.rst new file mode 100644 index 0000000000..cbe2dd556d --- /dev/null +++ b/comm/taskcluster/docs/attributes.rst @@ -0,0 +1,5 @@ +=============== +Task Attributes +=============== + +Additional task attributes for Thunderbird's taskgraph. diff --git a/comm/taskcluster/docs/cron.rst b/comm/taskcluster/docs/cron.rst new file mode 100644 index 0000000000..cd8ee94f08 --- /dev/null +++ b/comm/taskcluster/docs/cron.rst @@ -0,0 +1,56 @@ +Periodic Taskgraphs +=================== + +The cron functionality allows in-tree scheduling of task graphs that run +periodically, instead of on a push. + + +Cron.yml +-------- + +In the root of the Comm directory, you will find `.cron.yml`. This defines +the periodic tasks ("cron jobs") that run for Thunderbird. + +See `the Firefox CI cron documentation <https://firefox-source-docs.mozilla.org/taskcluster/cron.html>`_ +for a description of `.cron.yml`. + + +Disabling Cron Jobs +------------------- + +Sometimes due to build bustage, it's desirable to disable the automatic +Thunderbird Daily builds. The best way to do this is to change `.cron.yml` +so that the jobs are never scheduled. This is done by changing `when` to an +empty list. + +The Daily build is started by the `nightly-desktop` job. Additionally, there +is a `searchfox-index` job that should be disabled as well when disabling +Dailies. + +For both of these, comment out the `when` line that sets a time and uncomment +the next line which sets an empty list. + +.. code-block:: yaml + :linenos: + :emphasize-lines: 8,9 + + - name: nightly-desktop + job: + type: decision-task + treeherder-symbol: Nd + target-tasks-method: nightly_desktop + run-on-projects: + - comm-central + when: [{hour: 11, minute: 0}] + # when: [] + + +L10n-bump Cron Job +------------------ + +`l10n-bump` runs daily on comm-beta. For RC week (week 4) it needs to be +disabled by pushing a change directly to `comm-beta` setting ``when`` to +an empty list as above. + +**Do not make the change on comm-central first.** `l10n-bump` will be +re-enabled by merge day activities automatically. diff --git a/comm/taskcluster/docs/index.rst b/comm/taskcluster/docs/index.rst new file mode 100644 index 0000000000..3229405387 --- /dev/null +++ b/comm/taskcluster/docs/index.rst @@ -0,0 +1,22 @@ +.. taskcluster_index: + +TaskCluster Task-Graph Generation +================================= + +The ``comm/taskcluster`` directory contains support for defining the graph of tasks +that must be executed to build and test Thunderbird. + +As Thunderbird is built on top of Firefox's source, the +`Firefox Taskgraph documentation <https://firefox-source-docs.mozilla.org/taskcluster/index.html>`_ +is an invaluable resource. + +The documentation here describes Thunderbird specifics. + +.. toctree:: + + kinds + attributes + parameters + loading + transforms + cron diff --git a/comm/taskcluster/docs/kinds.rst b/comm/taskcluster/docs/kinds.rst new file mode 100644 index 0000000000..85618c43da --- /dev/null +++ b/comm/taskcluster/docs/kinds.rst @@ -0,0 +1,65 @@ +Task Kinds +========== + +This section lists and documents the additional task kinds that are specific +to Thunderbird and are implemented in it's source tree. + +shippable-l10n-pre +------------------ +Prepares a build artifact containing the translated strings from all locales. +The artifact is consumed by `shippable-l10n` to produce the localized +Thunderbird builds. + +Using +..... + +- kind-dependencies: + Must include `build` +- transforms: + Must include `comm_taskgraph.transforms.l10n_pre:transforms` +- only-for-attributes: + Must include `shippable` +- only-for-build-platforms: + This is set to `linux64-shippable/opt` so that it only runs for that + platform. All platforms will consume the build artifact from + `linux64-shippable/opt`. (It's just string data; nothing platform-specific + in there.) + +Parameters +.......... + +There are some task parameters specific to this job kind. + +- locale-list: + Points to either `shipped-locales` or `all-locales`. This file is used to + select the locales that are included in the build artifact. +- comm-locales-file: + This file contains the revision of the `comm-l10n` monorepo to checkout. +- browser-locales-file: + This file contains the revisions of the `l10n-central` repositories to checkout. + This is for toolkit and other strings used from mozilla-central. + +Other notes +........... + +The mozharness script reads its configuration from `thunderbird_split_l10n.py`. +In that file, `hg_l10n_base` refers to the `l10n-central` repository root. +This is used with `browser-locales-file` to get the strings from toolkit and +devtools that are needed. + +Also in the mozharness config file is `hg_comm_l10n_repo`, set to the URL of +the `comm-l10n` monorepo. + +The mozharness script will clone the necessary repositories from `l10n-central`, +and `comm-l10n`, merge them, and create a tar file. + + +shippable-l10n-pre-signing +-------------------------- + +Signing job for shippable-l10n-pre artifacts + +beetmover-strings-source +------------------------ + +Upload strings source files to FTP. diff --git a/comm/taskcluster/docs/loading.rst b/comm/taskcluster/docs/loading.rst new file mode 100644 index 0000000000..3c9c5b1468 --- /dev/null +++ b/comm/taskcluster/docs/loading.rst @@ -0,0 +1,34 @@ +Loading +======= + +Overview +-------- + +See the Firefox loading documentation. In addition to those loaders, in +``comm_taskgraph.loader`` there are loaders for Thunderbird. + + +comm_taskgraph.loader +--------------------- + +reference +......... + +Loads selected jobs from a different taskgraph hierarchy. + +The ``reference`` loader is used to import kinds from the Firefox Taskcluster +base path. + +``packages``, and ``fetch`` use this loader. + +merge +..... + +Loads jobs for a kind from two Taskcluster base paths. The results are "merged" +together into a single kind, allowing for using the Firefox defined jobs as +a base and adding additional jobs. + +First jobs are imported from Firefox's Taskcluster base path using the reference +loader. Then jobs are read from the kind directory using ``jobs-from``. + +``docker-image`` and ``toolchain`` use this loader. diff --git a/comm/taskcluster/docs/parameters.rst b/comm/taskcluster/docs/parameters.rst new file mode 100644 index 0000000000..fa5ac309aa --- /dev/null +++ b/comm/taskcluster/docs/parameters.rst @@ -0,0 +1,38 @@ +========== +Parameters +========== + +Overview +-------- + +See the gecko_taskgraph parameters documentation. + +Comm Push Information +--------------------- + +These parameters correspond to the repository and revision of the comm-central +repository to checkout. All parameters are required. + +``comm_base_repository`` + The repository from which to do an initial clone, utilizing any available + caching. In practice this is always set to ``https://hg.mozilla.org/comm-central``. + +``comm_base_rev`` + The previous revision before ``comm_head_rev`` got merged into. + +``comm_base_ref`` + Reference where ``comm_head_rev`` got merged into. It is usually a branch or a tag. + +``comm_head_repository`` + The repository containing the changeset to be built. This may differ from + ``comm_base_repository``. + +``comm_head_rev`` + The revision to check out; this can be a short revision string. + +``comm_head_ref`` + This is the same as ``head_rev``. + +``comm_src_path`` + This will effectively always be "comm/". It's used in `comm_taskgraph.files_changed.get_files_changed_extended` + to handle multiple VCS repositories without hardcoding parameters. diff --git a/comm/taskcluster/docs/transforms/index.rst b/comm/taskcluster/docs/transforms/index.rst new file mode 100644 index 0000000000..877c04d2e2 --- /dev/null +++ b/comm/taskcluster/docs/transforms/index.rst @@ -0,0 +1,23 @@ +Transforms +========== + +Overview +-------- + +Most transforms come from Firefox, and it's assumed that the reader is +familiar with those already. + +The transforms here are mostly used to work around assumptions made in +Firefox transforms. The general idea is to keep Thunderbird-specific exceptions +to those assumptions in this tree. + +Loaders +------- + +See :doc:`loading`. + + +Transforms +---------- + +TODO: Document the transforms found in comm_taskgraph.transforms diff --git a/comm/taskcluster/docs/transforms/job.rst b/comm/taskcluster/docs/transforms/job.rst new file mode 100644 index 0000000000..b626d20e78 --- /dev/null +++ b/comm/taskcluster/docs/transforms/job.rst @@ -0,0 +1,10 @@ +Job Transforms +============== + +Run-Using +--------- + +In order to build toolchains specific to Thunderbird, there is a ``run-using`` +implementation in `comm_taskgraph.transforms.job.toolchain`: + +* ``comm-toolchain-script`` diff --git a/comm/taskcluster/moz.build b/comm/taskcluster/moz.build new file mode 100644 index 0000000000..26bf2ca15f --- /dev/null +++ b/comm/taskcluster/moz.build @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Thunderbird", "Build Config") + +PYTHON_UNITTEST_MANIFESTS += [ + "comm_taskgraph/test/python.ini", +] diff --git a/comm/taskcluster/scripts/are-we-esmified-yet.py b/comm/taskcluster/scripts/are-we-esmified-yet.py new file mode 100644 index 0000000000..20eaf108fd --- /dev/null +++ b/comm/taskcluster/scripts/are-we-esmified-yet.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import pathlib +import subprocess +import sys + +TBPL_FAILURE = 2 + +excluded_prefix = [ + "suite/", +] +EXCLUSION_FILES = [ + os.path.join("tools", "lint", "ThirdPartyPaths.txt"), +] + + +if not (pathlib.Path(".hg").exists() and pathlib.Path("mail/moz.configure").exists()): + print( + "This script needs to be run inside mozilla-central + comm-central " + "checkout of mercurial. " + ) + sys.exit(TBPL_FAILURE) + + +def load_exclusion_files(): + for path in EXCLUSION_FILES: + with open(path, "r") as f: + for line in f: + excluded_prefix.append(line.strip()) + + +def is_excluded(path): + """Returns true if the JSM file shouldn't be converted to ESM.""" + path_str = str(path) + for prefix in excluded_prefix: + if path_str.startswith(prefix): + return True + + return False + + +def new_files_struct(): + return { + "jsm": [], + "esm": [], + "subdir": {}, + } + + +def put_file(files, kind, path): + """Put a path into files tree structure.""" + + if is_excluded(path): + return + + name = path.name + + current_files = files + for part in path.parent.parts: + if part not in current_files["subdir"]: + current_files["subdir"][part] = new_files_struct() + current_files = current_files["subdir"][part] + + current_files[kind].append(name) + + +def run(cmd): + """Run command and return output as lines, excluding empty line.""" + lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode() + return filter(lambda x: x != "", lines.split("\n")) + + +def collect_jsm(files): + """Collect JSM files.""" + kind = "jsm" + + # jsm files + cmd = ["hg", "files", "set:glob:**/*.jsm"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + # js files with EXPORTED_SYMBOLS + cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + +def collect_esm(files): + """Collect system ESM files.""" + kind = "esm" + + # sys.mjs files + cmd = ["hg", "files", "set:glob:**/*.sys.mjs"] + + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + +def to_stat(files): + """Convert files tree into status tree.""" + jsm = len(files["jsm"]) + esm = len(files["esm"]) + subdir = {} + + for key, sub_files in files["subdir"].items(): + sub_stat = to_stat(sub_files) + + subdir[key] = sub_stat + jsm += sub_stat["jsm"] + esm += sub_stat["esm"] + + stat = { + "jsm": jsm, + "esm": esm, + } + if len(subdir): + stat["subdir"] = subdir + + return stat + + +def main(): + cmd = ["hg", "parent", "--template", "{node}"] + commit_hash = list(run(cmd))[0] + + cmd = ["hg", "parent", "--template", "{date|shortdate}"] + date = list(run(cmd))[0] + + files = new_files_struct() + collect_jsm(files) + collect_esm(files) + + stat = to_stat(files) + stat["hash"] = commit_hash + stat["date"] = date + + print(json.dumps(stat, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/scripts/build-l10n-pre.sh b/comm/taskcluster/scripts/build-l10n-pre.sh new file mode 100755 index 0000000000..7482dc7c5a --- /dev/null +++ b/comm/taskcluster/scripts/build-l10n-pre.sh @@ -0,0 +1,90 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for performing fx desktop l10n repacks via mozharness. +# Based on ./build-l10n.sh +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} +: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-l10n-pre.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export TINDERBOX_OUTPUT=1 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi + +# set up mozharness configuration, via command line, env, etc. + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +export MOZ_OBJDIR + +config_path_cmds="" +for path in ${MOZHARNESS_CONFIG_PATHS}; do + config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}" +done + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /builds/worker + +$GECKO_PATH/mach python -- \ + $GECKO_PATH/${MOZHARNESS_SCRIPT} \ + ${config_path_cmds} \ + ${config_cmds} \ + $actions \ + $options \ + --log-level=debug \ + --work-dir=$WORKSPACE \ diff --git a/comm/taskcluster/scripts/build-libotr.sh b/comm/taskcluster/scripts/build-libotr.sh new file mode 100755 index 0000000000..60df92c531 --- /dev/null +++ b/comm/taskcluster/scripts/build-libotr.sh @@ -0,0 +1,360 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -x -eE -v + +_TARGET_OS="$1" + +# Environment variables that are set by Taskcluster. +GECKO_PATH=${GECKO_PATH:-"/builds/worker/workspace/build/src"} +MOZ_FETCHES_DIR=${MOZ_FETCHES_DIR:-"/builds/worker/fetches"} +UPLOAD_DIR=${UPLOAD_DIR:-"/builds/worker/artifacts"} +WORKSPACE=${WORKSPACE:-"${HOME}/workspace"} +MACOS_SDK_DIR=${MACOS_SDK_DIR:-"MacOSX11.3.sdk"} +MACOS_TARGET_SDK=${MACOS_TARGET_SDK:-"10.12"} + + +# Set $DEVEL_TESTING during script development on a local machine +if [[ -n ${DEVEL_TESTING} ]]; then + rm -rf "${UPLOAD_DIR}" "${WORKSPACE}" + mkdir "${UPLOAD_DIR}" "${WORKSPACE}" +fi + +cd "$WORKSPACE" +if [[ ! -d build ]]; then + mkdir build +fi +for _d in build/libgpg-error build/libgcrypt build/libotr build/build_prefix; do + if [[ -e "${_d}" ]]; then + rm -rf "${_d}" + fi +done +BUILD="${WORKSPACE}/build" + +COMPRESS_EXT=xz + +_INSTDIR="build_prefix" +_PREFIX="${BUILD}/${_INSTDIR}" + +_ARTIFACT_STAGEDIR="${BUILD}/libotr_stage" + +THIRD_PARTY_SRC="${GECKO_PATH}/comm/third_party" + +GPG_ERROR_SRC="${BUILD}/libgpg-error" +GCRYPT_SRC="${BUILD}/libgcrypt" +OTR_SRC="${BUILD}/libotr" + +# Set environment variables needed for all dependencies +_BASE_CONFIGURE=(--build=x86_64-pc-linux --prefix="${_PREFIX}" --disable-silent-rules) + +_OS_CONFIGURE_FLAGS=() # May be overridden per target OS + + +function clang_cfg() { + # autotools and friends seem to work better with Clang if the compiler + # is named <target>-clang. This applies to macOS only. It does not seem + # necessary when building for Linux. + local _i _clang_cfg_dir _clang_dir + + _clang_cfg_dir="${THIRD_PARTY_SRC}/clang" + _clang_dir="${MOZ_FETCHES_DIR}/clang/bin" + + cp -a "${_clang_cfg_dir}"/*.cfg "${_clang_dir}" + for _i in x86_64-apple-darwin aarch64-apple-darwin aarch64-linux-gnu i686-linux-gnu; do + ln -s clang "${_clang_dir}/${_i}-clang" + done + return 0 +} + +function copy_sources() { + # The checkout directory should be treated readonly + local _pkg + cd "${BUILD}" + for _pkg in libgpg-error libgcrypt libotr; do + cp -a "${THIRD_PARTY_SRC}/${_pkg}" . + done +} + +function build_libgpg-error() { + echo "Building libgpg-error" + cd "${GPG_ERROR_SRC}" + + ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \ + --disable-tests --disable-doc --with-pic + + # Hack... *sigh* + if [[ "${_TARGET_OS}" == "linux-aarch64" ]]; then + cp src/syscfg/lock-obj-pub.aarch64-unknown-linux-gnu.h src/lock-obj-pub.native.h + fi + + make "${_MAKE_FLAGS}" -C src code-to-errno.h + make "${_MAKE_FLAGS}" -C src code-from-errno.h + make "${_MAKE_FLAGS}" -C src gpg-error.h + make "${_MAKE_FLAGS}" -C src libgpg-error.la + + make -C src install-nodist_includeHEADERS install-pkgconfigDATA \ + install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES + return $? +} + +function build_libgcrypt() { + echo "Building libgcrypt" + cd "${GCRYPT_SRC}" + ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \ + --disable-doc --with-pic "${_GCRYPT_CONF_FLAGS}" \ + --with-libgpg-error-prefix="${_PREFIX}" + + make "${_MAKE_FLAGS}" -C cipher libcipher.la + make "${_MAKE_FLAGS}" -C random librandom.la + make "${_MAKE_FLAGS}" -C mpi libmpi.la + make "${_MAKE_FLAGS}" -C compat libcompat.la + + make "${_MAKE_FLAGS}" -C src libgcrypt.la + + make -C src install-nodist_includeHEADERS \ + install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES + return $? +} + +function build_libotr() { + local _f + + echo "Building libotr" + cd "${OTR_SRC}" + + aclocal -I "${_PREFIX}/share/aclocal" + autoconf + automake + + ./configure "${_CONFIGURE_FLAGS[@]}" --enable-shared --with-pic \ + --with-libgcrypt-prefix="${_PREFIX}" + + # libtool archive (*.la) files are the devil's work + rm -f "${_PREFIX}"/lib/*.la + sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool + sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool + + make "${_MAKE_FLAGS}" -C src + + case "${_TARGET_OS}" in + win*) + cd src + "${CC}" -static-libgcc -s -shared -Wl,-no-undefined "${LDFLAGS[@]}" -o libotr.dll \ + ./*.o \ + -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + -L"${_LIBDIR}" -lws2_32 -lssp + cp libotr.dll "${_PREFIX}/bin" + ;; + linux*) + cd src + "${CC}" -shared "${LDFLAGS[@]}" -Wl,-soname -Wl,libotr.so \ + .libs/*.o \ + -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + --sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}" \ + -Wl,-soname -Wl,libotr.so -o libotr.so + cp libotr.so "${_PREFIX}/lib" + ;; + macos*) + cd src + "${CC}" -dynamiclib -Wl,-flat_namespace -Wl,-undefined -Wl,suppress -o libotr.dylib \ + .libs/*.o \ + "-L${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + -isysroot "${MACOS_SDK_DIR}" \ + -install_name "@executable_path/libotr.dylib" \ + -compatibility_version 7 -current_version 7.1 -Wl,-single_module + cp libotr.dylib "${_PREFIX}/lib" + esac + + return $? +} + +function package_libotr_artifact() { + local _f + + cd "${BUILD}" + rm -rf "${_ARTIFACT_STAGEDIR}" + + mkdir "${_ARTIFACT_STAGEDIR}" + + for _f in ${_TARGET_LIBS}; do + install "${_INSTDIR}/${_f}" "${_ARTIFACT_STAGEDIR}" + done + case "${_TARGET_OS}" in + win*) + install "${_LIBDIR}/libssp-0.dll" "${_ARTIFACT_STAGEDIR}" + ;; + esac + + rm -rf "${UPLOAD_DIR}" && mkdir -p "${UPLOAD_DIR}" + TARFILE="${UPLOAD_DIR}/libotr.tar.${COMPRESS_EXT}" + tar -acf "${TARFILE}" -C "${_ARTIFACT_STAGEDIR}" . + + return 0 +} + +# variables specific to an arch, but apply to all dependencies +case "${_TARGET_OS}" in + win32) + export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH" + export _TARGET_TRIPLE="i686-w64-mingw32" + export CC="${_TARGET_TRIPLE}-gcc" + _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32" + export LDFLAGS="-L${_LIBDIR}" + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _CONF_STATIC=(--enable-static --enable-shared) + + _TARGET_LIBS="bin/libotr.dll" + ;; + win64) + export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH" + export _TARGET_TRIPLE="x86_64-w64-mingw32" + export CC="${_TARGET_TRIPLE}-gcc" + _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32" + export LDFLAGS="-L${_LIBDIR}" + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _CONF_STATIC=(--enable-static --enable-shared) + + _TARGET_LIBS="bin/libotr.dll" + ;; + macosx64) + for _t in cctools/bin clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + export _TARGET_TRIPLE="x86_64-apple-darwin" + export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}" + export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks" + export CROSS_SYSROOT="${MACOS_SDK_DIR}" + + export CC="${_TARGET_TRIPLE}-clang" + export LD="${_TARGET_TRIPLE}-ld" + export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}" + export LDFLAGS="-isysroot ${CROSS_SYSROOT}" + export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil" + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _GCRYPT_CONF_FLAGS="--disable-asm" + _CONF_STATIC=(--enable-static --disable-shared) + + _TARGET_LIBS="lib/libotr.dylib" + ;; + macosx64-aarch64) + for _t in cctools/bin clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + export _TARGET_TRIPLE="aarch64-apple-darwin" + export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}" + export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks" + export CROSS_SYSROOT="${MACOS_SDK_DIR}" + + export CC="${_TARGET_TRIPLE}-clang" + export LD="${_TARGET_TRIPLE}-ld" + export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}" + export LDFLAGS="-isysroot ${CROSS_SYSROOT}" + export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil" + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _GCRYPT_CONF_FLAGS="--disable-asm" + _CONF_STATIC=(--enable-static --disable-shared) + + _TARGET_LIBS="lib/libotr.dylib" + ;; + linux32) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-i686-linux-gnu" + export _TARGET_TRIPLE="i686-pc-linux" + export CC="i686-linux-gnu-clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CCASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="--target=${_TARGET_TRIPLE} -m32 -march=pentium-m -msse -msse2 -mfpmath=sse -fuse-ld=lld" + + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + linux64) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-x86_64-linux-gnu" + export _TARGET_TRIPLE="x86_64-pc-linux" + export CC="clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="-fuse-ld=lld" + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + linux-aarch64) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-aarch64-linux-gnu" + export _TARGET_TRIPLE="aarch64-pc-linux" + export CC="aarch64-linux-gnu-clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="-fuse-ld=lld" + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export OBJDUMP=llvm-objdump + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + *) + echo "Invalid target platform: ${_TARGET_OS}" + exit 1 + ;; +esac + +_CONFIGURE_FLAGS=("${_BASE_CONFIGURE[@]}" "${_OS_CONFIGURE_FLAGS[@]}") +_MAKE_FLAGS="-j$(nproc)" + +# Basic dependency structure. +# Build block, followed by packaging block. +# Each step in a block depends on the previous completing successfully. +# The packaging block depends on the build block's success. +{ + copy_sources && + clang_cfg && + build_libgpg-error && + build_libgcrypt && + build_libotr +} && { + package_libotr_artifact +} && exit 0 + +# Ideally, the "exit 0" above ran after the packaging block ran successfully. +# In case it didn't, error out here so CI catches it. +exit 1 diff --git a/comm/taskcluster/scripts/build-source-docs.sh b/comm/taskcluster/scripts/build-source-docs.sh new file mode 100755 index 0000000000..e8b104b448 --- /dev/null +++ b/comm/taskcluster/scripts/build-source-docs.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -xe + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_PROXY_URL/api/queue/v1" + +# Get RTD secret location from task definition +if [ -n "${TASK_ID}" ]; then + curl --location --retry 10 --retry-delay 10 -o /builds/worker/task.json "$queue_base/task/$TASK_ID" + RTD_SECRET=$(jq -r '.scopes[] | select(contains ("rtd-webhook"))' /builds/worker/task.json | awk -F: '{print $3}') +fi + +# Get the secret value from the secrets service +if [ -n "${RTD_SECRET}" ] && getent hosts taskcluster; then + set +x # Don't echo these + secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${RTD_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${RTD_TOKEN}" ]; then # Allow for local testing. + TOKEN="${RTD_TOKEN}" +fi + +if [ -n "${TOKEN}" ]; then + curl \ + -X POST \ + -d "branches=latest" \ + -d "token=${TOKEN}" \ + https://readthedocs.com/api/v2/webhook/thunderbird-thunderbird-source-docs/9778/ +fi + diff --git a/comm/taskcluster/scripts/desktop_comm_l10n.py b/comm/taskcluster/scripts/desktop_comm_l10n.py new file mode 100755 index 0000000000..ae395e50d1 --- /dev/null +++ b/comm/taskcluster/scripts/desktop_comm_l10n.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import sys + +GECKO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +MOZHARNESS = os.path.join(GECKO_PATH, "testing/mozharness") +COMM_PYTHON_L10N = os.path.join(GECKO_PATH, "comm/python/l10n") +sys.path.insert(1, MOZHARNESS) +sys.path.insert(1, COMM_PYTHON_L10N) + +from zstandard import ZstdCompressor + +from mozharness.base.script import BaseScript +from mozharness.base.vcs.vcsbase import VCSMixin +from mozharness.mozilla.automation import AutomationMixin +from mozharness.mozilla.l10n.locales import LocalesMixin +from mozpack.archive import create_tar_from_files +from mozpack.copier import FileRegistry +from mozpack.files import FileFinder +from tbxchannel.l10n_merge import COMM_STRINGS_PATTERNS, GECKO_STRINGS_PATTERNS + + +class CommMultiLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): + config_options = [ + [ + [ + "--locale-list", + ], + { + "action": "store", + "dest": "locale_list", + "type": "string", + "help": "File with locales to include. Either all-locales or shipped-locales", + }, + ], + [ + [ + "--comm-locales-file", + ], + { + "action": "store", + "dest": "hg_comm_locales_file", + "type": "string", + "help": "File with HG revision of comm-l10n monorepo to use", + }, + ], + [ + [ + "--browser-locales-file", + ], + { + "action": "store", + "dest": "locales_file", + "type": "string", + "help": "File with HG revisions of l10n-central repositories", + }, + ], + ] + + def __init__(self, require_config_file=False): + buildscript_kwargs = { + "all_actions": [ + "clone-gecko-locales", + "clone-monorepo", + "merge-repos", + "pack-merged", + "gen-changesets", + ], + "config": { + "ignore_locales": ["en-US"], + "log_name": "multi_locale", + "hg_merged_dir": "l10n_merged", + "objdir": "obj-build", + "upload_file": "strings_all.tar.zst", + "changesets_file": "l10n-changesets.json", + }, + } + + LocalesMixin.__init__(self) + BaseScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + **buildscript_kwargs, + ) + self.upload_env = None + self.file_registry = None + self.comm_l10n_revision = None + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(CommMultiLocale, self).query_abs_dirs() + c = self.config + dirs = {} + dirs["abs_checkout_dir"] = os.path.abspath(os.path.join(abs_dirs["abs_src_dir"], "..")) + dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"]) + # Needs to match abs_dirs["abs_l10n_dir"] set in mozharness.mozilla.l10n.locales + dirs["abs_l10n_central_dir"] = os.path.abspath( + os.path.join(dirs["abs_checkout_dir"], "l10n-central") + ) + dirs["abs_comm_l10n_dir"] = os.path.abspath( + os.path.join(dirs["abs_checkout_dir"], "comm-l10n") + ) + dirs["abs_merged_dir"] = os.path.abspath( + os.path.join(dirs["abs_work_dir"], "l10n-central") + ) + for key in dirs.keys(): + if key not in abs_dirs: + abs_dirs[key] = dirs[key] + self.abs_dirs = abs_dirs + return self.abs_dirs + + def _query_upload_env(self): + """returns the environment used for the upload step""" + if self.upload_env: + return self.upload_env + config = self.config + + upload_env = self.query_env(partial_env=config.get("upload_env")) + + self.upload_env = upload_env + return self.upload_env + + def _ensure_upload_path(self): + env = self._query_upload_env() + if "UPLOAD_PATH" in env and not os.path.exists(env["UPLOAD_PATH"]): + self.mkdir_p(env["UPLOAD_PATH"]) + + def get_gecko_l10n_revisions(self): + # Populate self.locales with Thunderbird's locales, and revisions + # from browser/locales/l10n-changesets.json + c = self.config + ignore_locales = c.get("ignore_locales", []) + + dirs = self.query_abs_dirs() + locale_list = os.path.join(dirs["abs_src_dir"], c["locale_list"]) + locales = self.parse_locales_file(locale_list) + locale_changesets_file = os.path.join(dirs["abs_src_dir"], c["locales_file"]) + # parse_locales_file fills in self.l10n_revisions with changesets + self.parse_locales_file(locale_changesets_file) + + for locale in ignore_locales: + if locale in locales: + self.debug("Ignoring locale %s." % locale) + locales.remove(locale) + + self.locales = locales + + # Actions {{{2 + def clone_gecko_locales(self): + self.get_gecko_l10n_revisions() + self.pull_locale_source() + + def clone_monorepo(self): + c = self.config + dirs = self.query_abs_dirs() + + locales_file = os.path.join(dirs["abs_src_dir"], c["hg_comm_locales_file"]) + locales_data = {} + if locales_file.endswith(".json"): + with open(locales_file) as fh: + locales_data = json.load(fh) + # would use en-US, but it's not in this file! + self.comm_l10n_revision = locales_data.get("en-GB", {}).get("revision") + + if self.comm_l10n_revision: + self.mkdir_p(dirs["abs_checkout_dir"]) + self.vcs_checkout( + dest=dirs["abs_comm_l10n_dir"], + repo=c["hg_comm_l10n_repo"], + branch=self.comm_l10n_revision, + vcs="hg", + ) + else: + raise Exception( + f"Unable to find revision from comm-l10n repo using " + f"{c['hg_comm_locales_file']}." + ) + + def merge_repos(self): + dirs = self.query_abs_dirs() + if os.path.exists(dirs["abs_merged_dir"]): + self.rmtree(dirs["abs_merged_dir"]) + + file_registry = FileRegistry() + + def add_to_registry(base_path, patterns): + finder = FileFinder(base_path) + for pattern in patterns: + for _lang in self.locales: + for _filepath, _fileobj in finder.find(pattern.format(lang=_lang)): + _filepath = os.path.join("l10n-central", _filepath) + file_registry.add(_filepath, _fileobj) + + add_to_registry(dirs["abs_l10n_central_dir"], GECKO_STRINGS_PATTERNS) + add_to_registry(dirs["abs_comm_l10n_dir"], COMM_STRINGS_PATTERNS) + + self.file_registry = file_registry + + def pack_merged(self): + self._ensure_upload_path() + upload_path = self.config["upload_env"]["UPLOAD_PATH"] + archive_path = os.path.join(upload_path, self.config["upload_file"]) + + with open(archive_path, "wb") as f: + with ZstdCompressor().stream_writer(f) as z: + create_tar_from_files(z, dict(self.file_registry)) + + def gen_changesets(self): + changeset_data = { + "gecko_strings": self.gecko_locale_revisions, + "comm_strings": { + "repo": self.config["hg_comm_l10n_repo"], + "revision": self.comm_l10n_revision, + }, + } + upload_path = self.config["upload_env"]["UPLOAD_PATH"] + changesets_file = os.path.join(upload_path, self.config["changesets_file"]) + with open(changesets_file, "w") as f: + json.dump(changeset_data, f, sort_keys=True, indent=2) + + +if __name__ == "__main__": + single_locale = CommMultiLocale() + single_locale.run_and_exit() diff --git a/comm/taskcluster/scripts/source-test-clang-format.sh b/comm/taskcluster/scripts/source-test-clang-format.sh new file mode 100755 index 0000000000..90f449a03b --- /dev/null +++ b/comm/taskcluster/scripts/source-test-clang-format.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Run the Firefox setup script +source "$HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-clang-setup.sh" + +# Append comm/.clang-format-ignore contents to $topsrcdir/.clang-format-ignore +sed -e 's%^\([a-z]\)%comm/\1%' comm/.clang-format-ignore >> .clang-format-ignore + +# Update mozconfig file with Thunderbird build options +cat <<EOT >> "$MOZCONFIG" +ac_add_options --enable-project=comm/mail +EOT + +# Run mach clang-format +# shellcheck disable=SC2068 +./mach --log-no-times clang-format --output "$HOME/clang-format.json" --format json -p $@ +# shellcheck disable=SC2068 +./mach --log-no-times clang-format --output "$HOME/clang-format.diff" --format diff -p $@ + +# Exit with an error code if clang-format.diff contains a proper diff. +# Needed because mach clang-format will exit 0 regardless of outcome. +# If no formatting is needed, clang-format.diff will have a single \n, +# so check for a file size > 1 byte. +DIFF_SIZE=$(stat -c %s "$HOME/clang-format.diff") +if [[ "$DIFF_SIZE" -gt 1 ]]; then + echo "Exiting with error status. DIFF_SIZE is $DIFF_SIZE." + exit 1 +else + exit 0 +fi |