diff options
Diffstat (limited to 'comm/taskcluster/comm_taskgraph/transforms')
18 files changed, 1236 insertions, 0 deletions
diff --git a/comm/taskcluster/comm_taskgraph/transforms/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py new file mode 100644 index 0000000000..a80c058a2b --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running toolchain-building jobs via dedicated scripts in comm-central +""" + +import os.path + +import taskgraph +import taskgraph.util.path as util_path +from taskgraph.util.schema import resolve_keyed_by +from voluptuous import Any, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import docker_worker_add_artifacts +from gecko_taskgraph.transforms.job.toolchain import toolchain_defaults, toolchain_run_schema +from gecko_taskgraph.util.attributes import RELEASE_PROJECTS +from gecko_taskgraph.util.hash import hash_paths as hash_paths_gecko_root + +from comm_taskgraph.util.hash import hash_paths_extended + +CACHE_TYPE = "toolchains.v3" + +TOOLCHAIN_SCRIPT_PATH = "comm/taskcluster/scripts" + + +comm_toolchain_run_schema = toolchain_run_schema.extend( + { + Required("using"): Any("comm-toolchain-script"), + Optional("script"): str, + } +) + + +def hash_paths(*args): + """ + Helper function while the single repository project is in development. + The extended version of hash_paths found in comm_taskgraph.util.hash is + not necessary (and does not work) with single-repo. This is a wrapper + function to pick the right function based on the presence of a comm/.hg + directory. + """ + comm_hg_path = util_path.join(GECKO, "comm", ".hg") + if os.path.exists(comm_hg_path): + return hash_paths_extended(*args) + else: + return hash_paths_gecko_root(*args) + + +def get_digest_data(config, run, taskdesc): + """ + Copied from gecko_taskgraph.transforms.job.toolchain, with minor + modifications to support the required script path. + """ + files = list(run.pop("resources", [])) + # This file + files.append("comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py") + # The script + if "script" in run: + files.append("{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run["script"])) + # Tooltool manifest if any is defined: + tooltool_manifest = taskdesc["worker"]["env"].get("TOOLTOOL_MANIFEST") + if tooltool_manifest: + files.append(tooltool_manifest) + + # Accumulate dependency hashes for index generation. + data = [hash_paths(GECKO, files)] + + data.append(taskdesc["attributes"]["toolchain-artifact"]) + + # If the task uses an in-tree docker image, we want it to influence + # the index path as well. Ideally, the content of the docker image itself + # should have an influence, but at the moment, we can't get that + # information here. So use the docker image name as a proxy. Not a lot of + # changes to docker images actually have an impact on the resulting + # toolchain artifact, so we'll just rely on such important changes to be + # accompanied with a docker image name change. + image = taskdesc["worker"].get("docker-image", {}).get("in-tree") + if image: + data.append(image) + + # Likewise script arguments should influence the index. + args = run.get("arguments") + if args: + data.extend(args) + + if taskdesc["attributes"].get("rebuild-on-release"): + # Add whether this is a release branch or not + data.append(str(config.params["project"] in RELEASE_PROJECTS)) + return data + + +@run_job_using( + "docker-worker", + "comm-toolchain-script", + schema=comm_toolchain_run_schema, + defaults=toolchain_defaults, +) +def docker_worker_toolchain(config, job, taskdesc): + run = job["run"] + run["comm-checkout"] = True + + worker = taskdesc["worker"] = job["worker"] + worker["chain-of-trust"] = True + + # If the task doesn't have a docker-image, set a default + worker.setdefault("docker-image", {"in-tree": "deb11-toolchain-build"}) + + # Toolchain checkouts don't live under {workdir}/checkouts + workspace = "{workdir}/workspace/build".format(**run) + gecko_path = "{}/src".format(workspace) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "GECKO_PATH": gecko_path, + "TOOLCHAIN_ARTIFACT": run["toolchain-artifact"], + } + ) + + attributes = taskdesc.setdefault("attributes", {}) + attributes["toolchain-artifact"] = run.pop("toolchain-artifact") + toolchain_artifact = attributes["toolchain-artifact"] + if not toolchain_artifact.startswith("public/build/"): + attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact) + + resolve_keyed_by( + run, + "toolchain-alias", + item_name=taskdesc["label"], + project=config.params["project"], + ) + alias = run.pop("toolchain-alias", None) + if alias: + attributes["toolchain-alias"] = alias + if "toolchain-env" in run: + attributes["toolchain-env"] = run.pop("toolchain-env") + + # Allow the job to specify where artifacts come from, but add + # public/build if it's not there already. + artifacts = worker.setdefault("artifacts", []) + if not artifacts: + docker_worker_add_artifacts(config, job, taskdesc) + + digest_data = get_digest_data(config, run, taskdesc) + + if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast: + name = taskdesc["label"].replace(f"{config.kind}-", "", 1) + taskdesc["cache"] = { + "type": CACHE_TYPE, + "name": name, + "digest-data": digest_data, + } + + run["using"] = "run-task" + run["cwd"] = run["workdir"] + run["command"] = [ + "workspace/build/src/{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run.pop("script")) + ] + run.pop("arguments", []) + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n.py b/comm/taskcluster/comm_taskgraph/transforms/l10n.py new file mode 100644 index 0000000000..51246ad888 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Do transforms specific to l10n kind +""" + +from taskgraph.transforms.base import TransformSequence, ValidateSchema +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.l10n import ( + all_locales_attribute, + chunk_locales, + copy_in_useful_magic, + handle_artifact_prefix, + handle_keyed_by, + l10n_description_schema, + make_job_description, + set_extra_config, + setup_name, +) + + +def setup_signing_dependency(config, jobs): + """Sets up a task dependency to the signing job this relates to""" + for job in jobs: + job["dependencies"].update( + { + "build": job["dependent-tasks"]["build"].label, + } + ) + + if job["attributes"]["build_platform"].startswith("win"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + + if "shippable" in job["attributes"]["build_platform"]: + if job["attributes"]["build_platform"].startswith("macosx"): + job["dependencies"].update( + {"repackage": job["dependent-tasks"]["repackage"].label} + ) + if job["attributes"]["build_platform"].startswith("linux"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + yield job + + +def handle_keyed_by_local(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locales-file", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +transforms = TransformSequence() + + +for transform_func in ( + setup_name, + copy_in_useful_magic, + handle_keyed_by_local, + ValidateSchema(l10n_description_schema), + setup_signing_dependency, + handle_keyed_by, + handle_artifact_prefix, + all_locales_attribute, + chunk_locales, + ValidateSchema(l10n_description_schema), + set_extra_config, + make_job_description, +): + transforms.add(transform_func) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py new file mode 100644 index 0000000000..a642b0e592 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Create a strings build artifact to be consumed by shippable-l10n. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locale-list", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + locale_list = job.pop("locale-list") + comm_locales_file = job.pop("comm-locales-file") + browser_locales_file = job.pop("browser-locales-file") + job["run"].update( + { + "job-script": "comm/taskcluster/scripts/build-l10n-pre.sh", + "options": [ + f"locale-list={locale_list}", + f"comm-locales-file={comm_locales_file}", + f"browser-locales-file={browser_locales_file}", + ], + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py new file mode 100644 index 0000000000..92a95cb612 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_path + +from gecko_taskgraph.transforms.build_signing import add_signed_routes +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +transforms = TransformSequence() + +transforms.add(add_signed_routes) + + +@transforms.add +def define_upstream_artifacts(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) + + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, "strings_all.tar.zst"), + get_artifact_path(job, "l10n-changesets.json"), + ], + "formats": ["autograph_gpg"], + } + ] + + task_ref = f"<{upstream_artifact_task.kind}>" + task_type = "build" + if "notarization" in upstream_artifact_task.kind: + task_type = "scriptworker" + + job["upstream-artifacts"] = [ + { + "taskId": {"task-reference": task_ref}, + "taskType": task_type, + "paths": spec["artifacts"], + "formats": spec["formats"], + } + for spec in artifacts_specifications + ] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py new file mode 100644 index 0000000000..75ced147bb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Fix-ups for comm-central merge automation +""" + +import os +import re + +from taskgraph.transforms.base import TransformSequence + +from comm_taskgraph import COMM + +transforms = TransformSequence() + + +def do_suite_verbump(replacements): + """Bump the minor version of suite version files.""" + allowed_files = ("suite/config/version.txt", "suite/config/version_display.txt") + old_version, new_version = None, None + + new_replacements = [] + for file, old, new in replacements: + if file not in allowed_files: + break + if old_version is None or new_version is None: + path = os.path.join(COMM, file) + data = open(path).read() + match = re.match(r"^(2)\.(\d+)(a1)$", data) + if match: + old_version = match.group(0) + + old_minor = match.group(2) + new_minor = str(int(old_minor) + 1) + + new_version = f"{match.group(1)}.{new_minor}{match.group(3)}" + + new_replacements.append([file, old_version, new_version]) + + if len(new_replacements) == len(replacements): + return new_replacements + else: + raise Exception(f"do_suite_version failed: {replacements}, {new_replacements}") + + +@transforms.add +def update_suite_versions(config, tasks): + for task in tasks: + if "merge_config" not in config.params: + break + behavior = config.params["merge_config"]["behavior"] + if behavior == "comm-bump-central": + merge_config = task["worker"]["merge-info"] + replacements = merge_config["replacements"] + merge_config["replacements"] = do_suite_verbump(replacements) + + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/partials.py b/comm/taskcluster/comm_taskgraph/transforms/partials.py new file mode 100644 index 0000000000..39d6cf95a7 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/partials.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to partial update building +""" +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def update_scopes(config, jobs): + """ + Firefox does some caching when building partial updates, but there's no bucket for Thunderbird + at the moment. In the meantime, remove the scope from the task to avoid an error. + """ + # If no balrog release history, then don't run + if not config.params.get("release_history"): + return + + MBSDIFF_SCOPE = "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" + + for job in jobs: + task = job["task"] + if MBSDIFF_SCOPE in task["scopes"]: + task["scopes"].remove(MBSDIFF_SCOPE) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py new file mode 100644 index 0000000000..8f6c7a5d5f --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-push-langpacks task into an actual task description. +""" + +import json +import os +from contextlib import contextmanager + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, taskref_or_string +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from mozbuild.action.langpack_manifest import get_version_maybe_buildid + +transforms = TransformSequence() + +langpack_push_description_schema = schema.extend( + { + Required("label"): str, + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): { + Required("docker-image"): {"in-tree": str}, + Required("implementation"): "docker-worker", + Required("os"): "linux", + Optional("max-run-time"): int, + Required("env"): {str: taskref_or_string}, + Required("channel"): optionally_keyed_by( + "project", "platform", Any("listed", "unlisted") + ), + Required("command"): [taskref_or_string], + }, + Required("run-on-projects"): [], + Required("scopes"): optionally_keyed_by("release-level", [str]), + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + } +) + + +@transforms.add +def set_label(config, jobs): + for job in jobs: + label = "push-langpacks-{}".format(job["primary-dependency"].label) + job["label"] = label + + yield job + + +transforms.add_validate(langpack_push_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "worker.channel", + item_name=job["label"], + project=config.params["project"], + platform=job["primary-dependency"].attributes["build_platform"], + ) + + yield job + + +@transforms.add +def copy_attributes(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + job["attributes"]["chunk_locales"] = dep_job.attributes.get("chunk_locales", ["en-US"]) + + yield job + + +@transforms.add +def filter_out_macos_jobs_but_mac_only_locales(config, jobs): + for job in jobs: + build_platform = job["primary-dependency"].attributes.get("build_platform") + + if build_platform == "linux64-shippable": + yield job + elif ( + build_platform == "macosx64-shippable" + and "ja-JP-mac" in job["attributes"]["chunk_locales"] + ): + # Other locales of the same job shouldn't be processed + job["attributes"]["chunk_locales"] = ["ja-JP-mac"] + job["label"] = job["label"].replace( + # Guard against a chunk 10 or chunk 1 (latter on try) weird munging + "-{}/".format(job["attributes"]["l10n_chunk"]), + "-ja-JP-mac/", + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(P{})".format(job["attributes"].get("l10n_chunk", "")) + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + ) + + job["dependencies"] = {dep_job.kind: dep_job.label} + job["treeherder"] = treeherder + + yield job + + +def generate_upstream_artifacts(upstream_task_ref, locales): + return [ + { + "task": upstream_task_ref, + "extract": False, + "dest": f"{locale}", + "artifact": "public/build{locale}/target.langpack.xpi".format( + locale="" if locale == "en-US" else "/" + locale + ), + } + for locale in locales + ] + + +@transforms.add +def make_fetches(config, jobs): + for job in jobs: + upstream_task_ref = get_upstream_task_ref(job, expected_kinds=("build", "shippable-l10n")) + + worker = job.setdefault("worker", {}) + worker["taskcluster-proxy"] = True + + env = worker.setdefault("env", {}) + + job_fetches = generate_upstream_artifacts( + upstream_task_ref, job["attributes"]["chunk_locales"] + ) + env["MOZ_FETCHES"] = { + "task-reference": json.dumps( + sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True + ) + } + env["MOZ_SCM_LEVEL"] = config.params["level"] + + yield job + + +def get_upstream_task_ref(job, expected_kinds): + upstream_tasks = [ + job_kind for job_kind in job["dependencies"].keys() if job_kind in expected_kinds + ] + + if len(upstream_tasks) > 1: + raise Exception("Only one dependency expected") + + return f"<{upstream_tasks[0]}>" + + +@contextmanager +def environment(key, value): + """Set an environment variable in a context""" + old_value = None + if key in os.environ: + old_value = os.environ[key] + os.environ[key] = value + try: + yield True + finally: + if old_value is None: + del os.environ[key] + else: + os.environ[key] = old_value + + +@transforms.add +def set_env(config, jobs): + buildid = config.params["moz_build_date"] + app_version = config.params.get("app_version") + + with environment("MOZ_BUILD_DATE", buildid): + langpack_version = get_version_maybe_buildid(app_version) + + for job in jobs: + job["worker"].get("env", {}).update( + { + "LANGPACK_VERSION": langpack_version, + "LOCALES": json.dumps(job["attributes"]["chunk_locales"]), + "MOZ_FETCHES_DIR": "fetches", + "ATN_CHANNEL": job["worker"].get("channel"), + } + ) + + yield job + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + del job["worker"]["channel"] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py new file mode 100644 index 0000000000..c4f39fc4eb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-flatpak-push kind into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +push_flatpak_description_schema = Schema( + { + Required("name"): str, + Required("job-from"): task_description_schema["job-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Required("flathub-scope"): str, + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(push_flatpak_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + if len(job["dependencies"]) != 1: + raise Exception("Exactly 1 dependency is required") + + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(job["dependencies"]) + + resolve_keyed_by( + job, + "worker.channel", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + if release_level(config.params["project"]) == "production": + job.setdefault("scopes", []).append( + add_scope_prefix( + config, + "{}:{}".format(job["flathub-scope"], job["worker"]["channel"]), + ) + ) + del job["flathub-scope"] + + yield job + + +def generate_upstream_artifacts(dependencies): + return [ + { + "taskId": {"task-reference": f"<{task_kind}>"}, + "taskType": "build", + "paths": ["public/build/target.flatpak.tar.xz"], + } + for task_kind in dependencies.keys() + ] diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_started.py b/comm/taskcluster/comm_taskgraph/transforms/release_started.py new file mode 100644 index 0000000000..ce0cfa9fab --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_started.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add notifications via taskcluster-notify for release tasks +""" + +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def add_notifications(config, jobs): + for job in jobs: + label = "{}-{}".format(config.kind, job["name"]) + + resolve_keyed_by(job, "emails", label, project=config.params["project"]) + emails = [email.format(config=config.__dict__) for email in job.pop("emails")] + + command = [ + "release", + "send-buglist-email", + "--version", + config.params["version"], + "--product", + job["shipping-product"], + "--revision", + config.params["comm_head_rev"], + "--build-number", + str(config.params["build_number"]), + "--repo", + config.params["comm_head_repository"], + ] + for address in emails: + command += ["--address", address] + command += [ + # We wrap this in `{'task-reference': ...}` below + "--task-group-id", + "<decision>", + ] + + job["scopes"] = ["notify:email:{}".format(address) for address in emails] + job["run"] = { + "using": "mach", + "sparse-profile": "mach", + "mach": {"task-reference": " ".join(map(shell_quote, command))}, + } + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py new file mode 100644 index 0000000000..793519f6ff --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def add_langpack_fetches(config, jobs): + """Adds the fetch configuration for the langpacks. This is done here + because Thunderbird langpacks are not signed and therefore not found as + artifacts of "shippable-l10n-signing" like they are for Firefox. Need to + use "shippable-l10n". + """ + + def depends_filter(dep_task): + return ( + dep_task.kind == "shippable-l10n" + and dep_task.attributes["build_platform"] == "linux64-shippable" + and dep_task.attributes["build_type"] == "opt" + ) + + for job in jobs: + dependencies = job.get("dependencies", {}) + fetches = job.setdefault("fetches", {}) + + # The keys are unique, like `shippable-l10n-linux64-shippable-1/opt`, so we + # can't ask for the tasks directly, we must filter for them. + for t in filter(depends_filter, config.kind_dependencies_tasks.values()): + dependencies.update({t.label: t.label}) + + fetches.update( + { + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"distribution/extensions/{loc}", + } + for loc in t.attributes["chunk_locales"] + ] + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/signing.py b/comm/taskcluster/comm_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..297fec0d2e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/signing.py @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.signed_artifacts import is_notarization_kind + +transforms = TransformSequence() + + +def check_notarization(dependencies): + """ + Determine whether a signing job is the last step of a notarization + by looking at its dependencies. + """ + for dep in dependencies: + if is_notarization_kind(dep): + return True + + +@transforms.add +def remove_widevine(config, jobs): + """ + Remove references to widevine signing. + + This is to avoid adding special cases for handling signed artifacts + in mozilla-central code. Artifact signature formats are determined in + gecko_taskgraph.util.signed_artifacts. There's no override mechanism so we + remove the autograph_widevine format here. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + widevine_scope = "project:comm:thunderbird:releng:signing:format:autograph_widevine" + if widevine_scope in task["scopes"]: + task["scopes"].remove(widevine_scope) + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_widevine" in artifact.get("formats", []): + artifact["formats"].remove("autograph_widevine") + + yield job + + +@transforms.add +def no_sign_langpacks(config, jobs): + """ + Remove langpacks from signing jobs after they are automatically added. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_langpack" in artifact.get("formats", []): + artifact["formats"].remove("autograph_langpack") + + # Make sure that there are no .xpi files in the artifact list + if all([p.endswith("target.langpack.xpi") for p in artifact["paths"]]): + payload["upstreamArtifacts"].remove(artifact) + + yield job + + +@transforms.add +def check_for_no_formats(config, jobs): + """ + Check for signed artifacts without signature formats and remove them to + avoid scriptworker errors. + Signing jobs that use macOS notarization do not have formats, so keep + those. + """ + for job in jobs: + if not check_notarization(job["dependencies"]): + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "formats" in artifact and not artifact["formats"]: + for remove_path in artifact["paths"]: + job["release-artifacts"].remove(remove_path) + + payload["upstreamArtifacts"].remove(artifact) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/source_test.py b/comm/taskcluster/comm_taskgraph/transforms/source_test.py new file mode 100644 index 0000000000..d4ae39c134 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/source_test.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import shlex + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path + +from gecko_taskgraph.files_changed import get_changed_files + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def get_patterns(job): + """Get the "run on-changed" file patterns.""" + optimization = job.get("optimization", {}) + if optimization: + return optimization.copy().popitem()[1] + return [] + + +def shlex_join(split_command): + """shlex.join from Python 3.8+""" + return " ".join(shlex.quote(arg) for arg in split_command) + + +@transforms.add +def changed_clang_format(config, jobs): + """ + Transform for clang-format job to set the commandline to only check + C++ files that were changed in the current push rather than running on + the entire repository. + """ + for job in jobs: + if job.get("name", "") == "clang-format": + repository = config.params.get("comm_head_repository") + revision = config.params.get("comm_head_rev") + + match_patterns = get_patterns(job) + changed_files = { + join_path("comm", file) for file in get_changed_files(repository, revision) + } + + cpp_files = [] + for pattern in match_patterns: + for path in changed_files: + if match_path(path, pattern): + cpp_files.append(path) + + # In the event that no C/C++ files were changed in the current push, + # the commandline will end up being invalid. But, the clang-format + # job will get dropped by optimization, so it doesn't really matter. + if cpp_files: + job["run"]["command"] = job["run"]["command"].format( + changed_files=shlex_join(cpp_files) + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_build.py b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py new file mode 100644 index 0000000000..eff16d8e69 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def munge_environment(config, jobs): + for job in jobs: + env = job["worker"]["env"] + # Remove MOZ_SOURCE_CHANGESET/REPO from the job environment and discard + # if present. Having these variables set in the environment causes problems + # with generating debug sym files. Bug 1747879. + env.pop("MOZ_SOURCE_CHANGESET", None) + env.pop("MOZ_SOURCE_REPO", None) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py new file mode 100644 index 0000000000..ecdfd577af --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build a command to run `mach tb-l10n-x-channel`. +""" + +from shlex import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from rocbuild.notify import TB_BUILD_ADDR + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for item in ["ssh-key-secret", "run.actions"]: + resolve_keyed_by(job, item, item, **{"level": str(config.params["level"])}) + yield job + + +@transforms.add +def build_command(config, jobs): + for job in jobs: + command = [ + "tb-l10n-x-channel", + "-o", + "/builds/worker/artifacts/outgoing.diff", + "--attempts", + "5", + ] + ssh_key_secret = job.pop("ssh-key-secret") + if ssh_key_secret: + command.extend(["--ssh-secret", ssh_key_secret]) + job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") + job["scopes"].append(f"notify:email:{TB_BUILD_ADDR}") + + command.extend(job["run"].pop("actions", [])) + job.setdefault("run", {}).update( + {"using": "mach", "mach": " ".join(map(shell_quote, command))} + ) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tests.py b/comm/taskcluster/comm_taskgraph/transforms/tests.py new file mode 100644 index 0000000000..9c1df4663e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tests.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to test jobs +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def optimization_keyed_by(config, tasks): + """Used to set the optimization strategy""" + for task in tasks: + resolve_keyed_by(task, "optimization", item_name=task["test-name"]) + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py new file mode 100644 index 0000000000..2f9ee143fa --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from urllib.parse import urlsplit + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev +from gecko_taskgraph.transforms.update_verify_config import ensure_wrapped_singlequote +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +# The beta regexes do not match point releases. +# In the rare event that we do ship a point +# release to beta, we need to either: +# 1) update these regexes to match that specific version +# 2) pass a second include version that matches that specific version +INCLUDE_VERSION_REGEXES = { + "beta": r"'^(\d+\.\d+b\d+)$'", + "nonbeta": r"'^\d+\.\d+(\.\d+)?$'", + # Previous major versions, for update testing before we update users to a new esr + "release-next": r"'^(91|102)\.\d+(\.\d+)?$'", +} + +MAR_CHANNEL_ID_OVERRIDE_REGEXES = { + "beta": r"'^\d+\.\d+(\.\d+)?$$,thunderbird-comm-beta,thunderbird-comm-release'", +} + + +ensure_wrapped_singlequote(INCLUDE_VERSION_REGEXES) +ensure_wrapped_singlequote(MAR_CHANNEL_ID_OVERRIDE_REGEXES) + + +@transforms.add +def add_command(config, tasks): + keyed_by_args = [ + "channel", + "archive-prefix", + "previous-archive-prefix", + "aus-server", + "override-certs", + "include-version", + "mar-channel-id-override", + "last-watershed", + ] + optional_args = [ + "updater-platform", + ] + + release_config = get_release_config(config) + + for task in tasks: + task["description"] = "generate update verify config for {}".format( + task["attributes"]["build_platform"] + ) + + command = [ + "python", + "testing/mozharness/scripts/release/update-verify-config-creator.py", + "--product", + task["extra"]["product"], + "--stage-product", + task["shipping-product"], + "--app-name", + task["extra"]["app-name"], + "--branch-prefix", + task["extra"]["branch-prefix"], + "--platform", + task["extra"]["platform"], + "--to-version", + release_config["version"], + "--to-app-version", + release_config["appVersion"], + "--to-build-number", + str(release_config["build_number"]), + "--to-buildid", + config.params["moz_build_date"], + "--to-revision", + get_branch_rev(config), + "--output-file", + "update-verify.cfg", + ] + + repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/") + command.extend(["--repo-path", repo_path]) + + if release_config.get("partial_versions"): + for partial in release_config["partial_versions"].split(","): + command.extend(["--partial-version", partial.split("build")[0]]) + + for arg in optional_args: + if task["extra"].get(arg): + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + for arg in keyed_by_args: + thing = f"extra.{arg}" + resolve_keyed_by( + task, + thing, + item_name=task["name"], + platform=task["attributes"]["build_platform"], + **{ + "release-type": config.params["release_type"], + "release-level": release_level(config.params["project"]), + }, + ) + # ignore things that resolved to null + if not task["extra"].get(arg): + continue + if arg == "include-version": + task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]] + if arg == "mar-channel-id-override": + task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]] + + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + task["run"].update( + { + "using": "mach", + "mach": " ".join(command), + } + ) + + yield task |