diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py')
| -rw-r--r-- | taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py | 326 |
1 files changed, 326 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py new file mode 100644 index 0000000000..40dc370f33 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py @@ -0,0 +1,326 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import logging +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.partners import get_ftp_platform, get_partner_config_by_kind +from gecko_taskgraph.util.scriptworker import ( + add_scope_prefix, + get_beetmover_bucket_scope, +) + +logger = logging.getLogger(__name__) + + +beetmover_description_schema = schema.extend( + { + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + Required("partner-bucket-scope"): optionally_keyed_by("release-level", str), + Required("partner-public-path"): Any(None, str), + Required("partner-private-path"): Any(None, str), + Optional("extra"): object, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("priority"): task_description_schema["priority"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "partner-bucket-scope", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + repack_id = dep_job.task.get("extra", {}).get("repack_id") + if not repack_id: + raise Exception("Cannot find repack id!") + + attributes = dep_job.attributes + build_platform = attributes.get("build_platform") + if not build_platform: + raise Exception("Cannot find build platform!") + + label = dep_job.label.replace("repackage-signing-l10n", "beetmover-") + label = dep_job.label.replace("repackage-signing-", "beetmover-") + label = label.replace("repackage-", "beetmover-") + label = label.replace("chunking-dummy-", "beetmover-") + description = ( + "Beetmover submission for repack_id '{repack_id}' for build '" + "{build_platform}/{build_type}'".format( + repack_id=repack_id, + build_platform=build_platform, + build_type=attributes.get("build_type"), + ) + ) + + dependencies = {} + + base_label = "release-partner-repack" + if "eme" in config.kind: + base_label = "release-eme-free-repack" + dependencies["build"] = f"{base_label}-{build_platform}" + if "macosx" in build_platform or "win" in build_platform: + dependencies["repackage"] = "{}-repackage-{}-{}".format( + base_label, build_platform, repack_id.replace("/", "-") + ) + dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format( + base_label, build_platform, repack_id.replace("/", "-") + ) + + attributes = copy_attributes_from_dependent_job(dep_job) + + task = { + "label": label, + "description": description, + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "shipping-phase": job["shipping-phase"], + "shipping-product": job.get("shipping-product"), + "partner-private-path": job["partner-private-path"], + "partner-public-path": job["partner-public-path"], + "partner-bucket-scope": job["partner-bucket-scope"], + "extra": { + "repack_id": repack_id, + }, + } + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + + yield task + + +def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False): + action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") + + task = deepcopy(job) + task["scopes"] = [bucket_scope, action_scope] + task["worker-type"] = "beetmover" + task["partner_public"] = partner_public + if partner_public: + task["label"] = "{}-public".format(task["label"]) + return task + + +@transforms.add +def split_public_and_private(config, jobs): + public_bucket_scope = get_beetmover_bucket_scope(config) + partner_config = get_partner_config_by_kind(config, config.kind) + + for job in jobs: + partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) + partner, subpartner, _ = job["extra"]["repack_id"].split("/") + + if partner_config[partner][subpartner].get("upload_to_candidates"): + # public + yield populate_scopes_and_worker_type( + config, job, public_bucket_scope, partner_public=True + ) + else: + # private + yield populate_scopes_and_worker_type( + config, job, partner_bucket_scope, partner_public=False + ) + + +def generate_upstream_artifacts( + job, + build_task_ref, + repackage_task_ref, + repackage_signing_task_ref, + platform, + repack_id, + partner_path, + repack_stub_installer=False, +): + + upstream_artifacts = [] + artifact_prefix = get_artifact_prefix(job) + + if "linux" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": build_task_ref}, + "taskType": "build", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2.asc"], + "locale": partner_path, + } + ) + elif "macosx" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"], + "locale": partner_path, + } + ) + elif "win" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"], + "locale": partner_path, + } + ) + if platform.startswith("win32") and repack_stub_installer: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [ + "{}/{}/target.stub-installer.exe".format( + artifact_prefix, repack_id + ) + ], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [ + "{}/{}/target.stub-installer.exe.asc".format( + artifact_prefix, repack_id + ) + ], + "locale": partner_path, + } + ) + + if not upstream_artifacts: + raise Exception("Couldn't find any upstream artifacts.") + + return upstream_artifacts + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + platform = job["attributes"]["build_platform"] + repack_id = job["extra"]["repack_id"] + partner, subpartner, locale = job["extra"]["repack_id"].split("/") + partner_config = get_partner_config_by_kind(config, config.kind) + repack_stub_installer = partner_config[partner][subpartner].get( + "repack_stub_installer" + ) + build_task = None + repackage_task = None + repackage_signing_task = None + + for dependency in job["dependencies"].keys(): + if "repackage-signing" in dependency: + repackage_signing_task = dependency + elif "repackage" in dependency: + repackage_task = dependency + else: + build_task = "build" + + build_task_ref = "<" + str(build_task) + ">" + repackage_task_ref = "<" + str(repackage_task) + ">" + repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">" + + # generate the partner path; we'll send this to beetmover as the "locale" + ftp_platform = get_ftp_platform(platform) + repl_dict = { + "build_number": config.params["build_number"], + "locale": locale, + "partner": partner, + "platform": ftp_platform, + "release_partner_build_number": config.params[ + "release_partner_build_number" + ], + "subpartner": subpartner, + "version": config.params["version"], + } + partner_public = job["partner_public"] + if partner_public: + partner_path_key = "partner-public-path" + else: + partner_path_key = "partner-private-path" + # Kinds can set these to None + if not job[partner_path_key]: + continue + partner_path = job[partner_path_key].format(**repl_dict) + del job["partner_public"] + del job["partner-private-path"] + del job["partner-public-path"] + del job["partner-bucket-scope"] + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_upstream_artifacts( + job, + build_task_ref, + repackage_task_ref, + repackage_signing_task_ref, + platform, + repack_id, + partner_path, + repack_stub_installer, + ), + "partner-public": partner_public, + } + job["worker"] = worker + + yield job |