diff options
Diffstat (limited to 'taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py')
| -rw-r--r-- | taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py | 243 |
1 files changed, 243 insertions, 0 deletions
diff --git a/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py new file mode 100644 index 0000000000..7ed9a11f33 --- /dev/null +++ b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py @@ -0,0 +1,243 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import six +from six import text_type +from taskgraph.loader.single_dep import schema +from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.beetmover import craft_release_properties +from taskgraph.util.attributes import copy_attributes_from_dependent_job +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.scriptworker import ( + get_beetmover_bucket_scope, + get_beetmover_action_scope, + generate_beetmover_upstream_artifacts, + generate_beetmover_artifact_map, +) +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from taskgraph.transforms.task import task_description_schema +from voluptuous import Required, Optional + +import logging +import copy + +logger = logging.getLogger(__name__) + + +transforms = TransformSequence() + + +beetmover_description_schema = schema.extend( + { + # attributes is used for enabling artifact-map by declarative artifacts + Required("attributes"): {text_type: object}, + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): text_type, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Required("description"): text_type, + Required("worker-type"): optionally_keyed_by("release-level", text_type), + Required("run-on-projects"): [], + # locale is passed only for l10n beetmoving + Optional("locale"): text_type, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + + +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for field in ("worker-type", "attributes.artifact_map"): + resolve_keyed_by( + job, + field, + item_name=job["label"], + **{ + "release-level": config.params.release_level(), + "release-type": config.params["release_type"], + "project": config.params["project"], + } + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(BM{})".format(attributes.get("l10n_chunk", "")) + ) + + job["attributes"].update(copy_attributes_from_dependent_job(dep_job)) + job["attributes"]["chunk_locales"] = dep_job.attributes.get( + "chunk_locales", ["en-US"] + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + platform=job["attributes"]["build_platform"], + ) + + job["scopes"] = [ + get_beetmover_bucket_scope(config), + get_beetmover_action_scope(config), + ] + + job["dependencies"] = {"langpack-copy": dep_job.label} + + job["run-on-projects"] = job.get( + "run_on_projects", dep_job.attributes["run_on_projects"] + ) + job["treeherder"] = treeherder + job["shipping-phase"] = job.get( + "shipping-phase", dep_job.attributes["shipping_phase"] + ) + job["shipping-product"] = dep_job.attributes["shipping_product"] + + yield job + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + platform = job["attributes"]["build_platform"] + locale = job["attributes"]["chunk_locales"] + + job["worker"] = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, + job, + platform, + locale, + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locale + ), + } + + yield job + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + + yield job + + +@transforms.add +def yield_all_platform_jobs(config, jobs): + # Even though langpacks are now platform independent, we keep beetmoving them at old + # platform-specific locations. That's why this transform exist + # The linux64 and mac specific ja-JP-mac are beetmoved along with the signing beetmover + # So while the dependent jobs are linux here, we only yield jobs for other platforms + for job in jobs: + platforms = ("linux", "macosx64", "win32", "win64") + if "devedition" in job["attributes"]["build_platform"]: + platforms = ("{}-devedition".format(plat) for plat in platforms) + for platform in platforms: + platform_job = copy.deepcopy(job) + if "ja" in platform_job["attributes"]["chunk_locales"] and platform in ( + "macosx64", + "macosx64-devedition", + ): + platform_job = _strip_ja_data_from_linux_job(platform_job) + + platform_job = _change_platform_data(config, platform_job, platform) + + yield platform_job + + +def _strip_ja_data_from_linux_job(platform_job): + # Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where + # "ja" could be any of "aa", "bb", "cc", "dd" + platform_job["description"] = platform_job["description"].replace("ja/", "") + platform_job["description"] = platform_job["description"].replace("/ja", "") + + platform_job["worker"]["upstream-artifacts"] = [ + artifact + for artifact in platform_job["worker"]["upstream-artifacts"] + if artifact["locale"] != "ja" + ] + + return platform_job + + +def _change_platform_in_artifact_map_paths(paths, orig_platform, new_platform): + amended_paths = {} + for artifact, artifact_info in six.iteritems(paths): + amended_artifact_info = { + "checksums_path": artifact_info["checksums_path"].replace( + orig_platform, new_platform + ), + "destinations": [ + d.replace(orig_platform, new_platform) + for d in artifact_info["destinations"] + ], + } + amended_paths[artifact] = amended_artifact_info + + return amended_paths + + +def _change_platform_data(config, platform_job, platform): + orig_platform = "linux64" + if "devedition" in platform: + orig_platform = "linux64-devedition" + platform_job["attributes"]["build_platform"] = platform + platform_job["label"] = platform_job["label"].replace(orig_platform, platform) + platform_job["description"] = platform_job["description"].replace( + orig_platform, platform + ) + platform_job["treeherder"]["platform"] = platform_job["treeherder"][ + "platform" + ].replace(orig_platform, platform) + platform_job["worker"]["release-properties"]["platform"] = platform + + # amend artifactMap entries as well + platform_mapping = { + "linux64": "linux-x86_64", + "linux": "linux-i686", + "macosx64": "mac", + "win32": "win32", + "win64": "win64", + "linux64-devedition": "linux-x86_64", + "linux-devedition": "linux-i686", + "macosx64-devedition": "mac", + "win32-devedition": "win32", + "win64-devedition": "win64", + } + orig_platform = platform_mapping.get(orig_platform, orig_platform) + platform = platform_mapping.get(platform, platform) + platform_job["worker"]["artifact-map"] = [ + { + "locale": entry["locale"], + "taskId": entry["taskId"], + "paths": _change_platform_in_artifact_map_paths( + entry["paths"], orig_platform, platform + ), + } + for entry in platform_job["worker"]["artifact-map"] + ] + + return platform_job |