diff options
Diffstat (limited to 'taskcluster/android_taskgraph/transforms')
14 files changed, 1261 insertions, 0 deletions
diff --git a/taskcluster/android_taskgraph/transforms/__init__.py b/taskcluster/android_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/__init__.py diff --git a/taskcluster/android_taskgraph/transforms/beetmover.py b/taskcluster/android_taskgraph/transforms/beetmover.py new file mode 100644 index 0000000000..75978a4ec9 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/beetmover.py @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies +from taskgraph.util.schema import resolve_keyed_by + +from .build_components import craft_path_version, get_nightly_version + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "treeherder.symbol", + "worker.bucket", + "worker.beetmover-application-name", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def set_artifact_map(config, tasks): + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + maven_destination = task.pop("maven-destination") + deps = get_dependencies(config, task) + task["worker"]["artifact-map"] = [ + { + "paths": { + artifact_path: { + "destinations": [ + maven_destination.format( + component=task["attributes"]["component"], + version=craft_path_version( + version, + task["attributes"]["build-type"], + nightly_version, + ), + artifact_file_name=os.path.basename(artifact_path), + ) + ] + } + for artifact_path in dep.attributes["artifacts"].values() + }, + "taskId": {"task-reference": f"<{dep.kind}>"}, + } + for dep in deps + ] + + yield task + + +@transforms.add +def add_version(config, tasks): + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + task["worker"]["version"] = craft_path_version( + version, task["attributes"]["build-type"], nightly_version + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/beetmover_android_app.py b/taskcluster/android_taskgraph/transforms/beetmover_android_app.py new file mode 100644 index 0000000000..59ec6651cc --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/beetmover_android_app.py @@ -0,0 +1,113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.task import task_description_schema +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from voluptuous import ALLOW_EXTRA, Optional, Required, Schema + +from android_taskgraph.util.scriptworker import generate_beetmover_artifact_map + +logger = logging.getLogger(__name__) + +beetmover_description_schema = Schema( + { + # unique name to describe this beetmover task, defaults to {dep.label}-beetmover + Required("name"): str, + Required("worker"): {"upstream-artifacts": [dict]}, + # treeherder is allowed here to override any defaults we use for beetmover. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("bucket-scope"): optionally_keyed_by("level", "build-type", str), + }, + extra=ALLOW_EXTRA, +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def make_task_description(config, tasks): + for task in tasks: + attributes = task["attributes"] + + label = "beetmover-{}".format(task["name"]) + description = "Beetmover submission for build type '{build_type}'".format( + build_type=attributes.get("build-type"), + ) + + if task.get("locale"): + attributes["locale"] = task["locale"] + + resolve_keyed_by( + task, + "bucket-scope", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + bucket_scope = task.pop("bucket-scope") + + taskdesc = { + "label": label, + "description": description, + "worker-type": "beetmover-android", + "worker": task["worker"], + "scopes": [ + bucket_scope, + "project:releng:beetmover:action:direct-push-to-bucket", + ], + "dependencies": task["dependencies"], + "attributes": attributes, + "treeherder": task["treeherder"], + } + + yield taskdesc + + +_STAGING_PREFIX = "staging-" + + +def craft_release_properties(config, task): + params = config.params + + return { + "app-name": "fenix", # TODO: Support focus + "app-version": str(params["version"]), + "branch": params["project"], + "build-id": str(params["moz_build_date"]), + "hash-type": "sha512", + "platform": "android", + } + + +@transforms.add +def make_task_worker(config, tasks): + for task in tasks: + locale = task["attributes"].get("locale") + build_type = task["attributes"]["build-type"] + + task["worker"].update( + { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, task), + "artifact-map": generate_beetmover_artifact_map( + config, task, platform=build_type, locale=locale + ), + } + ) + + if locale: + task["worker"]["locale"] = locale + + yield task diff --git a/taskcluster/android_taskgraph/transforms/build_android_app.py b/taskcluster/android_taskgraph/transforms/build_android_app.py new file mode 100644 index 0000000000..67fc210ad5 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/build_android_app.py @@ -0,0 +1,277 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the +build-apk and build-bundle kinds. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util import path + +from android_taskgraph.build_config import get_variant + +transforms = TransformSequence() + + +@transforms.add +def add_common_config(config, tasks): + for task in tasks: + fetches = task.setdefault("fetches", {}) + fetches["toolchain"] = [ + "android-sdk-linux", + "android-gradle-dependencies", + "android-gradle-python-envs", + "linux64-jdk", + ] + fetches["build-fat-aar"] = [ + "target.maven.tar.xz", + {"artifact": "mozconfig", "extract": False}, + ] + + run = task.setdefault("run", {}) + run["using"] = "gradlew" + run["use-caches"] = False + + treeherder = task.setdefault("treeherder", {}) + treeherder["kind"] = "build" + treeherder["tier"] = 1 + + task["worker-type"] = "b-linux-medium-gcp" + + worker = task.setdefault("worker", {}) + worker["docker-image"] = {} + worker["docker-image"]["in-tree"] = "android-components" + worker["max-run-time"] = 7200 + worker["chain-of-trust"] = True + worker.setdefault("env", {}).setdefault( + "MOZCONFIG", "/builds/worker/fetches/mozconfig" + ) + build_fat_aar = config.kind_dependencies_tasks[ + task["dependencies"]["build-fat-aar"] + ] + if build_fat_aar.attributes.get("shippable", False): + worker["env"].setdefault( + "MOZ_UPDATE_CHANNEL", + build_fat_aar.attributes.get( + "update-channel", "nightly-{}".format(config.params["project"]) + ), + ) + + yield task + + +@transforms.add +def add_variant_config(config, tasks): + for task in tasks: + attributes = task.setdefault("attributes", {}) + if not attributes.get("build-type"): + attributes["build-type"] = task["name"] + yield task + + +@transforms.add +def add_shippable_secrets(config, tasks): + for task in tasks: + secrets = task["run"].setdefault("secrets", []) + dummy_secrets = task["run"].setdefault("dummy-secrets", []) + + if ( + task.pop("include-shippable-secrets", False) + and config.params["level"] == "3" + ): + secrets.extend( + [ + { + "key": key, + "name": _get_secret_index(task["name"]), + "path": target_file, + } + for key, target_file in _get_secrets_keys_and_target_files(task) + ] + ) + else: + dummy_secrets.extend( + [ + { + "content": fake_value, + "path": target_file, + } + for fake_value, target_file in ( + ("faketoken", ".adjust_token"), + ("faketoken", ".mls_token"), + ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), + ) + ] + ) + + yield task + + +def _get_secrets_keys_and_target_files(task): + secrets = [ + ("adjust", ".adjust_token"), + ("sentry_dsn", ".sentry_token"), + ("mls", ".mls_token"), + ("nimbus_url", ".nimbus"), + ] + + if task["name"].startswith("fenix-"): + gradle_build_type = task["run"]["gradle-build-type"] + secrets.extend( + [ + ( + "firebase", + "app/src/{}/res/values/firebase.xml".format(gradle_build_type), + ), + ("wallpaper_url", ".wallpaper_url"), + ("pocket_consumer_key", ".pocket_consumer_key"), + ] + ) + + return secrets + + +def _get_secret_index(task_name): + product_name = task_name.split("-")[0] + secret_name = task_name[len(product_name) + 1 :] + secret_project_name = ( + "focus-android" if product_name in ("focus", "klar") else product_name + ) + return f"project/mobile/firefox-android/{secret_project_name}/{secret_name}" + + +@transforms.add +def build_pre_gradle_command(config, tasks): + for task in tasks: + source_project_name = task["source-project-name"] + pre_gradlew = task["run"].setdefault("pre-gradlew", []) + pre_gradlew.append(["cd", path.join("mobile", "android", source_project_name)]) + + yield task + + +@transforms.add +def build_gradle_command(config, tasks): + for task in tasks: + gradle_build_type = task["run"]["gradle-build-type"] + gradle_build_name = task["run"]["gradle-build-name"] + variant_config = get_variant(gradle_build_type, gradle_build_name) + variant_name = variant_config["name"][0].upper() + variant_config["name"][1:] + + package_command = task["run"].pop("gradle-package-command", "assemble") + gradle_command = [ + "clean", + f"{package_command}{variant_name}", + ] + + if task["run"].pop("track-apk-size", False): + gradle_command.append(f"apkSize{variant_name}") + + task["run"]["gradlew"] = gradle_command + yield task + + +@transforms.add +def extra_gradle_options(config, tasks): + for task in tasks: + for extra in task["run"].pop("gradle-extra-options", []): + task["run"]["gradlew"].append(extra) + + yield task + + +@transforms.add +def add_test_build_type(config, tasks): + for task in tasks: + test_build_type = task["run"].pop("test-build-type", "") + if test_build_type: + task["run"]["gradlew"].append(f"-PtestBuildType={test_build_type}") + yield task + + +@transforms.add +def add_disable_optimization(config, tasks): + for task in tasks: + if task.pop("disable-optimization", False): + task["run"]["gradlew"].append("-PdisableOptimization") + yield task + + +@transforms.add +def add_nightly_version(config, tasks): + for task in tasks: + if task.pop("include-nightly-version", False): + task["run"]["gradlew"].extend( + [ + # We only set the `official` flag here. The actual version name will be determined + # by Gradle (depending on the Gecko/A-C version being used) + "-Pofficial" + ] + ) + yield task + + +@transforms.add +def add_release_version(config, tasks): + for task in tasks: + if task.pop("include-release-version", False): + task["run"]["gradlew"].extend( + ["-PversionName={}".format(config.params["version"]), "-Pofficial"] + ) + yield task + + +@transforms.add +def add_artifacts(config, tasks): + for task in tasks: + gradle_build_type = task["run"].pop("gradle-build-type") + gradle_build_name = task["run"].pop("gradle-build-name") + gradle_build = task["run"].pop("gradle-build") + variant_config = get_variant(gradle_build_type, gradle_build_name) + artifacts = task.setdefault("worker", {}).setdefault("artifacts", []) + source_project_name = task.pop("source-project-name") + + task["attributes"]["apks"] = apks = {} + + if "apk-artifact-template" in task: + artifact_template = task.pop("apk-artifact-template") + + for apk in variant_config["apks"]: + apk_name = artifact_template["name"].format( + gradle_build=gradle_build, **apk + ) + artifacts.append( + { + "type": artifact_template["type"], + "name": apk_name, + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + source_project_name=source_project_name, + **apk, + ), + } + ) + apks[apk["abi"]] = { + "name": apk_name, + } + elif "aab-artifact-template" in task: + variant_name = variant_config["name"] + artifact_template = task.pop("aab-artifact-template") + artifacts.append( + { + "type": artifact_template["type"], + "name": artifact_template["name"], + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + source_project_name=source_project_name, + variant_name=variant_name, + ), + } + ) + task["attributes"]["aab"] = artifact_template["name"] + + yield task diff --git a/taskcluster/android_taskgraph/transforms/build_components.py b/taskcluster/android_taskgraph/transforms/build_components.py new file mode 100644 index 0000000000..d4e1f8c508 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/build_components.py @@ -0,0 +1,207 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import datetime + +from mozilla_version.mobile import GeckoVersion +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from ..build_config import get_extensions, get_path + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for field in ( + "include-coverage", + "run-on-projects", + "shipping-phase", + "run.gradlew", + "treeherder.symbol", + "dependencies.build-fat-aar", + ): + resolve_keyed_by( + task, + field, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "component": task["attributes"]["component"], + }, + ) + + yield task + + +@transforms.add +def handle_update_channel(config, tasks): + for task in tasks: + build_fat_aar = config.kind_dependencies_tasks[ + task["dependencies"]["build-fat-aar"] + ] + if build_fat_aar.attributes.get("shippable"): + task["worker"].setdefault("env", {}).setdefault( + "MOZ_UPDATE_CHANNEL", + build_fat_aar.attributes.get("update-channel", "default"), + ) + yield task + + +@transforms.add +def handle_coverage(config, tasks): + for task in tasks: + if task.pop("include-coverage", False): + task["run"]["gradlew"].insert(0, "-Pcoverage") + yield task + + +@transforms.add +def interpolate_missing_values(config, tasks): + timestamp = _get_timestamp(config) + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + for field in ("description", "run.gradlew", "treeherder.symbol"): + component = task["attributes"]["component"] + _deep_format( + task, + field, + component=component, + nightlyVersion=nightly_version, + timestamp=timestamp, + treeherder_group=component[:25], + ) + + yield task + + +def _get_timestamp(config): + push_date_string = config.params["moz_build_date"] + push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S") + return push_date_time.strftime("%Y%m%d.%H%M%S-1") + + +def _get_buildid(config): + return config.params.moz_build_date.strftime("%Y%m%d%H%M%S") + + +def get_nightly_version(config, version): + buildid = _get_buildid(config) + parsed_version = GeckoVersion.parse(version) + return f"{parsed_version.major_number}.{parsed_version.minor_number}.{buildid}" + + +def craft_path_version(version, build_type, nightly_version): + """Helper function to craft the correct version to bake in the artifacts full + path section""" + path_version = version + # XXX: for nightly releases we need to s/X.0.0/X.0.<buildid>/g in versions + if build_type == "nightly": + path_version = path_version.replace(version, nightly_version) + return path_version + + +def _deep_format(object, field, **format_kwargs): + keys = field.split(".") + last_key = keys[-1] + for key in keys[:-1]: + object = object[key] + + one_before_last_object = object + object = object[last_key] + + if isinstance(object, str): + one_before_last_object[last_key] = object.format(**format_kwargs) + elif isinstance(object, list): + one_before_last_object[last_key] = [ + item.format(**format_kwargs) for item in object + ] + else: + raise ValueError(f"Unsupported type for object: {object}") + + +@transforms.add +def add_artifacts(config, tasks): + _get_timestamp(config) + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + artifact_template = task.pop("artifact-template", {}) + task["attributes"]["artifacts"] = artifacts = {} + + component = task["attributes"]["component"] + build_artifact_definitions = task.setdefault("worker", {}).setdefault( + "artifacts", [] + ) + + for key in [ + "tests-artifact-template", + "lint-artifact-template", + "jacoco-coverage-template", + ]: + if key in task: + optional_artifact_template = task.pop(key, {}) + build_artifact_definitions.append( + { + "type": optional_artifact_template["type"], + "name": optional_artifact_template["name"], + "path": optional_artifact_template["path"].format( + component_path=get_path(component) + ), + } + ) + + if artifact_template: + all_extensions = get_extensions(component) + artifact_file_names_per_extension = { + extension: "{component}-{version}{timestamp}{extension}".format( + component=component, + version=version, + timestamp="", + extension=extension, + ) + for extension in all_extensions + } + # XXX: rather than adding more complex logic above, we simply post-adjust the + # dictionary for `nightly` types of graphs + if task["attributes"]["build-type"] == "nightly": + for ext, path in artifact_file_names_per_extension.items(): + if version in path: + artifact_file_names_per_extension[ext] = path.replace( + version, nightly_version + ) + + for ( + extension, + artifact_file_name, + ) in artifact_file_names_per_extension.items(): + artifact_full_name = artifact_template["name"].format( + artifact_file_name=artifact_file_name, + ) + build_artifact_definitions.append( + { + "type": artifact_template["type"], + "name": artifact_full_name, + "path": artifact_template["path"].format( + component_path=get_path(component), + component=component, + version=craft_path_version( + version, + task["attributes"]["build-type"], + nightly_version, + ), + artifact_file_name=artifact_file_name, + ), + } + ) + + artifacts[extension] = artifact_full_name + + yield task diff --git a/taskcluster/android_taskgraph/transforms/chunk.py b/taskcluster/android_taskgraph/transforms/chunk.py new file mode 100644 index 0000000000..bb17a99f4e --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/chunk.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from copy import deepcopy + +from taskgraph import MAX_DEPENDENCIES +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import add_suffix + +# XXX Docker images may be added after this transform, so we allow one more dep to be added +MAX_NUMBER_OF_DEPS = MAX_DEPENDENCIES - 1 + +transforms = TransformSequence() + + +def build_task_definition(orig_task, deps, soft_deps, count): + task = deepcopy(orig_task) + task["dependencies"] = {label: label for label in deps} + task["soft-dependencies"] = list(soft_deps) + task["name"] = "{}-{}".format(orig_task["name"], count) + if "treeherder" in task: + task["treeherder"]["symbol"] = add_suffix( + task["treeherder"]["symbol"], f"-{count}" + ) + + task["attributes"]["is_final_chunked_task"] = False + return task + + +def get_chunked_label(config, chunked_task): + return "{}-{}".format(config.kind, chunked_task["name"]) + + +@transforms.add +def add_dependencies(config, tasks): + for task in tasks: + count = 1 + soft_deps = set() + regular_deps = set() + chunked_labels = set() + + soft_dep_labels = list(task.pop("soft-dependencies", [])) + regular_dep_labels = list(task.get("dependencies", {}).keys()) + # sort for deterministic chunking + all_dep_labels = sorted(set(soft_dep_labels + regular_dep_labels)) + + for dep_label in all_dep_labels: + if dep_label in regular_dep_labels: + regular_deps.add(dep_label) + else: + soft_deps.add(dep_label) + + if len(regular_deps) + len(soft_deps) == MAX_NUMBER_OF_DEPS: + chunked_task = build_task_definition( + task, regular_deps, soft_deps, count + ) + chunked_label = get_chunked_label(config, chunked_task) + chunked_labels.add(chunked_label) + yield chunked_task + soft_deps.clear() + regular_deps.clear() + count += 1 + + if regular_deps or soft_deps: + chunked_task = build_task_definition(task, regular_deps, soft_deps, count) + chunked_label = get_chunked_label(config, chunked_task) + chunked_labels.add(chunked_label) + yield chunked_task + + task["dependencies"] = {label: label for label in chunked_labels} + # Chunk yields a last task that doesn't have a number appended to it. + # It helps configuring Github which waits on a single label. + # Setting this attribute also enables multi_dep to select the right + # task to depend on. + task["attributes"]["is_final_chunked_task"] = True + yield task diff --git a/taskcluster/android_taskgraph/transforms/notify.py b/taskcluster/android_taskgraph/transforms/notify.py new file mode 100644 index 0000000000..20fea476e3 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/notify.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("notifications.message", "notifications.emails"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def add_notify_email(config, tasks): + for task in tasks: + notify = task.pop("notify", {}) + email_config = notify.get("email") + if email_config: + extra = task.setdefault("extra", {}) + notify = extra.setdefault("notify", {}) + notify["email"] = { + "content": email_config["content"], + "subject": email_config["subject"], + "link": email_config.get("link", None), + } + + routes = task.setdefault("routes", []) + routes.extend( + [ + "notify.email.{}.on-{}".format(address, reason) + for address in email_config["to-addresses"] + for reason in email_config["on-reasons"] + ] + ) + + yield task diff --git a/taskcluster/android_taskgraph/transforms/post_dummy.py b/taskcluster/android_taskgraph/transforms/post_dummy.py new file mode 100644 index 0000000000..3a5e9c1f50 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/post_dummy.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def set_name_and_clear_artifacts(config, tasks): + for task in tasks: + task["name"] = task["attributes"]["build-type"] + task["attributes"]["artifacts"] = {} + yield task + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + resolve_keyed_by( + task, + "treeherder.symbol", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + } + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/push_android_app.py b/taskcluster/android_taskgraph/transforms/push_android_app.py new file mode 100644 index 0000000000..bca9b3c8de --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/push_android_app.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "worker.channel", + "worker.dep", + "worker.certificate-alias", + "worker.product", + "routes", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def add_startup_test(config, tasks): + for task in tasks: + if "nightly" not in task["attributes"].get("build-type", ""): + yield task + continue + for dep_label, dep_task in config.kind_dependencies_tasks.items(): + if ( + dep_task.kind == "android-startup-test" + and dep_task.attributes["shipping-product"] + == task["attributes"]["shipping-product"] + ): + task["dependencies"]["android-startup-test"] = dep_label + yield task diff --git a/taskcluster/android_taskgraph/transforms/signing.py b/taskcluster/android_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..9aeec6a043 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/signing.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from ..build_config import CHECKSUMS_EXTENSIONS + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "index", + "worker-type", + "treeherder.symbol", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def set_signing_attributes(config, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def filter_out_checksums(config, tasks): + for task in tasks: + task["attributes"]["artifacts"] = { + extension: path + for extension, path in task["attributes"]["artifacts"].items() + if not any(map(path.endswith, CHECKSUMS_EXTENSIONS)) + } + + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["paths"] = [ + path + for path in upstream_artifact["paths"] + if not any(map(path.endswith, CHECKSUMS_EXTENSIONS)) + ] + + yield task + + +_DETACHED_SIGNATURE_EXTENSION = ".asc" + + +@transforms.add +def set_detached_signature_artifacts(config, tasks): + for task in tasks: + task["attributes"]["artifacts"] = { + extension + + _DETACHED_SIGNATURE_EXTENSION: path + + _DETACHED_SIGNATURE_EXTENSION + for extension, path in task["attributes"]["artifacts"].items() + } + + yield task + + +@transforms.add +def set_signing_format(config, tasks): + for task in tasks: + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = ["autograph_gpg"] + + yield task + + +@transforms.add +def add_signing_cert_scope(config, tasks): + signing_cert_scope = get_signing_cert_scope(config) + for task in tasks: + task.setdefault("scopes", []).append(signing_cert_scope) + yield task diff --git a/taskcluster/android_taskgraph/transforms/signing_android_app.py b/taskcluster/android_taskgraph/transforms/signing_android_app.py new file mode 100644 index 0000000000..aa2cbbb5b7 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/signing_android_app.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the +APK and AAB signing kinds. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + +PRODUCTION_SIGNING_BUILD_TYPES = [ + "focus-nightly", + "focus-beta", + "focus-release", + "klar-release", + "fenix-nightly", + "fenix-beta", + "fenix-release", + "fenix-beta-mozillaonline", + "fenix-release-mozillaonline", +] + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "index", + "signing-format", + "notify", + "treeherder.platform", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def set_worker_type(config, tasks): + for task in tasks: + worker_type = "linux-depsigning" + if ( + str(config.params["level"]) == "3" + and task["attributes"]["build-type"] in PRODUCTION_SIGNING_BUILD_TYPES + ): + worker_type = "linux-signing" + task["worker-type"] = worker_type + yield task + + +@transforms.add +def add_signing_cert_scope(config, tasks): + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + for task in tasks: + cert = "dep-signing" + if str(config.params["level"]) == "3": + if task["attributes"]["build-type"] in ("fenix-beta", "fenix-release"): + cert = "fennec-production-signing" + elif task["attributes"]["build-type"] in PRODUCTION_SIGNING_BUILD_TYPES: + cert = "production-signing" + task.setdefault("scopes", []).append(f"{scope_prefix}:signing:cert:{cert}") + yield task + + +@transforms.add +def set_index_job_name(config, tasks): + for task in tasks: + if task.get("index"): + task["index"]["job-name"] = task["attributes"]["build-type"] + yield task + + +@transforms.add +def set_signing_attributes(config, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def set_signing_format(config, tasks): + for task in tasks: + signing_format = task.pop("signing-format") + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = [signing_format] + yield task + + +@transforms.add +def format_email(config, tasks): + version = config.params["version"] + + for task in tasks: + if "notify" in task: + email = task["notify"].get("email") + if email: + email["subject"] = email["subject"].format(version=version) + email["content"] = email["content"].format(version=version) + + yield task diff --git a/taskcluster/android_taskgraph/transforms/treeherder.py b/taskcluster/android_taskgraph/transforms/treeherder.py new file mode 100644 index 0000000000..d0a1eede07 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/treeherder.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies, get_primary_dependency +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + if not task["attributes"].get("build-type"): + task["attributes"]["build-type"] = task["name"] + + resolve_keyed_by( + task, + "treeherder.symbol", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def build_treeherder_definition(config, tasks): + for task in tasks: + primary_dep = get_primary_dependency(config, task) + if not primary_dep and task.get("primary-dependency"): + primary_dep = task.pop("primary-dependency") + + elif not primary_dep: + deps = list(get_dependencies(config, task)) or list( + task["dependent-tasks"].values() + ) + primary_dep = deps[0] + + task.setdefault("treeherder", {}).update( + inherit_treeherder_from_dep(task, primary_dep) + ) + task_group = primary_dep.task["extra"]["treeherder"].get("groupSymbol", "?") + job_symbol = task["treeherder"].pop("symbol") + full_symbol = join_symbol(task_group, job_symbol) + task["treeherder"]["symbol"] = full_symbol + + yield task diff --git a/taskcluster/android_taskgraph/transforms/ui_tests.py b/taskcluster/android_taskgraph/transforms/ui_tests.py new file mode 100644 index 0000000000..0e447b76d4 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/ui_tests.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +_ANDROID_TASK_NAME_PREFIX = "android-" + + +@transforms.add +def set_component_attribute(config, tasks): + for task in tasks: + component_name = task.pop("component", None) + if not component_name: + task_name = task["name"] + if task_name.startswith(_ANDROID_TASK_NAME_PREFIX): + component_name = task_name[len(_ANDROID_TASK_NAME_PREFIX) :] + else: + raise NotImplementedError( + f"Cannot determine component name from task {task_name}" + ) + + attributes = task.setdefault("attributes", {}) + attributes["component"] = component_name + + yield task + + +@transforms.add +def define_ui_test_command_line(config, tasks): + for task in tasks: + run = task.setdefault("run", {}) + post_gradlew = run.setdefault("post-gradlew", []) + post_gradlew.append( + [ + "automation/taskcluster/androidTest/ui-test.sh", + task["attributes"]["component"], + "arm", + "1", + ] + ) + + yield task + + +@transforms.add +def define_treeherder_symbol(config, tasks): + for task in tasks: + treeherder = task.setdefault("treeherder") + treeherder.setdefault("symbol", f"{task['attributes']['component']}(unit)") + + yield task + + +@transforms.add +def define_description(config, tasks): + for task in tasks: + task.setdefault( + "description", + f"Run unit/ui tests on device for {task['attributes']['component']}", + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/upstream_artifacts.py b/taskcluster/android_taskgraph/transforms/upstream_artifacts.py new file mode 100644 index 0000000000..895719d411 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/upstream_artifacts.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies + +from android_taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts + +transforms = TransformSequence() + + +def _get_task_type(dep_kind): + if dep_kind.startswith("build-"): + return "build" + elif dep_kind.startswith("signing-"): + return "signing" + return dep_kind + + +@transforms.add +def build_upstream_artifacts(config, tasks): + for task in tasks: + worker_definition = { + "upstream-artifacts": [], + } + if "artifact_map" in task["attributes"]: + # Beetmover-apk tasks use declarative artifacts. + locale = task["attributes"].get("locale") + build_type = task["attributes"]["build-type"] + worker_definition[ + "upstream-artifacts" + ] = generate_beetmover_upstream_artifacts(config, task, build_type, locale) + else: + for dep in get_dependencies(config, task): + paths = list(dep.attributes.get("artifacts", {}).values()) + paths.extend( + [ + apk_metadata["name"] + for apk_metadata in dep.attributes.get("apks", {}).values() + ] + ) + if dep.attributes.get("aab"): + paths.extend([dep.attributes.get("aab")]) + if paths: + worker_definition["upstream-artifacts"].append( + { + "taskId": {"task-reference": f"<{dep.kind}>"}, + "taskType": _get_task_type(dep.kind), + "paths": sorted(paths), + } + ) + + task.setdefault("worker", {}).update(worker_definition) + yield task |