diff options
Diffstat (limited to 'taskcluster/android_taskgraph')
26 files changed, 2408 insertions, 0 deletions
diff --git a/taskcluster/android_taskgraph/__init__.py b/taskcluster/android_taskgraph/__init__.py new file mode 100644 index 0000000000..116e7d7637 --- /dev/null +++ b/taskcluster/android_taskgraph/__init__.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +from importlib import import_module + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.realpath( + os.path.join(CURRENT_DIR, "..", "..", "mobile", "android") +) +ANDROID_COMPONENTS_DIR = os.path.join(PROJECT_DIR, "android-components") +FOCUS_DIR = os.path.join(PROJECT_DIR, "focus-android") +FENIX_DIR = os.path.join(PROJECT_DIR, "fenix") + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules( + [ + "job", + "parameters", + "target_tasks", + "util.group_by", + "worker_types", + ] + ) + + +def _import_modules(modules): + for module in modules: + import_module(f".{module}", package=__name__) diff --git a/taskcluster/android_taskgraph/build_config.py b/taskcluster/android_taskgraph/build_config.py new file mode 100644 index 0000000000..368b251e41 --- /dev/null +++ b/taskcluster/android_taskgraph/build_config.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +import yaml +from taskgraph.util.memoize import memoize + +from android_taskgraph import ANDROID_COMPONENTS_DIR, FENIX_DIR, FOCUS_DIR + +EXTENSIONS = { + "aar": (".aar", ".pom", "-sources.jar"), + "jar": (".jar", ".pom", "-sources.jar"), +} +CHECKSUMS_EXTENSIONS = (".md5", ".sha1", ".sha256", ".sha512") + + +def get_components(): + build_config = _read_build_config(ANDROID_COMPONENTS_DIR) + return [ + {"name": name, "path": project["path"], "shouldPublish": project["publish"]} + for (name, project) in build_config["projects"].items() + ] + + +def get_path(component): + return _read_build_config(ANDROID_COMPONENTS_DIR)["projects"][component]["path"] + + +def get_extensions(component): + artifact_type = _read_build_config(ANDROID_COMPONENTS_DIR)["projects"][ + component + ].get("artifact-type", "aar") + if artifact_type not in EXTENSIONS: + raise ValueError( + "For '{}', 'artifact-type' must be one of {}".format( + component, repr(EXTENSIONS.keys()) + ) + ) + + return [ + extension + checksum_extension + for extension in EXTENSIONS[artifact_type] + for checksum_extension in ("",) + CHECKSUMS_EXTENSIONS + ] + + +@memoize +def _read_build_config(root_dir): + with open(os.path.join(root_dir, ".buildconfig.yml"), "rb") as f: + return yaml.safe_load(f) + + +def get_apk_based_projects(): + return [ + { + "name": "focus", + "path": FOCUS_DIR, + }, + { + "name": "fenix", + "path": FENIX_DIR, + }, + ] + + +def get_variant(build_type, build_name): + all_variants = _get_all_variants() + matching_variants = [ + variant + for variant in all_variants + if variant["build_type"] == build_type and variant["name"] == build_name + ] + number_of_matching_variants = len(matching_variants) + if number_of_matching_variants == 0: + raise ValueError('No variant found for build type "{}"'.format(build_type)) + elif number_of_matching_variants > 1: + raise ValueError( + 'Too many variants found for build type "{}"": {}'.format( + build_type, matching_variants + ) + ) + + return matching_variants.pop() + + +def _get_all_variants(): + all_variants_including_duplicates = ( + _read_build_config(FOCUS_DIR)["variants"] + + _read_build_config(FENIX_DIR)["variants"] + ) + all_unique_variants = [] + for variant in all_variants_including_duplicates: + if ( + # androidTest is a special case that can't be prefixed with fenix or focus. + # Hence, this variant exist in both build_config and we need to expose it + # once only. + ( + variant["build_type"] != "androidTest" + and variant["name"] != "androidTest" + ) + or variant not in all_unique_variants + ): + all_unique_variants.append(variant) + + return all_unique_variants diff --git a/taskcluster/android_taskgraph/job.py b/taskcluster/android_taskgraph/job.py new file mode 100644 index 0000000000..520b8788a3 --- /dev/null +++ b/taskcluster/android_taskgraph/job.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from pipes import quote as shell_quote + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from taskgraph.util import path +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Optional, Required + +secret_schema = { + Required("name"): str, + Required("path"): str, + Required("key"): str, + Optional("json"): bool, + Optional("decode"): bool, +} + +dummy_secret_schema = { + Required("content"): str, + Required("path"): str, + Optional("json"): bool, +} + +gradlew_schema = Schema( + { + Required("using"): "gradlew", + Optional("pre-gradlew"): [[str]], + Required("gradlew"): [str], + Optional("post-gradlew"): [[str]], + # Base work directory used to set up the task. + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], + } +) + +run_commands_schema = Schema( + { + Required("using"): "run-commands", + Optional("pre-commands"): [[str]], + Required("commands"): [[taskref_or_string]], + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], + } +) + + +@run_job_using("docker-worker", "run-commands", schema=run_commands_schema) +def configure_run_commands_schema(config, job, taskdesc): + run = job["run"] + pre_commands = run.pop("pre-commands", []) + pre_commands += [ + _generate_dummy_secret_command(secret) + for secret in run.pop("dummy-secrets", []) + ] + pre_commands += [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + + all_commands = pre_commands + run.pop("commands", []) + + run["command"] = _convert_commands_to_string(all_commands) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +@run_job_using("docker-worker", "gradlew", schema=gradlew_schema) +def configure_gradlew(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + + fetches_dir = "/builds/worker/fetches" + topsrc_dir = "/builds/worker/checkouts/gecko" + worker.setdefault("env", {}).update( + { + "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"), + "GRADLE_USER_HOME": path.join( + topsrc_dir, "mobile/android/gradle/dotgradle-online" + ), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + } + ) + worker["env"].setdefault( + "MOZCONFIG", + path.join( + topsrc_dir, + "mobile/android/config/mozconfigs/android-arm/nightly-android-lints", + ), + ) + worker["env"].setdefault( + "MOZ_ANDROID_FAT_AAR_ARCHITECTURES", "armeabi-v7a,arm64-v8a,x86,x86_64" + ) + + dummy_secrets = [ + _generate_dummy_secret_command(secret) + for secret in run.pop("dummy-secrets", []) + ] + secrets = [_generate_secret_command(secret) for secret in run.get("secrets", [])] + worker["env"].update( + { + "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])), + "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets), + "GRADLEW_ARGS": " ".join(run.pop("gradlew")), + "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])), + } + ) + run[ + "command" + ] = "/builds/worker/checkouts/gecko/taskcluster/scripts/builder/build-android.sh" + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +def _generate_secret_command(secret): + secret_command = [ + "/builds/worker/checkouts/gecko/taskcluster/scripts/get-secret.py", + "-s", + secret["name"], + "-k", + secret["key"], + "-f", + secret["path"], + ] + if secret.get("json"): + secret_command.append("--json") + + if secret.get("decode"): + secret_command.append("--decode") + + return secret_command + + +def _generate_dummy_secret_command(secret): + secret_command = [ + "/builds/worker/checkouts/gecko/taskcluster/scripts/write-dummy-secret.py", + "-f", + secret["path"], + "-c", + secret["content"], + ] + if secret.get("json"): + secret_command.append("--json") + + return secret_command + + +def _convert_commands_to_string(commands): + should_artifact_reference = False + should_task_reference = False + + sanitized_commands = [] + for command in commands: + sanitized_parts = [] + for part in command: + if isinstance(part, dict): + if "artifact-reference" in part: + part_string = part["artifact-reference"] + should_artifact_reference = True + elif "task-reference" in part: + part_string = part["task-reference"] + should_task_reference = True + else: + raise ValueError(f"Unsupported dict: {part}") + else: + part_string = part + + sanitized_parts.append(part_string) + sanitized_commands.append(sanitized_parts) + + shell_quoted_commands = [ + " ".join(map(shell_quote, command)) for command in sanitized_commands + ] + full_string_command = " && ".join(shell_quoted_commands) + + if should_artifact_reference and should_task_reference: + raise NotImplementedError( + '"arifact-reference" and "task-reference" cannot be both used' + ) + elif should_artifact_reference: + return {"artifact-reference": full_string_command} + elif should_task_reference: + return {"task-reference": full_string_command} + else: + return full_string_command + + +def _inject_secrets_scopes(run, taskdesc): + secrets = run.pop("secrets", []) + scopes = taskdesc.setdefault("scopes", []) + new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] + new_secret_scopes = list( + set(new_secret_scopes) + ) # Scopes must not have any duplicates + scopes.extend(new_secret_scopes) + + +def _set_run_task_attributes(job): + run = job["run"] + run["cwd"] = "{checkout}" + run["using"] = "run-task" diff --git a/taskcluster/android_taskgraph/loader/build_config.py b/taskcluster/android_taskgraph/loader/build_config.py new file mode 100644 index 0000000000..abcdc223b4 --- /dev/null +++ b/taskcluster/android_taskgraph/loader/build_config.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.loader.transform import loader as base_loader +from taskgraph.util.templates import merge + +from ..build_config import get_apk_based_projects, get_components + + +def components_loader(kind, path, config, params, loaded_tasks): + """Loader that yields one task per android-component. + + Android-components are read from android-component/.buildconfig.yml + """ + config["jobs"] = _get_components_tasks(config) + return base_loader(kind, path, config, params, loaded_tasks) + + +def components_and_apks_loader(kind, path, config, params, loaded_tasks): + """Loader that yields one task per android-component and per apk-based project. + + For instance focus-android yields one task. + Config is read from various .buildconfig.yml files. + + Additional tasks can be provided in the kind.yml under the key `jobs`. + """ + + components_tasks = _get_components_tasks(config, for_build_type="regular") + apks_tasks = _get_apks_tasks(config) + config["jobs"] = merge(config["jobs"], components_tasks, apks_tasks) + return base_loader(kind, path, config, params, loaded_tasks) + + +def _get_components_tasks(config, for_build_type=None): + not_for_components = config.get("not-for-components", []) + tasks = { + "{}{}".format( + "" if build_type == "regular" else build_type + "-", component["name"] + ): { + "attributes": { + "build-type": build_type, + "component": component["name"], + } + } + for component in get_components() + for build_type in ("regular", "nightly", "beta", "release") + if ( + component["name"] not in not_for_components + and (component["shouldPublish"] or build_type == "regular") + and (for_build_type is None or build_type == for_build_type) + ) + } + + return tasks + + +def _get_apks_tasks(config): + not_for_apks = config.get("not-for-apks", []) + tasks = { + apk["name"]: {} + for apk in get_apk_based_projects() + if apk["name"] not in not_for_apks + } + return tasks diff --git a/taskcluster/android_taskgraph/manifests/apk_releases.yml b/taskcluster/android_taskgraph/manifests/apk_releases.yml new file mode 100644 index 0000000000..6184290bff --- /dev/null +++ b/taskcluster/android_taskgraph/manifests/apk_releases.yml @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacts that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + by-build-type: + fenix-nightly: + - pub/fenix/nightly + fenix-(release|beta): + - pub/fenix/releases + focus-nightly: + - pub/focus/nightly + focus-beta|focus-release|klar-release: + - pub/focus/releases +default_locales: + - multi +tasktype_map: + signing-apk: signing + signing-bundle: signing +platform_names: + product: + by-platform: + focus.*: focus + fenix.*: fenix + klar.*: klar + +default: &default + all_locales: true + description: "TO_BE_OVERRIDDEN" + # Hard coded 'multi' locale + locale_prefix: '${locale}' + source_path_modifier: + by-locale: + default: '${locale}' + multi: '' + checksums_path: "TODO" + +mapping: + target.arm64-v8a.apk: + <<: *default + description: "Android package for arm64-v8a" + pretty_name: ${product}-${version}.${locale}.android-arm64-v8a.apk + destinations: + - ${folder_prefix}${product}-${version}-android-arm64-v8a + from: + - signing-apk + target.armeabi-v7a.apk: + <<: *default + description: "Android package for armeabi-v7a" + pretty_name: ${product}-${version}.${locale}.android-armeabi-v7a.apk + destinations: + - ${folder_prefix}${product}-${version}-android-armeabi-v7a + from: + - signing-apk + target.x86.apk: + <<: *default + description: "Android package for x86" + pretty_name: ${product}-${version}.${locale}.android-x86.apk + destinations: + - ${folder_prefix}${product}-${version}-android-x86 + from: + - signing-apk + target.x86_64.apk: + <<: *default + description: "Android package for x86_64" + pretty_name: ${product}-${version}.${locale}.android-x86_64.apk + destinations: + - ${folder_prefix}${product}-${version}-android-x86_64 + from: + - signing-apk + target.aab: + <<: *default + description: "Android app bundle" + pretty_name: ${product}-${version}.${locale}.android.aab + destinations: + - ${folder_prefix}${product}-${version}-android + from: + - signing-bundle diff --git a/taskcluster/android_taskgraph/parameters.py b/taskcluster/android_taskgraph/parameters.py new file mode 100644 index 0000000000..a5ed1b7dca --- /dev/null +++ b/taskcluster/android_taskgraph/parameters.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.parameters import extend_parameters_schema +from voluptuous import Any, Required + + +def get_defaults(repo_root): + return { + "next_version": None, + "release_type": "", + } + + +extend_parameters_schema( + { + Required("next_version"): Any(str, None), + Required("release_type"): str, + }, + defaults_fn=get_defaults, +) + + +def get_decision_parameters(graph_config, parameters): + parameters.setdefault("next_version", None) + parameters.setdefault("release_type", "") diff --git a/taskcluster/android_taskgraph/release_type.py b/taskcluster/android_taskgraph/release_type.py new file mode 100644 index 0000000000..2ae4348b0b --- /dev/null +++ b/taskcluster/android_taskgraph/release_type.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def does_task_match_release_type(task, release_type): + if task.attributes.get("build-type", task.attributes.get("release-type")) is None: + return True + return bool( + {task.attributes.get("build-type"), task.attributes.get("release-type")} + & {release_type} + ) diff --git a/taskcluster/android_taskgraph/target_tasks.py b/taskcluster/android_taskgraph/target_tasks.py new file mode 100644 index 0000000000..38b1f7b772 --- /dev/null +++ b/taskcluster/android_taskgraph/target_tasks.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.target_tasks import _target_task + +from android_taskgraph.release_type import does_task_match_release_type + + +@_target_task("promote_android") +def target_tasks_promote(full_task_graph, parameters, graph_config): + return _filter_release_promotion( + full_task_graph, + parameters, + filtered_for_candidates=[], + shipping_phase="promote", + ) + + +@_target_task("push_android") +def target_tasks_push(full_task_graph, parameters, graph_config): + filtered_for_candidates = target_tasks_promote( + full_task_graph, + parameters, + graph_config, + ) + return _filter_release_promotion( + full_task_graph, parameters, filtered_for_candidates, shipping_phase="push" + ) + + +@_target_task("ship_android") +def target_tasks_ship(full_task_graph, parameters, graph_config): + filtered_for_candidates = target_tasks_push( + full_task_graph, + parameters, + graph_config, + ) + return _filter_release_promotion( + full_task_graph, parameters, filtered_for_candidates, shipping_phase="ship" + ) + + +def _filter_release_promotion( + full_task_graph, parameters, filtered_for_candidates, shipping_phase +): + def filter(task, parameters): + # Include promotion tasks; these will be optimized out + if task.label in filtered_for_candidates: + return True + + # Ship geckoview in firefox-android ship graph + if ( + shipping_phase == "ship" + and task.attributes.get("shipping_product") == "fennec" + and task.kind in ("beetmover-geckoview", "upload-symbols") + and parameters["release_product"] == "firefox-android" + ): + return True + + # TODO: get rid of this release_type match + if ( + task.attributes.get("shipping_phase") == shipping_phase + and task.attributes.get("shipping_product") == parameters["release_product"] + and does_task_match_release_type(task, parameters["release_type"]) + ): + return True + + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("screenshots") +def target_tasks_screnshots(full_task_graph, parameters, graph_config): + """Select the set of tasks required to generate screenshots on a real device.""" + + def filter(task, parameters): + return task.attributes.get("screenshots", False) + + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("legacy_api_ui_tests") +def target_tasks_legacy_api_ui_tests(full_task_graph, parameters, graph_config): + """Select the set of tasks required to run select UI tests on other API.""" + + def filter(task, parameters): + return task.attributes.get("legacy", False) + + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] diff --git a/taskcluster/android_taskgraph/transforms/__init__.py b/taskcluster/android_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/__init__.py diff --git a/taskcluster/android_taskgraph/transforms/beetmover.py b/taskcluster/android_taskgraph/transforms/beetmover.py new file mode 100644 index 0000000000..75978a4ec9 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/beetmover.py @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies +from taskgraph.util.schema import resolve_keyed_by + +from .build_components import craft_path_version, get_nightly_version + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "treeherder.symbol", + "worker.bucket", + "worker.beetmover-application-name", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def set_artifact_map(config, tasks): + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + maven_destination = task.pop("maven-destination") + deps = get_dependencies(config, task) + task["worker"]["artifact-map"] = [ + { + "paths": { + artifact_path: { + "destinations": [ + maven_destination.format( + component=task["attributes"]["component"], + version=craft_path_version( + version, + task["attributes"]["build-type"], + nightly_version, + ), + artifact_file_name=os.path.basename(artifact_path), + ) + ] + } + for artifact_path in dep.attributes["artifacts"].values() + }, + "taskId": {"task-reference": f"<{dep.kind}>"}, + } + for dep in deps + ] + + yield task + + +@transforms.add +def add_version(config, tasks): + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + task["worker"]["version"] = craft_path_version( + version, task["attributes"]["build-type"], nightly_version + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/beetmover_android_app.py b/taskcluster/android_taskgraph/transforms/beetmover_android_app.py new file mode 100644 index 0000000000..59ec6651cc --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/beetmover_android_app.py @@ -0,0 +1,113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.task import task_description_schema +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from voluptuous import ALLOW_EXTRA, Optional, Required, Schema + +from android_taskgraph.util.scriptworker import generate_beetmover_artifact_map + +logger = logging.getLogger(__name__) + +beetmover_description_schema = Schema( + { + # unique name to describe this beetmover task, defaults to {dep.label}-beetmover + Required("name"): str, + Required("worker"): {"upstream-artifacts": [dict]}, + # treeherder is allowed here to override any defaults we use for beetmover. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("bucket-scope"): optionally_keyed_by("level", "build-type", str), + }, + extra=ALLOW_EXTRA, +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def make_task_description(config, tasks): + for task in tasks: + attributes = task["attributes"] + + label = "beetmover-{}".format(task["name"]) + description = "Beetmover submission for build type '{build_type}'".format( + build_type=attributes.get("build-type"), + ) + + if task.get("locale"): + attributes["locale"] = task["locale"] + + resolve_keyed_by( + task, + "bucket-scope", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + bucket_scope = task.pop("bucket-scope") + + taskdesc = { + "label": label, + "description": description, + "worker-type": "beetmover-android", + "worker": task["worker"], + "scopes": [ + bucket_scope, + "project:releng:beetmover:action:direct-push-to-bucket", + ], + "dependencies": task["dependencies"], + "attributes": attributes, + "treeherder": task["treeherder"], + } + + yield taskdesc + + +_STAGING_PREFIX = "staging-" + + +def craft_release_properties(config, task): + params = config.params + + return { + "app-name": "fenix", # TODO: Support focus + "app-version": str(params["version"]), + "branch": params["project"], + "build-id": str(params["moz_build_date"]), + "hash-type": "sha512", + "platform": "android", + } + + +@transforms.add +def make_task_worker(config, tasks): + for task in tasks: + locale = task["attributes"].get("locale") + build_type = task["attributes"]["build-type"] + + task["worker"].update( + { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, task), + "artifact-map": generate_beetmover_artifact_map( + config, task, platform=build_type, locale=locale + ), + } + ) + + if locale: + task["worker"]["locale"] = locale + + yield task diff --git a/taskcluster/android_taskgraph/transforms/build_android_app.py b/taskcluster/android_taskgraph/transforms/build_android_app.py new file mode 100644 index 0000000000..67fc210ad5 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/build_android_app.py @@ -0,0 +1,277 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the +build-apk and build-bundle kinds. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util import path + +from android_taskgraph.build_config import get_variant + +transforms = TransformSequence() + + +@transforms.add +def add_common_config(config, tasks): + for task in tasks: + fetches = task.setdefault("fetches", {}) + fetches["toolchain"] = [ + "android-sdk-linux", + "android-gradle-dependencies", + "android-gradle-python-envs", + "linux64-jdk", + ] + fetches["build-fat-aar"] = [ + "target.maven.tar.xz", + {"artifact": "mozconfig", "extract": False}, + ] + + run = task.setdefault("run", {}) + run["using"] = "gradlew" + run["use-caches"] = False + + treeherder = task.setdefault("treeherder", {}) + treeherder["kind"] = "build" + treeherder["tier"] = 1 + + task["worker-type"] = "b-linux-medium-gcp" + + worker = task.setdefault("worker", {}) + worker["docker-image"] = {} + worker["docker-image"]["in-tree"] = "android-components" + worker["max-run-time"] = 7200 + worker["chain-of-trust"] = True + worker.setdefault("env", {}).setdefault( + "MOZCONFIG", "/builds/worker/fetches/mozconfig" + ) + build_fat_aar = config.kind_dependencies_tasks[ + task["dependencies"]["build-fat-aar"] + ] + if build_fat_aar.attributes.get("shippable", False): + worker["env"].setdefault( + "MOZ_UPDATE_CHANNEL", + build_fat_aar.attributes.get( + "update-channel", "nightly-{}".format(config.params["project"]) + ), + ) + + yield task + + +@transforms.add +def add_variant_config(config, tasks): + for task in tasks: + attributes = task.setdefault("attributes", {}) + if not attributes.get("build-type"): + attributes["build-type"] = task["name"] + yield task + + +@transforms.add +def add_shippable_secrets(config, tasks): + for task in tasks: + secrets = task["run"].setdefault("secrets", []) + dummy_secrets = task["run"].setdefault("dummy-secrets", []) + + if ( + task.pop("include-shippable-secrets", False) + and config.params["level"] == "3" + ): + secrets.extend( + [ + { + "key": key, + "name": _get_secret_index(task["name"]), + "path": target_file, + } + for key, target_file in _get_secrets_keys_and_target_files(task) + ] + ) + else: + dummy_secrets.extend( + [ + { + "content": fake_value, + "path": target_file, + } + for fake_value, target_file in ( + ("faketoken", ".adjust_token"), + ("faketoken", ".mls_token"), + ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), + ) + ] + ) + + yield task + + +def _get_secrets_keys_and_target_files(task): + secrets = [ + ("adjust", ".adjust_token"), + ("sentry_dsn", ".sentry_token"), + ("mls", ".mls_token"), + ("nimbus_url", ".nimbus"), + ] + + if task["name"].startswith("fenix-"): + gradle_build_type = task["run"]["gradle-build-type"] + secrets.extend( + [ + ( + "firebase", + "app/src/{}/res/values/firebase.xml".format(gradle_build_type), + ), + ("wallpaper_url", ".wallpaper_url"), + ("pocket_consumer_key", ".pocket_consumer_key"), + ] + ) + + return secrets + + +def _get_secret_index(task_name): + product_name = task_name.split("-")[0] + secret_name = task_name[len(product_name) + 1 :] + secret_project_name = ( + "focus-android" if product_name in ("focus", "klar") else product_name + ) + return f"project/mobile/firefox-android/{secret_project_name}/{secret_name}" + + +@transforms.add +def build_pre_gradle_command(config, tasks): + for task in tasks: + source_project_name = task["source-project-name"] + pre_gradlew = task["run"].setdefault("pre-gradlew", []) + pre_gradlew.append(["cd", path.join("mobile", "android", source_project_name)]) + + yield task + + +@transforms.add +def build_gradle_command(config, tasks): + for task in tasks: + gradle_build_type = task["run"]["gradle-build-type"] + gradle_build_name = task["run"]["gradle-build-name"] + variant_config = get_variant(gradle_build_type, gradle_build_name) + variant_name = variant_config["name"][0].upper() + variant_config["name"][1:] + + package_command = task["run"].pop("gradle-package-command", "assemble") + gradle_command = [ + "clean", + f"{package_command}{variant_name}", + ] + + if task["run"].pop("track-apk-size", False): + gradle_command.append(f"apkSize{variant_name}") + + task["run"]["gradlew"] = gradle_command + yield task + + +@transforms.add +def extra_gradle_options(config, tasks): + for task in tasks: + for extra in task["run"].pop("gradle-extra-options", []): + task["run"]["gradlew"].append(extra) + + yield task + + +@transforms.add +def add_test_build_type(config, tasks): + for task in tasks: + test_build_type = task["run"].pop("test-build-type", "") + if test_build_type: + task["run"]["gradlew"].append(f"-PtestBuildType={test_build_type}") + yield task + + +@transforms.add +def add_disable_optimization(config, tasks): + for task in tasks: + if task.pop("disable-optimization", False): + task["run"]["gradlew"].append("-PdisableOptimization") + yield task + + +@transforms.add +def add_nightly_version(config, tasks): + for task in tasks: + if task.pop("include-nightly-version", False): + task["run"]["gradlew"].extend( + [ + # We only set the `official` flag here. The actual version name will be determined + # by Gradle (depending on the Gecko/A-C version being used) + "-Pofficial" + ] + ) + yield task + + +@transforms.add +def add_release_version(config, tasks): + for task in tasks: + if task.pop("include-release-version", False): + task["run"]["gradlew"].extend( + ["-PversionName={}".format(config.params["version"]), "-Pofficial"] + ) + yield task + + +@transforms.add +def add_artifacts(config, tasks): + for task in tasks: + gradle_build_type = task["run"].pop("gradle-build-type") + gradle_build_name = task["run"].pop("gradle-build-name") + gradle_build = task["run"].pop("gradle-build") + variant_config = get_variant(gradle_build_type, gradle_build_name) + artifacts = task.setdefault("worker", {}).setdefault("artifacts", []) + source_project_name = task.pop("source-project-name") + + task["attributes"]["apks"] = apks = {} + + if "apk-artifact-template" in task: + artifact_template = task.pop("apk-artifact-template") + + for apk in variant_config["apks"]: + apk_name = artifact_template["name"].format( + gradle_build=gradle_build, **apk + ) + artifacts.append( + { + "type": artifact_template["type"], + "name": apk_name, + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + source_project_name=source_project_name, + **apk, + ), + } + ) + apks[apk["abi"]] = { + "name": apk_name, + } + elif "aab-artifact-template" in task: + variant_name = variant_config["name"] + artifact_template = task.pop("aab-artifact-template") + artifacts.append( + { + "type": artifact_template["type"], + "name": artifact_template["name"], + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + source_project_name=source_project_name, + variant_name=variant_name, + ), + } + ) + task["attributes"]["aab"] = artifact_template["name"] + + yield task diff --git a/taskcluster/android_taskgraph/transforms/build_components.py b/taskcluster/android_taskgraph/transforms/build_components.py new file mode 100644 index 0000000000..d4e1f8c508 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/build_components.py @@ -0,0 +1,207 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import datetime + +from mozilla_version.mobile import GeckoVersion +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from ..build_config import get_extensions, get_path + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for field in ( + "include-coverage", + "run-on-projects", + "shipping-phase", + "run.gradlew", + "treeherder.symbol", + "dependencies.build-fat-aar", + ): + resolve_keyed_by( + task, + field, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "component": task["attributes"]["component"], + }, + ) + + yield task + + +@transforms.add +def handle_update_channel(config, tasks): + for task in tasks: + build_fat_aar = config.kind_dependencies_tasks[ + task["dependencies"]["build-fat-aar"] + ] + if build_fat_aar.attributes.get("shippable"): + task["worker"].setdefault("env", {}).setdefault( + "MOZ_UPDATE_CHANNEL", + build_fat_aar.attributes.get("update-channel", "default"), + ) + yield task + + +@transforms.add +def handle_coverage(config, tasks): + for task in tasks: + if task.pop("include-coverage", False): + task["run"]["gradlew"].insert(0, "-Pcoverage") + yield task + + +@transforms.add +def interpolate_missing_values(config, tasks): + timestamp = _get_timestamp(config) + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + for field in ("description", "run.gradlew", "treeherder.symbol"): + component = task["attributes"]["component"] + _deep_format( + task, + field, + component=component, + nightlyVersion=nightly_version, + timestamp=timestamp, + treeherder_group=component[:25], + ) + + yield task + + +def _get_timestamp(config): + push_date_string = config.params["moz_build_date"] + push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S") + return push_date_time.strftime("%Y%m%d.%H%M%S-1") + + +def _get_buildid(config): + return config.params.moz_build_date.strftime("%Y%m%d%H%M%S") + + +def get_nightly_version(config, version): + buildid = _get_buildid(config) + parsed_version = GeckoVersion.parse(version) + return f"{parsed_version.major_number}.{parsed_version.minor_number}.{buildid}" + + +def craft_path_version(version, build_type, nightly_version): + """Helper function to craft the correct version to bake in the artifacts full + path section""" + path_version = version + # XXX: for nightly releases we need to s/X.0.0/X.0.<buildid>/g in versions + if build_type == "nightly": + path_version = path_version.replace(version, nightly_version) + return path_version + + +def _deep_format(object, field, **format_kwargs): + keys = field.split(".") + last_key = keys[-1] + for key in keys[:-1]: + object = object[key] + + one_before_last_object = object + object = object[last_key] + + if isinstance(object, str): + one_before_last_object[last_key] = object.format(**format_kwargs) + elif isinstance(object, list): + one_before_last_object[last_key] = [ + item.format(**format_kwargs) for item in object + ] + else: + raise ValueError(f"Unsupported type for object: {object}") + + +@transforms.add +def add_artifacts(config, tasks): + _get_timestamp(config) + version = config.params["version"] + nightly_version = get_nightly_version(config, version) + + for task in tasks: + artifact_template = task.pop("artifact-template", {}) + task["attributes"]["artifacts"] = artifacts = {} + + component = task["attributes"]["component"] + build_artifact_definitions = task.setdefault("worker", {}).setdefault( + "artifacts", [] + ) + + for key in [ + "tests-artifact-template", + "lint-artifact-template", + "jacoco-coverage-template", + ]: + if key in task: + optional_artifact_template = task.pop(key, {}) + build_artifact_definitions.append( + { + "type": optional_artifact_template["type"], + "name": optional_artifact_template["name"], + "path": optional_artifact_template["path"].format( + component_path=get_path(component) + ), + } + ) + + if artifact_template: + all_extensions = get_extensions(component) + artifact_file_names_per_extension = { + extension: "{component}-{version}{timestamp}{extension}".format( + component=component, + version=version, + timestamp="", + extension=extension, + ) + for extension in all_extensions + } + # XXX: rather than adding more complex logic above, we simply post-adjust the + # dictionary for `nightly` types of graphs + if task["attributes"]["build-type"] == "nightly": + for ext, path in artifact_file_names_per_extension.items(): + if version in path: + artifact_file_names_per_extension[ext] = path.replace( + version, nightly_version + ) + + for ( + extension, + artifact_file_name, + ) in artifact_file_names_per_extension.items(): + artifact_full_name = artifact_template["name"].format( + artifact_file_name=artifact_file_name, + ) + build_artifact_definitions.append( + { + "type": artifact_template["type"], + "name": artifact_full_name, + "path": artifact_template["path"].format( + component_path=get_path(component), + component=component, + version=craft_path_version( + version, + task["attributes"]["build-type"], + nightly_version, + ), + artifact_file_name=artifact_file_name, + ), + } + ) + + artifacts[extension] = artifact_full_name + + yield task diff --git a/taskcluster/android_taskgraph/transforms/chunk.py b/taskcluster/android_taskgraph/transforms/chunk.py new file mode 100644 index 0000000000..bb17a99f4e --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/chunk.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from copy import deepcopy + +from taskgraph import MAX_DEPENDENCIES +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import add_suffix + +# XXX Docker images may be added after this transform, so we allow one more dep to be added +MAX_NUMBER_OF_DEPS = MAX_DEPENDENCIES - 1 + +transforms = TransformSequence() + + +def build_task_definition(orig_task, deps, soft_deps, count): + task = deepcopy(orig_task) + task["dependencies"] = {label: label for label in deps} + task["soft-dependencies"] = list(soft_deps) + task["name"] = "{}-{}".format(orig_task["name"], count) + if "treeherder" in task: + task["treeherder"]["symbol"] = add_suffix( + task["treeherder"]["symbol"], f"-{count}" + ) + + task["attributes"]["is_final_chunked_task"] = False + return task + + +def get_chunked_label(config, chunked_task): + return "{}-{}".format(config.kind, chunked_task["name"]) + + +@transforms.add +def add_dependencies(config, tasks): + for task in tasks: + count = 1 + soft_deps = set() + regular_deps = set() + chunked_labels = set() + + soft_dep_labels = list(task.pop("soft-dependencies", [])) + regular_dep_labels = list(task.get("dependencies", {}).keys()) + # sort for deterministic chunking + all_dep_labels = sorted(set(soft_dep_labels + regular_dep_labels)) + + for dep_label in all_dep_labels: + if dep_label in regular_dep_labels: + regular_deps.add(dep_label) + else: + soft_deps.add(dep_label) + + if len(regular_deps) + len(soft_deps) == MAX_NUMBER_OF_DEPS: + chunked_task = build_task_definition( + task, regular_deps, soft_deps, count + ) + chunked_label = get_chunked_label(config, chunked_task) + chunked_labels.add(chunked_label) + yield chunked_task + soft_deps.clear() + regular_deps.clear() + count += 1 + + if regular_deps or soft_deps: + chunked_task = build_task_definition(task, regular_deps, soft_deps, count) + chunked_label = get_chunked_label(config, chunked_task) + chunked_labels.add(chunked_label) + yield chunked_task + + task["dependencies"] = {label: label for label in chunked_labels} + # Chunk yields a last task that doesn't have a number appended to it. + # It helps configuring Github which waits on a single label. + # Setting this attribute also enables multi_dep to select the right + # task to depend on. + task["attributes"]["is_final_chunked_task"] = True + yield task diff --git a/taskcluster/android_taskgraph/transforms/notify.py b/taskcluster/android_taskgraph/transforms/notify.py new file mode 100644 index 0000000000..20fea476e3 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/notify.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("notifications.message", "notifications.emails"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def add_notify_email(config, tasks): + for task in tasks: + notify = task.pop("notify", {}) + email_config = notify.get("email") + if email_config: + extra = task.setdefault("extra", {}) + notify = extra.setdefault("notify", {}) + notify["email"] = { + "content": email_config["content"], + "subject": email_config["subject"], + "link": email_config.get("link", None), + } + + routes = task.setdefault("routes", []) + routes.extend( + [ + "notify.email.{}.on-{}".format(address, reason) + for address in email_config["to-addresses"] + for reason in email_config["on-reasons"] + ] + ) + + yield task diff --git a/taskcluster/android_taskgraph/transforms/post_dummy.py b/taskcluster/android_taskgraph/transforms/post_dummy.py new file mode 100644 index 0000000000..3a5e9c1f50 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/post_dummy.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def set_name_and_clear_artifacts(config, tasks): + for task in tasks: + task["name"] = task["attributes"]["build-type"] + task["attributes"]["artifacts"] = {} + yield task + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + resolve_keyed_by( + task, + "treeherder.symbol", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + } + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/push_android_app.py b/taskcluster/android_taskgraph/transforms/push_android_app.py new file mode 100644 index 0000000000..bca9b3c8de --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/push_android_app.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "worker.channel", + "worker.dep", + "worker.certificate-alias", + "worker.product", + "routes", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def add_startup_test(config, tasks): + for task in tasks: + if "nightly" not in task["attributes"].get("build-type", ""): + yield task + continue + for dep_label, dep_task in config.kind_dependencies_tasks.items(): + if ( + dep_task.kind == "android-startup-test" + and dep_task.attributes["shipping-product"] + == task["attributes"]["shipping-product"] + ): + task["dependencies"]["android-startup-test"] = dep_label + yield task diff --git a/taskcluster/android_taskgraph/transforms/signing.py b/taskcluster/android_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..9aeec6a043 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/signing.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from ..build_config import CHECKSUMS_EXTENSIONS + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "index", + "worker-type", + "treeherder.symbol", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + } + ) + yield task + + +@transforms.add +def set_signing_attributes(config, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def filter_out_checksums(config, tasks): + for task in tasks: + task["attributes"]["artifacts"] = { + extension: path + for extension, path in task["attributes"]["artifacts"].items() + if not any(map(path.endswith, CHECKSUMS_EXTENSIONS)) + } + + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["paths"] = [ + path + for path in upstream_artifact["paths"] + if not any(map(path.endswith, CHECKSUMS_EXTENSIONS)) + ] + + yield task + + +_DETACHED_SIGNATURE_EXTENSION = ".asc" + + +@transforms.add +def set_detached_signature_artifacts(config, tasks): + for task in tasks: + task["attributes"]["artifacts"] = { + extension + + _DETACHED_SIGNATURE_EXTENSION: path + + _DETACHED_SIGNATURE_EXTENSION + for extension, path in task["attributes"]["artifacts"].items() + } + + yield task + + +@transforms.add +def set_signing_format(config, tasks): + for task in tasks: + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = ["autograph_gpg"] + + yield task + + +@transforms.add +def add_signing_cert_scope(config, tasks): + signing_cert_scope = get_signing_cert_scope(config) + for task in tasks: + task.setdefault("scopes", []).append(signing_cert_scope) + yield task diff --git a/taskcluster/android_taskgraph/transforms/signing_android_app.py b/taskcluster/android_taskgraph/transforms/signing_android_app.py new file mode 100644 index 0000000000..aa2cbbb5b7 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/signing_android_app.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the +APK and AAB signing kinds. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + +PRODUCTION_SIGNING_BUILD_TYPES = [ + "focus-nightly", + "focus-beta", + "focus-release", + "klar-release", + "fenix-nightly", + "fenix-beta", + "fenix-release", + "fenix-beta-mozillaonline", + "fenix-release-mozillaonline", +] + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "index", + "signing-format", + "notify", + "treeherder.platform", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def set_worker_type(config, tasks): + for task in tasks: + worker_type = "linux-depsigning" + if ( + str(config.params["level"]) == "3" + and task["attributes"]["build-type"] in PRODUCTION_SIGNING_BUILD_TYPES + ): + worker_type = "linux-signing" + task["worker-type"] = worker_type + yield task + + +@transforms.add +def add_signing_cert_scope(config, tasks): + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + for task in tasks: + cert = "dep-signing" + if str(config.params["level"]) == "3": + if task["attributes"]["build-type"] in ("fenix-beta", "fenix-release"): + cert = "fennec-production-signing" + elif task["attributes"]["build-type"] in PRODUCTION_SIGNING_BUILD_TYPES: + cert = "production-signing" + task.setdefault("scopes", []).append(f"{scope_prefix}:signing:cert:{cert}") + yield task + + +@transforms.add +def set_index_job_name(config, tasks): + for task in tasks: + if task.get("index"): + task["index"]["job-name"] = task["attributes"]["build-type"] + yield task + + +@transforms.add +def set_signing_attributes(config, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def set_signing_format(config, tasks): + for task in tasks: + signing_format = task.pop("signing-format") + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = [signing_format] + yield task + + +@transforms.add +def format_email(config, tasks): + version = config.params["version"] + + for task in tasks: + if "notify" in task: + email = task["notify"].get("email") + if email: + email["subject"] = email["subject"].format(version=version) + email["content"] = email["content"].format(version=version) + + yield task diff --git a/taskcluster/android_taskgraph/transforms/treeherder.py b/taskcluster/android_taskgraph/transforms/treeherder.py new file mode 100644 index 0000000000..d0a1eede07 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/treeherder.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies, get_primary_dependency +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + if not task["attributes"].get("build-type"): + task["attributes"]["build-type"] = task["name"] + + resolve_keyed_by( + task, + "treeherder.symbol", + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def build_treeherder_definition(config, tasks): + for task in tasks: + primary_dep = get_primary_dependency(config, task) + if not primary_dep and task.get("primary-dependency"): + primary_dep = task.pop("primary-dependency") + + elif not primary_dep: + deps = list(get_dependencies(config, task)) or list( + task["dependent-tasks"].values() + ) + primary_dep = deps[0] + + task.setdefault("treeherder", {}).update( + inherit_treeherder_from_dep(task, primary_dep) + ) + task_group = primary_dep.task["extra"]["treeherder"].get("groupSymbol", "?") + job_symbol = task["treeherder"].pop("symbol") + full_symbol = join_symbol(task_group, job_symbol) + task["treeherder"]["symbol"] = full_symbol + + yield task diff --git a/taskcluster/android_taskgraph/transforms/ui_tests.py b/taskcluster/android_taskgraph/transforms/ui_tests.py new file mode 100644 index 0000000000..0e447b76d4 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/ui_tests.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +_ANDROID_TASK_NAME_PREFIX = "android-" + + +@transforms.add +def set_component_attribute(config, tasks): + for task in tasks: + component_name = task.pop("component", None) + if not component_name: + task_name = task["name"] + if task_name.startswith(_ANDROID_TASK_NAME_PREFIX): + component_name = task_name[len(_ANDROID_TASK_NAME_PREFIX) :] + else: + raise NotImplementedError( + f"Cannot determine component name from task {task_name}" + ) + + attributes = task.setdefault("attributes", {}) + attributes["component"] = component_name + + yield task + + +@transforms.add +def define_ui_test_command_line(config, tasks): + for task in tasks: + run = task.setdefault("run", {}) + post_gradlew = run.setdefault("post-gradlew", []) + post_gradlew.append( + [ + "automation/taskcluster/androidTest/ui-test.sh", + task["attributes"]["component"], + "arm", + "1", + ] + ) + + yield task + + +@transforms.add +def define_treeherder_symbol(config, tasks): + for task in tasks: + treeherder = task.setdefault("treeherder") + treeherder.setdefault("symbol", f"{task['attributes']['component']}(unit)") + + yield task + + +@transforms.add +def define_description(config, tasks): + for task in tasks: + task.setdefault( + "description", + f"Run unit/ui tests on device for {task['attributes']['component']}", + ) + yield task diff --git a/taskcluster/android_taskgraph/transforms/upstream_artifacts.py b/taskcluster/android_taskgraph/transforms/upstream_artifacts.py new file mode 100644 index 0000000000..895719d411 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/upstream_artifacts.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_dependencies + +from android_taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts + +transforms = TransformSequence() + + +def _get_task_type(dep_kind): + if dep_kind.startswith("build-"): + return "build" + elif dep_kind.startswith("signing-"): + return "signing" + return dep_kind + + +@transforms.add +def build_upstream_artifacts(config, tasks): + for task in tasks: + worker_definition = { + "upstream-artifacts": [], + } + if "artifact_map" in task["attributes"]: + # Beetmover-apk tasks use declarative artifacts. + locale = task["attributes"].get("locale") + build_type = task["attributes"]["build-type"] + worker_definition[ + "upstream-artifacts" + ] = generate_beetmover_upstream_artifacts(config, task, build_type, locale) + else: + for dep in get_dependencies(config, task): + paths = list(dep.attributes.get("artifacts", {}).values()) + paths.extend( + [ + apk_metadata["name"] + for apk_metadata in dep.attributes.get("apks", {}).values() + ] + ) + if dep.attributes.get("aab"): + paths.extend([dep.attributes.get("aab")]) + if paths: + worker_definition["upstream-artifacts"].append( + { + "taskId": {"task-reference": f"<{dep.kind}>"}, + "taskType": _get_task_type(dep.kind), + "paths": sorted(paths), + } + ) + + task.setdefault("worker", {}).update(worker_definition) + yield task diff --git a/taskcluster/android_taskgraph/util/__init__.py b/taskcluster/android_taskgraph/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/android_taskgraph/util/__init__.py diff --git a/taskcluster/android_taskgraph/util/group_by.py b/taskcluster/android_taskgraph/util/group_by.py new file mode 100644 index 0000000000..3c3b10980c --- /dev/null +++ b/taskcluster/android_taskgraph/util/group_by.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.util.dependencies import group_by + + +@group_by("component") +def component_grouping(config, tasks): + groups = {} + for task in tasks: + component = task.attributes.get("component") + if component == "all": + continue + + build_type = task.attributes.get("build-type") + groups.setdefault((component, build_type), []).append(task) + + tasks_for_all_components = [ + task + for task in tasks + if task.attributes.get("component") == "all" + # We just want to depend on the task that waits on all chunks. This way + # we have a single dependency for that kind + and task.attributes.get("is_final_chunked_task", True) + ] + for (_, build_type), tasks in groups.items(): + tasks.extend( + [ + task + for task in tasks_for_all_components + if task.attributes.get("build-type") == build_type + ] + ) + + return groups.values() + + +@group_by("build-type") +def build_type_grouping(config, tasks): + groups = {} + for task in tasks: + # We just want to depend on the task that waits on all chunks. This way + # we have a single dependency for that kind + if not task.attributes.get("is_final_chunked_task", True): + continue + + build_type = task.attributes.get("build-type") + groups.setdefault(build_type, []).append(task) + + return groups.values() diff --git a/taskcluster/android_taskgraph/util/scriptworker.py b/taskcluster/android_taskgraph/util/scriptworker.py new file mode 100644 index 0000000000..e438f19d26 --- /dev/null +++ b/taskcluster/android_taskgraph/util/scriptworker.py @@ -0,0 +1,283 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import itertools +import os +from copy import deepcopy +from datetime import datetime + +import jsone +from taskgraph.util.memoize import memoize +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.yaml import load_yaml + +cached_load_yaml = memoize(load_yaml) + + +def generate_beetmover_upstream_artifacts( + config, job, platform, locale=None, dependencies=None, **kwargs +): + """Generate the upstream artifacts for beetmover, using the artifact map. + + Currently only applies to beetmover tasks. + + Args: + job (dict): The current job being generated + dependencies (list): A list of the job's dependency labels. + platform (str): The current build platform + locale (str): The current locale being beetmoved. + + Returns: + list: A list of dictionaries conforming to the upstream_artifacts spec. + """ + base_artifact_prefix = get_artifact_prefix(job) + resolve_keyed_by( + job, + "attributes.artifact_map", + "artifact map", + **{ + "release-type": config.params["release_type"], + "platform": platform, + }, + ) + map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) + upstream_artifacts = list() + + if not locale: + locales = map_config["default_locales"] + elif isinstance(locale, list): + locales = locale + else: + locales = [locale] + + if not dependencies: + if job.get("dependencies"): + dependencies = job["dependencies"].keys() + elif job.get("primary-dependency"): + dependencies = [job["primary-dependency"].kind] + else: + raise Exception("Unsupported type of dependency. Got job: {}".format(job)) + + for locale, dep in itertools.product(locales, dependencies): + paths = list() + + for filename in map_config["mapping"]: + if dep not in map_config["mapping"][filename]["from"]: + continue + if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: + continue + if ( + "only_for_platforms" in map_config["mapping"][filename] + and platform + not in map_config["mapping"][filename]["only_for_platforms"] + ): + continue + if ( + "not_for_platforms" in map_config["mapping"][filename] + and platform in map_config["mapping"][filename]["not_for_platforms"] + ): + continue + if "partials_only" in map_config["mapping"][filename]: + continue + # The next time we look at this file it might be a different locale. + file_config = deepcopy(map_config["mapping"][filename]) + resolve_keyed_by( + file_config, + "source_path_modifier", + "source path modifier", + locale=locale, + ) + + kwargs["locale"] = locale + + paths.append( + os.path.join( + base_artifact_prefix, + jsone.render(file_config["source_path_modifier"], kwargs), + jsone.render(filename, kwargs), + ) + ) + + if job.get("dependencies") and getattr( + job["dependencies"][dep], "release_artifacts", None + ): + paths = [ + path + for path in paths + if path in job["dependencies"][dep].release_artifacts + ] + + if not paths: + continue + + upstream_artifacts.append( + { + "taskId": {"task-reference": "<{}>".format(dep)}, + "taskType": map_config["tasktype_map"].get(dep), + "paths": sorted(paths), + "locale": locale, + } + ) + + upstream_artifacts.sort(key=lambda u: u["paths"]) + return upstream_artifacts + + +def generate_beetmover_artifact_map(config, job, **kwargs): + """Generate the beetmover artifact map. + + Currently only applies to beetmover tasks. + + Args: + config (): Current taskgraph configuration. + job (dict): The current job being generated + Common kwargs: + platform (str): The current build platform + locale (str): The current locale being beetmoved. + + Returns: + list: A list of dictionaries containing source->destination + maps for beetmover. + """ + platform = kwargs.get("platform", "") + resolve_keyed_by( + job, + "attributes.artifact_map", + job["label"], + **{ + "release-type": config.params["release_type"], + "platform": platform, + }, + ) + map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) + base_artifact_prefix = map_config.get( + "base_artifact_prefix", get_artifact_prefix(job) + ) + + artifacts = list() + + dependencies = job["dependencies"].keys() + + if kwargs.get("locale"): + if isinstance(kwargs["locale"], list): + locales = kwargs["locale"] + else: + locales = [kwargs["locale"]] + else: + locales = map_config["default_locales"] + + resolve_keyed_by( + map_config, + "s3_bucket_paths", + job["label"], + **{"build-type": job["attributes"]["build-type"]}, + ) + + for locale, dep in sorted(itertools.product(locales, dependencies)): + paths = dict() + for filename in map_config["mapping"]: + # Relevancy checks + if dep not in map_config["mapping"][filename]["from"]: + # We don't get this file from this dependency. + continue + if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: + # This locale either doesn't produce or shouldn't upload this file. + continue + if ( + "only_for_platforms" in map_config["mapping"][filename] + and platform + not in map_config["mapping"][filename]["only_for_platforms"] + ): + # This platform either doesn't produce or shouldn't upload this file. + continue + if ( + "not_for_platforms" in map_config["mapping"][filename] + and platform in map_config["mapping"][filename]["not_for_platforms"] + ): + # This platform either doesn't produce or shouldn't upload this file. + continue + if "partials_only" in map_config["mapping"][filename]: + continue + + # deepcopy because the next time we look at this file the locale will differ. + file_config = deepcopy(map_config["mapping"][filename]) + + for field in [ + "destinations", + "locale_prefix", + "source_path_modifier", + "update_balrog_manifest", + "pretty_name", + "checksums_path", + ]: + resolve_keyed_by(file_config, field, job["label"], locale=locale) + + # This format string should ideally be in the configuration file, + # but this would mean keeping variable names in sync between code + config. + destinations = [ + "{s3_bucket_path}/{dest_path}/{filename}".format( + s3_bucket_path=bucket_path, + dest_path=dest_path, + filename=file_config.get("pretty_name", filename), + ) + for dest_path, bucket_path in itertools.product( + file_config["destinations"], map_config["s3_bucket_paths"] + ) + ] + # Creating map entries + # Key must be artifact path, to avoid trampling duplicates, such + # as public/build/target.apk and public/build/multi/target.apk + key = os.path.join( + base_artifact_prefix, + file_config["source_path_modifier"], + filename, + ) + + paths[key] = { + "destinations": destinations, + } + if file_config.get("checksums_path"): + paths[key]["checksums_path"] = file_config["checksums_path"] + + # optional flag: balrog manifest + if file_config.get("update_balrog_manifest"): + paths[key]["update_balrog_manifest"] = True + if file_config.get("balrog_format"): + paths[key]["balrog_format"] = file_config["balrog_format"] + + if not paths: + # No files for this dependency/locale combination. + continue + + # Render all variables for the artifact map + platforms = deepcopy(map_config.get("platform_names", {})) + if platform: + for key in platforms.keys(): + resolve_keyed_by(platforms, key, job["label"], platform=platform) + + version = config.params["version"] + upload_date = datetime.fromtimestamp(config.params["build_date"]) + + if "nightly" in job["attributes"].get("build-type", ""): + folder_prefix = upload_date.strftime("%Y/%m/%Y-%m-%d-%H-%M-%S-") + # TODO: Remove this when version.txt has versioning fixed + version = version.split("-")[0] + else: + folder_prefix = f"{version}/android/" + + kwargs.update( + {"locale": locale, "version": version, "folder_prefix": folder_prefix} + ) + kwargs.update(**platforms) + paths = jsone.render(paths, kwargs) + artifacts.append( + { + "taskId": {"task-reference": "<{}>".format(dep)}, + "locale": locale, + "paths": paths, + } + ) + + return artifacts diff --git a/taskcluster/android_taskgraph/worker_types.py b/taskcluster/android_taskgraph/worker_types.py new file mode 100644 index 0000000000..98b08cb7e7 --- /dev/null +++ b/taskcluster/android_taskgraph/worker_types.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.transforms.task import payload_builder +from taskgraph.util.schema import taskref_or_string +from voluptuous import Any, Optional, Required + + +@payload_builder( + "scriptworker-beetmover", + schema={ + Required("action"): str, + Required("version"): str, + Required("artifact-map"): [ + { + Required("paths"): { + Any(str): { + Required("destinations"): [str], + }, + }, + Required("taskId"): taskref_or_string, + } + ], + Required("beetmover-application-name"): str, + Required("bucket"): str, + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + }, +) +def build_scriptworker_beetmover_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + # Needed by beetmover-scriptworker + for map_ in worker["artifact-map"]: + map_["locale"] = "en-US" + for path_config in map_["paths"].values(): + path_config["checksums_path"] = "" + + task_def["payload"] = { + "artifactMap": worker["artifact-map"], + "releaseProperties": {"appName": worker.pop("beetmover-application-name")}, + "upstreamArtifacts": worker["upstream-artifacts"], + "version": worker["version"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].extend( + [ + "{}:beetmover:action:{}".format(scope_prefix, worker["action"]), + "{}:beetmover:bucket:{}".format(scope_prefix, worker["bucket"]), + ] + ) + + +@payload_builder( + "scriptworker-pushapk", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + Required("certificate-alias"): str, + Required("channel"): str, + Required("commit"): bool, + Required("product"): str, + Required("dep"): bool, + }, +) +def build_push_apk_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "certificate_alias": worker["certificate-alias"], + "channel": worker["channel"], + "commit": worker["commit"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].append( + "{}:googleplay:product:{}{}".format( + scope_prefix, worker["product"], ":dep" if worker["dep"] else "" + ) + ) + + +@payload_builder( + "scriptworker-shipit", + schema={ + Required("release-name"): str, + }, +) +def build_shipit_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = {"release_name": worker["release-name"]} + + +@payload_builder( + "scriptworker-tree", + schema={ + Optional("upstream-artifacts"): [ + { + Optional("taskId"): taskref_or_string, + Optional("taskType"): str, + Optional("paths"): [str], + } + ], + Required("bump"): bool, + Optional("bump-files"): [str], + Optional("push"): bool, + Optional("branch"): str, + }, +) +def build_version_bump_payload(config, task, task_def): + worker = task["worker"] + task_def["tags"]["worker-implementation"] = "scriptworker" + + scopes = task_def.setdefault("scopes", []) + scope_prefix = f"project:mobile:{config.params['project']}:treescript:action" + task_def["payload"] = {} + + if worker["bump"]: + if not worker["bump-files"]: + raise Exception("Version Bump requested without bump-files") + + bump_info = {} + bump_info["next_version"] = config.params["next_version"] + bump_info["files"] = worker["bump-files"] + task_def["payload"]["version_bump_info"] = bump_info + scopes.append(f"{scope_prefix}:version_bump") + + if worker["push"]: + task_def["payload"]["push"] = True + + if worker.get("force-dry-run"): + task_def["payload"]["dry_run"] = True + + if worker.get("branch"): + task_def["payload"]["branch"] = worker["branch"] |