# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from pipes import quote as shell_quote from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using from taskgraph.util import path from taskgraph.util.schema import Schema, taskref_or_string from voluptuous import Optional, Required secret_schema = { Required("name"): str, Required("path"): str, Required("key"): str, Optional("json"): bool, Optional("decode"): bool, } dummy_secret_schema = { Required("content"): str, Required("path"): str, Optional("json"): bool, } gradlew_schema = Schema( { Required("using"): "gradlew", Optional("pre-gradlew"): [[str]], Required("gradlew"): [str], Optional("post-gradlew"): [[str]], # Base work directory used to set up the task. Required("workdir"): str, Optional("use-caches"): bool, Optional("secrets"): [secret_schema], Optional("dummy-secrets"): [dummy_secret_schema], } ) run_commands_schema = Schema( { Required("using"): "run-commands", Optional("pre-commands"): [[str]], Required("commands"): [[taskref_or_string]], Required("workdir"): str, Optional("use-caches"): bool, Optional("secrets"): [secret_schema], Optional("dummy-secrets"): [dummy_secret_schema], } ) @run_job_using("docker-worker", "run-commands", schema=run_commands_schema) def configure_run_commands_schema(config, job, taskdesc): run = job["run"] pre_commands = run.pop("pre-commands", []) pre_commands += [ _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", []) ] pre_commands += [ _generate_secret_command(secret) for secret in run.get("secrets", []) ] all_commands = pre_commands + run.pop("commands", []) run["command"] = _convert_commands_to_string(all_commands) _inject_secrets_scopes(run, taskdesc) _set_run_task_attributes(job) configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) @run_job_using("docker-worker", "gradlew", schema=gradlew_schema) def configure_gradlew(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"] fetches_dir = "/builds/worker/fetches" topsrc_dir = "/builds/worker/checkouts/gecko" worker.setdefault("env", {}).update( { "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"), "GRADLE_USER_HOME": path.join( topsrc_dir, "mobile/android/gradle/dotgradle-online" ), "MOZ_BUILD_DATE": config.params["moz_build_date"], } ) worker["env"].setdefault( "MOZCONFIG", path.join( topsrc_dir, "mobile/android/config/mozconfigs/android-arm/nightly-android-lints", ), ) worker["env"].setdefault( "MOZ_ANDROID_FAT_AAR_ARCHITECTURES", "armeabi-v7a,arm64-v8a,x86,x86_64" ) dummy_secrets = [ _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", []) ] secrets = [_generate_secret_command(secret) for secret in run.get("secrets", [])] worker["env"].update( { "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])), "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets), "GRADLEW_ARGS": " ".join(run.pop("gradlew")), "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])), } ) run[ "command" ] = "/builds/worker/checkouts/gecko/taskcluster/scripts/builder/build-android.sh" _inject_secrets_scopes(run, taskdesc) _set_run_task_attributes(job) configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) def _generate_secret_command(secret): secret_command = [ "/builds/worker/checkouts/gecko/taskcluster/scripts/get-secret.py", "-s", secret["name"], "-k", secret["key"], "-f", secret["path"], ] if secret.get("json"): secret_command.append("--json") if secret.get("decode"): secret_command.append("--decode") return secret_command def _generate_dummy_secret_command(secret): secret_command = [ "/builds/worker/checkouts/gecko/taskcluster/scripts/write-dummy-secret.py", "-f", secret["path"], "-c", secret["content"], ] if secret.get("json"): secret_command.append("--json") return secret_command def _convert_commands_to_string(commands): should_artifact_reference = False should_task_reference = False sanitized_commands = [] for command in commands: sanitized_parts = [] for part in command: if isinstance(part, dict): if "artifact-reference" in part: part_string = part["artifact-reference"] should_artifact_reference = True elif "task-reference" in part: part_string = part["task-reference"] should_task_reference = True else: raise ValueError(f"Unsupported dict: {part}") else: part_string = part sanitized_parts.append(part_string) sanitized_commands.append(sanitized_parts) shell_quoted_commands = [ " ".join(map(shell_quote, command)) for command in sanitized_commands ] full_string_command = " && ".join(shell_quoted_commands) if should_artifact_reference and should_task_reference: raise NotImplementedError( '"arifact-reference" and "task-reference" cannot be both used' ) elif should_artifact_reference: return {"artifact-reference": full_string_command} elif should_task_reference: return {"task-reference": full_string_command} else: return full_string_command def _inject_secrets_scopes(run, taskdesc): secrets = run.pop("secrets", []) scopes = taskdesc.setdefault("scopes", []) new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] new_secret_scopes = list( set(new_secret_scopes) ) # Scopes must not have any duplicates scopes.extend(new_secret_scopes) def _set_run_task_attributes(job): run = job["run"] run["cwd"] = "{checkout}" run["using"] = "run-task"