diff options
Diffstat (limited to 'taskcluster/android_taskgraph/job.py')
| -rw-r--r-- | taskcluster/android_taskgraph/job.py | 208 |
1 files changed, 208 insertions, 0 deletions
diff --git a/taskcluster/android_taskgraph/job.py b/taskcluster/android_taskgraph/job.py new file mode 100644 index 0000000000..520b8788a3 --- /dev/null +++ b/taskcluster/android_taskgraph/job.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from pipes import quote as shell_quote + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from taskgraph.util import path +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Optional, Required + +secret_schema = { + Required("name"): str, + Required("path"): str, + Required("key"): str, + Optional("json"): bool, + Optional("decode"): bool, +} + +dummy_secret_schema = { + Required("content"): str, + Required("path"): str, + Optional("json"): bool, +} + +gradlew_schema = Schema( + { + Required("using"): "gradlew", + Optional("pre-gradlew"): [[str]], + Required("gradlew"): [str], + Optional("post-gradlew"): [[str]], + # Base work directory used to set up the task. + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], + } +) + +run_commands_schema = Schema( + { + Required("using"): "run-commands", + Optional("pre-commands"): [[str]], + Required("commands"): [[taskref_or_string]], + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], + } +) + + +@run_job_using("docker-worker", "run-commands", schema=run_commands_schema) +def configure_run_commands_schema(config, job, taskdesc): + run = job["run"] + pre_commands = run.pop("pre-commands", []) + pre_commands += [ + _generate_dummy_secret_command(secret) + for secret in run.pop("dummy-secrets", []) + ] + pre_commands += [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + + all_commands = pre_commands + run.pop("commands", []) + + run["command"] = _convert_commands_to_string(all_commands) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +@run_job_using("docker-worker", "gradlew", schema=gradlew_schema) +def configure_gradlew(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + + fetches_dir = "/builds/worker/fetches" + topsrc_dir = "/builds/worker/checkouts/gecko" + worker.setdefault("env", {}).update( + { + "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"), + "GRADLE_USER_HOME": path.join( + topsrc_dir, "mobile/android/gradle/dotgradle-online" + ), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + } + ) + worker["env"].setdefault( + "MOZCONFIG", + path.join( + topsrc_dir, + "mobile/android/config/mozconfigs/android-arm/nightly-android-lints", + ), + ) + worker["env"].setdefault( + "MOZ_ANDROID_FAT_AAR_ARCHITECTURES", "armeabi-v7a,arm64-v8a,x86,x86_64" + ) + + dummy_secrets = [ + _generate_dummy_secret_command(secret) + for secret in run.pop("dummy-secrets", []) + ] + secrets = [_generate_secret_command(secret) for secret in run.get("secrets", [])] + worker["env"].update( + { + "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])), + "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets), + "GRADLEW_ARGS": " ".join(run.pop("gradlew")), + "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])), + } + ) + run[ + "command" + ] = "/builds/worker/checkouts/gecko/taskcluster/scripts/builder/build-android.sh" + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +def _generate_secret_command(secret): + secret_command = [ + "/builds/worker/checkouts/gecko/taskcluster/scripts/get-secret.py", + "-s", + secret["name"], + "-k", + secret["key"], + "-f", + secret["path"], + ] + if secret.get("json"): + secret_command.append("--json") + + if secret.get("decode"): + secret_command.append("--decode") + + return secret_command + + +def _generate_dummy_secret_command(secret): + secret_command = [ + "/builds/worker/checkouts/gecko/taskcluster/scripts/write-dummy-secret.py", + "-f", + secret["path"], + "-c", + secret["content"], + ] + if secret.get("json"): + secret_command.append("--json") + + return secret_command + + +def _convert_commands_to_string(commands): + should_artifact_reference = False + should_task_reference = False + + sanitized_commands = [] + for command in commands: + sanitized_parts = [] + for part in command: + if isinstance(part, dict): + if "artifact-reference" in part: + part_string = part["artifact-reference"] + should_artifact_reference = True + elif "task-reference" in part: + part_string = part["task-reference"] + should_task_reference = True + else: + raise ValueError(f"Unsupported dict: {part}") + else: + part_string = part + + sanitized_parts.append(part_string) + sanitized_commands.append(sanitized_parts) + + shell_quoted_commands = [ + " ".join(map(shell_quote, command)) for command in sanitized_commands + ] + full_string_command = " && ".join(shell_quoted_commands) + + if should_artifact_reference and should_task_reference: + raise NotImplementedError( + '"arifact-reference" and "task-reference" cannot be both used' + ) + elif should_artifact_reference: + return {"artifact-reference": full_string_command} + elif should_task_reference: + return {"task-reference": full_string_command} + else: + return full_string_command + + +def _inject_secrets_scopes(run, taskdesc): + secrets = run.pop("secrets", []) + scopes = taskdesc.setdefault("scopes", []) + new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] + new_secret_scopes = list( + set(new_secret_scopes) + ) # Scopes must not have any duplicates + scopes.extend(new_secret_scopes) + + +def _set_run_task_attributes(job): + run = job["run"] + run["cwd"] = "{checkout}" + run["using"] = "run-task" |