diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/docker_image.py')
| -rw-r--r-- | taskcluster/gecko_taskgraph/transforms/docker_image.py | 209 |
1 files changed, 209 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/docker_image.py b/taskcluster/gecko_taskgraph/transforms/docker_image.py new file mode 100644 index 0000000000..0ccf83bf38 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/docker_image.py @@ -0,0 +1,209 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import os +import re + +import mozpack.path as mozpath +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from voluptuous import Optional, Required + +from gecko_taskgraph.util.docker import ( + create_context_tar, + generate_context_hash, + image_path, +) + +from .. import GECKO +from .task import task_description_schema + +logger = logging.getLogger(__name__) + +CONTEXTS_DIR = "docker-contexts" + +DIGEST_RE = re.compile("^[0-9a-f]{64}$") + +IMAGE_BUILDER_IMAGE = ( + "mozillareleases/image_builder:5.0.0" + "@sha256:" + "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f" +) + +transforms = TransformSequence() + +docker_image_schema = Schema( + { + # Name of the docker image. + Required("name"): str, + # Name of the parent docker image. + Optional("parent"): str, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the docker image was defined + # in. + Optional("job-from"): str, + # Arguments to use for the Dockerfile. + Optional("args"): {str: str}, + # Name of the docker image definition under taskcluster/docker, when + # different from the docker image name. + Optional("definition"): str, + # List of package tasks this docker image depends on. + Optional("packages"): [str], + Optional( + "index", + description="information for indexing this build so its artifacts can be discovered", + ): task_description_schema["index"], + Optional( + "cache", + description="Whether this image should be cached based on inputs.", + ): bool, + } +) + + +transforms.add_validate(docker_image_schema) + + +@transforms.add +def fill_template(config, tasks): + if not taskgraph.fast and config.write_artifacts: + if not os.path.isdir(CONTEXTS_DIR): + os.makedirs(CONTEXTS_DIR) + + for task in tasks: + image_name = task.pop("name") + job_symbol = task.pop("symbol") + args = task.pop("args", {}) + packages = task.pop("packages", []) + parent = task.pop("parent", None) + + for p in packages: + if f"packages-{p}" not in config.kind_dependencies_tasks: + raise Exception( + "Missing package job for {}-{}: {}".format( + config.kind, image_name, p + ) + ) + + if not taskgraph.fast: + context_path = mozpath.relpath(image_path(image_name), GECKO) + if config.write_artifacts: + context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") + logger.info(f"Writing {context_file} for docker image {image_name}") + context_hash = create_context_tar( + GECKO, context_path, context_file, image_name, args + ) + else: + context_hash = generate_context_hash( + GECKO, context_path, image_name, args + ) + else: + if config.write_artifacts: + raise Exception("Can't write artifacts if `taskgraph.fast` is set.") + context_hash = "0" * 40 + digest_data = [context_hash] + digest_data += [json.dumps(args, sort_keys=True)] + + description = "Build the docker image {} for use by dependent tasks".format( + image_name + ) + + args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) + + # Adjust the zstandard compression level based on the execution level. + # We use faster compression for level 1 because we care more about + # end-to-end times. We use slower/better compression for other levels + # because images are read more often and it is worth the trade-off to + # burn more CPU once to reduce image size. + zstd_level = "3" if int(config.params["level"]) == 1 else "10" + + # include some information that is useful in reconstructing this task + # from JSON + taskdesc = { + "label": f"{config.kind}-{image_name}", + "description": description, + "attributes": { + "image_name": image_name, + "artifact_prefix": "public", + }, + "expiration-policy": "long", + "scopes": [], + "treeherder": { + "symbol": job_symbol, + "platform": "taskcluster-images/opt", + "kind": "other", + "tier": 1, + }, + "run-on-projects": [], + "worker-type": "images-gcp", + "worker": { + "implementation": "docker-worker", + "os": "linux", + "artifacts": [ + { + "type": "file", + "path": "/workspace/image.tar.zst", + "name": "public/image.tar.zst", + } + ], + "env": { + "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, + "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format( + image_name + ), + "HASH": context_hash, + "PROJECT": config.params["project"], + "IMAGE_NAME": image_name, + "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, + "DOCKER_BUILD_ARGS": {"task-reference": json.dumps(args)}, + "GECKO_BASE_REPOSITORY": config.params["base_repository"], + "GECKO_HEAD_REPOSITORY": config.params["head_repository"], + "GECKO_HEAD_REV": config.params["head_rev"], + }, + "chain-of-trust": True, + "max-run-time": 7200, + # FIXME: We aren't currently propagating the exit code + }, + } + # Retry for 'funsize-update-generator' if exit status code is -1 + if image_name in ["funsize-update-generator"]: + taskdesc["worker"]["retry-exit-status"] = [-1] + + worker = taskdesc["worker"] + + if image_name == "image_builder": + worker["docker-image"] = IMAGE_BUILDER_IMAGE + digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") + else: + worker["docker-image"] = {"in-tree": "image_builder"} + deps = taskdesc.setdefault("dependencies", {}) + deps["docker-image"] = f"{config.kind}-image_builder" + + if packages: + deps = taskdesc.setdefault("dependencies", {}) + for p in sorted(packages): + deps[p] = f"packages-{p}" + + if parent: + deps = taskdesc.setdefault("dependencies", {}) + deps["parent"] = f"{config.kind}-{parent}" + worker["env"]["PARENT_TASK_ID"] = { + "task-reference": "<parent>", + } + if "index" in task: + taskdesc["index"] = task["index"] + + if task.get("cache", True) and not taskgraph.fast: + taskdesc["cache"] = { + "type": "docker-images.v2", + "name": image_name, + "digest-data": digest_data, + } + + yield taskdesc |