diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/diffoscope.py')
| -rw-r--r-- | taskcluster/gecko_taskgraph/transforms/diffoscope.py | 172 |
1 files changed, 172 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py new file mode 100644 index 0000000000..b74dc5bb8f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -0,0 +1,172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform construct tasks to perform diffs between builds, as +defined in kind.yml +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from taskgraph.util.taskcluster import get_artifact_path +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema + +index_or_string = Any( + str, + {Required("index-search"): str}, +) + +diff_description_schema = Schema( + { + # Name of the diff task. + Required("name"): str, + # Treeherder tier. + Required("tier"): int, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the task was defined in. + Optional("job-from"): str, + # Original and new builds to compare. + Required("original"): index_or_string, + Required("new"): index_or_string, + # Arguments to pass to diffoscope, used for job-defaults in + # taskcluster/ci/diffoscope/kind.yml + Optional("args"): str, + # Extra arguments to pass to diffoscope, that can be set per job. + Optional("extra-args"): str, + # Fail the task when differences are detected. + Optional("fail-on-diff"): bool, + # What artifact to check the differences of. Defaults to target.tar.bz2 + # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for + # Android. + Optional("artifact"): str, + # Whether to unpack first. Diffoscope can normally work without unpacking, + # but when one needs to --exclude some contents, that doesn't work out well + # if said content is packed (e.g. in omni.ja). + Optional("unpack"): bool, + # Commands to run before performing the diff. + Optional("pre-diff-commands"): [str], + # Only run the task on a set of projects/branches. + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("optimization"): task_description_schema["optimization"], + } +) + +transforms = TransformSequence() +transforms.add_validate(diff_description_schema) + + +@transforms.add +def fill_template(config, tasks): + dummy_tasks = {} + + for task in tasks: + name = task["name"] + + deps = {} + urls = {} + previous_artifact = None + artifact = task.get("artifact") + for k in ("original", "new"): + value = task[k] + if isinstance(value, str): + deps[k] = value + dep_name = k + os_hint = value + else: + index = value["index-search"] + if index not in dummy_tasks: + dummy_tasks[index] = { + "label": "index-search-" + index, + "description": index, + "worker-type": "invalid/always-optimized", + "run": { + "using": "always-optimized", + }, + "optimization": { + "index-search": [index], + }, + } + yield dummy_tasks[index] + deps[index] = "index-search-" + index + dep_name = index + os_hint = index.split(".")[-1] + if artifact: + pass + elif "linux" in os_hint: + artifact = "target.tar.bz2" + elif "macosx" in os_hint: + artifact = "target.dmg" + elif "android" in os_hint: + artifact = "target.apk" + elif "win" in os_hint: + artifact = "target.zip" + else: + raise Exception(f"Cannot figure out the OS for {value!r}") + if previous_artifact is not None and previous_artifact != artifact: + raise Exception("Cannot compare builds from different OSes") + urls[k] = { + "artifact-reference": "<{}/{}>".format( + dep_name, get_artifact_path(task, artifact) + ), + } + previous_artifact = artifact + + taskdesc = { + "label": "diff-" + name, + "description": name, + "treeherder": { + "symbol": task["symbol"], + "platform": "diff/opt", + "kind": "other", + "tier": task["tier"], + }, + "worker-type": "b-linux-gcp", + "worker": { + "docker-image": {"in-tree": "diffoscope"}, + "artifacts": [ + { + "type": "file", + "path": f"/builds/worker/{f}", + "name": f"public/{f}", + } + for f in ( + "diff.html", + "diff.txt", + ) + ], + "env": { + "ORIG_URL": urls["original"], + "NEW_URL": urls["new"], + "DIFFOSCOPE_ARGS": " ".join( + task[k] for k in ("args", "extra-args") if k in task + ), + "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])), + }, + "max-run-time": 1800, + }, + "run": { + "using": "run-task", + "checkout": task.get("unpack", False), + "command": "/builds/worker/bin/get_and_diffoscope{}{}".format( + " --unpack" if task.get("unpack") else "", + " --fail" if task.get("fail-on-diff") else "", + ), + }, + "dependencies": deps, + "optimization": task.get("optimization"), + } + if "run-on-projects" in task: + taskdesc["run-on-projects"] = task["run-on-projects"] + + if artifact.endswith(".dmg"): + taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend( + [ + "linux64-cctools-port", + "linux64-libdmg", + ] + ) + + yield taskdesc |