# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ This transform construct tasks to perform diffs between builds, as defined in kind.yml """ from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import Schema from taskgraph.util.taskcluster import get_artifact_path from voluptuous import Any, Optional, Required from gecko_taskgraph.transforms.task import task_description_schema index_or_string = Any( str, {Required("index-search"): str}, ) diff_description_schema = Schema( { # Name of the diff task. Required("name"): str, # Treeherder tier. Required("tier"): int, # Treeherder symbol. Required("symbol"): str, # relative path (from config.path) to the file the task was defined in. Optional("job-from"): str, # Original and new builds to compare. Required("original"): index_or_string, Required("new"): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in # taskcluster/ci/diffoscope/kind.yml Optional("args"): str, # Extra arguments to pass to diffoscope, that can be set per job. Optional("extra-args"): str, # Fail the task when differences are detected. Optional("fail-on-diff"): bool, # What artifact to check the differences of. Defaults to target.tar.bz2 # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for # Android. Optional("artifact"): str, # Whether to unpack first. Diffoscope can normally work without unpacking, # but when one needs to --exclude some contents, that doesn't work out well # if said content is packed (e.g. in omni.ja). Optional("unpack"): bool, # Commands to run before performing the diff. Optional("pre-diff-commands"): [str], # Only run the task on a set of projects/branches. Optional("run-on-projects"): task_description_schema["run-on-projects"], Optional("optimization"): task_description_schema["optimization"], } ) transforms = TransformSequence() transforms.add_validate(diff_description_schema) @transforms.add def fill_template(config, tasks): dummy_tasks = {} for task in tasks: name = task["name"] deps = {} urls = {} previous_artifact = None artifact = task.get("artifact") for k in ("original", "new"): value = task[k] if isinstance(value, str): deps[k] = value dep_name = k os_hint = value else: index = value["index-search"] if index not in dummy_tasks: dummy_tasks[index] = { "label": "index-search-" + index, "description": index, "worker-type": "invalid/always-optimized", "run": { "using": "always-optimized", }, "optimization": { "index-search": [index], }, } yield dummy_tasks[index] deps[index] = "index-search-" + index dep_name = index os_hint = index.split(".")[-1] if artifact: pass elif "linux" in os_hint: artifact = "target.tar.bz2" elif "macosx" in os_hint: artifact = "target.dmg" elif "android" in os_hint: artifact = "target.apk" elif "win" in os_hint: artifact = "target.zip" else: raise Exception(f"Cannot figure out the OS for {value!r}") if previous_artifact is not None and previous_artifact != artifact: raise Exception("Cannot compare builds from different OSes") urls[k] = { "artifact-reference": "<{}/{}>".format( dep_name, get_artifact_path(task, artifact) ), } previous_artifact = artifact taskdesc = { "label": "diff-" + name, "description": name, "treeherder": { "symbol": task["symbol"], "platform": "diff/opt", "kind": "other", "tier": task["tier"], }, "worker-type": "b-linux-gcp", "worker": { "docker-image": {"in-tree": "diffoscope"}, "artifacts": [ { "type": "file", "path": f"/builds/worker/{f}", "name": f"public/{f}", } for f in ( "diff.html", "diff.txt", ) ], "env": { "ORIG_URL": urls["original"], "NEW_URL": urls["new"], "DIFFOSCOPE_ARGS": " ".join( task[k] for k in ("args", "extra-args") if k in task ), "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])), }, "max-run-time": 1800, }, "run": { "using": "run-task", "checkout": task.get("unpack", False), "command": "/builds/worker/bin/get_and_diffoscope{}{}".format( " --unpack" if task.get("unpack") else "", " --fail" if task.get("fail-on-diff") else "", ), }, "dependencies": deps, "optimization": task.get("optimization"), } if "run-on-projects" in task: taskdesc["run-on-projects"] = task["run-on-projects"] if artifact.endswith(".dmg"): taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend( [ "linux64-cctools-port", "linux64-libdmg", ] ) yield taskdesc