summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/transforms/diffoscope.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /taskcluster/gecko_taskgraph/transforms/diffoscope.py
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/diffoscope.py')
-rw-r--r--taskcluster/gecko_taskgraph/transforms/diffoscope.py172
1 files changed, 172 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
new file mode 100644
index 0000000000..b74dc5bb8f
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
@@ -0,0 +1,172 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This transform construct tasks to perform diffs between builds, as
+defined in kind.yml
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.taskcluster import get_artifact_path
+from voluptuous import Any, Optional, Required
+
+from gecko_taskgraph.transforms.task import task_description_schema
+
+index_or_string = Any(
+ str,
+ {Required("index-search"): str},
+)
+
+diff_description_schema = Schema(
+ {
+ # Name of the diff task.
+ Required("name"): str,
+ # Treeherder tier.
+ Required("tier"): int,
+ # Treeherder symbol.
+ Required("symbol"): str,
+ # relative path (from config.path) to the file the task was defined in.
+ Optional("job-from"): str,
+ # Original and new builds to compare.
+ Required("original"): index_or_string,
+ Required("new"): index_or_string,
+ # Arguments to pass to diffoscope, used for job-defaults in
+ # taskcluster/ci/diffoscope/kind.yml
+ Optional("args"): str,
+ # Extra arguments to pass to diffoscope, that can be set per job.
+ Optional("extra-args"): str,
+ # Fail the task when differences are detected.
+ Optional("fail-on-diff"): bool,
+ # What artifact to check the differences of. Defaults to target.tar.bz2
+ # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for
+ # Android.
+ Optional("artifact"): str,
+ # Whether to unpack first. Diffoscope can normally work without unpacking,
+ # but when one needs to --exclude some contents, that doesn't work out well
+ # if said content is packed (e.g. in omni.ja).
+ Optional("unpack"): bool,
+ # Commands to run before performing the diff.
+ Optional("pre-diff-commands"): [str],
+ # Only run the task on a set of projects/branches.
+ Optional("run-on-projects"): task_description_schema["run-on-projects"],
+ Optional("optimization"): task_description_schema["optimization"],
+ }
+)
+
+transforms = TransformSequence()
+transforms.add_validate(diff_description_schema)
+
+
+@transforms.add
+def fill_template(config, tasks):
+ dummy_tasks = {}
+
+ for task in tasks:
+ name = task["name"]
+
+ deps = {}
+ urls = {}
+ previous_artifact = None
+ artifact = task.get("artifact")
+ for k in ("original", "new"):
+ value = task[k]
+ if isinstance(value, str):
+ deps[k] = value
+ dep_name = k
+ os_hint = value
+ else:
+ index = value["index-search"]
+ if index not in dummy_tasks:
+ dummy_tasks[index] = {
+ "label": "index-search-" + index,
+ "description": index,
+ "worker-type": "invalid/always-optimized",
+ "run": {
+ "using": "always-optimized",
+ },
+ "optimization": {
+ "index-search": [index],
+ },
+ }
+ yield dummy_tasks[index]
+ deps[index] = "index-search-" + index
+ dep_name = index
+ os_hint = index.split(".")[-1]
+ if artifact:
+ pass
+ elif "linux" in os_hint:
+ artifact = "target.tar.bz2"
+ elif "macosx" in os_hint:
+ artifact = "target.dmg"
+ elif "android" in os_hint:
+ artifact = "target.apk"
+ elif "win" in os_hint:
+ artifact = "target.zip"
+ else:
+ raise Exception(f"Cannot figure out the OS for {value!r}")
+ if previous_artifact is not None and previous_artifact != artifact:
+ raise Exception("Cannot compare builds from different OSes")
+ urls[k] = {
+ "artifact-reference": "<{}/{}>".format(
+ dep_name, get_artifact_path(task, artifact)
+ ),
+ }
+ previous_artifact = artifact
+
+ taskdesc = {
+ "label": "diff-" + name,
+ "description": name,
+ "treeherder": {
+ "symbol": task["symbol"],
+ "platform": "diff/opt",
+ "kind": "other",
+ "tier": task["tier"],
+ },
+ "worker-type": "b-linux-gcp",
+ "worker": {
+ "docker-image": {"in-tree": "diffoscope"},
+ "artifacts": [
+ {
+ "type": "file",
+ "path": f"/builds/worker/{f}",
+ "name": f"public/{f}",
+ }
+ for f in (
+ "diff.html",
+ "diff.txt",
+ )
+ ],
+ "env": {
+ "ORIG_URL": urls["original"],
+ "NEW_URL": urls["new"],
+ "DIFFOSCOPE_ARGS": " ".join(
+ task[k] for k in ("args", "extra-args") if k in task
+ ),
+ "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])),
+ },
+ "max-run-time": 1800,
+ },
+ "run": {
+ "using": "run-task",
+ "checkout": task.get("unpack", False),
+ "command": "/builds/worker/bin/get_and_diffoscope{}{}".format(
+ " --unpack" if task.get("unpack") else "",
+ " --fail" if task.get("fail-on-diff") else "",
+ ),
+ },
+ "dependencies": deps,
+ "optimization": task.get("optimization"),
+ }
+ if "run-on-projects" in task:
+ taskdesc["run-on-projects"] = task["run-on-projects"]
+
+ if artifact.endswith(".dmg"):
+ taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend(
+ [
+ "linux64-cctools-port",
+ "linux64-libdmg",
+ ]
+ )
+
+ yield taskdesc