Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 172 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
new file mode 100644
index 0000000000..b74dc5bb8f
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
@@ -0,0 +1,172 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This transform construct tasks to perform diffs between builds, as
+defined in kind.yml
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.taskcluster import get_artifact_path
+from voluptuous import Any, Optional, Required
+
+from gecko_taskgraph.transforms.task import task_description_schema
+
+index_or_string = Any(
+    str,
+    {Required("index-search"): str},
+)
+
+diff_description_schema = Schema(
+    {
+        # Name of the diff task.
+        Required("name"): str,
+        # Treeherder tier.
+        Required("tier"): int,
+        # Treeherder symbol.
+        Required("symbol"): str,
+        # relative path (from config.path) to the file the task was defined in.
+        Optional("job-from"): str,
+        # Original and new builds to compare.
+        Required("original"): index_or_string,
+        Required("new"): index_or_string,
+        # Arguments to pass to diffoscope, used for job-defaults in
+        # taskcluster/ci/diffoscope/kind.yml
+        Optional("args"): str,
+        # Extra arguments to pass to diffoscope, that can be set per job.
+        Optional("extra-args"): str,
+        # Fail the task when differences are detected.
+        Optional("fail-on-diff"): bool,
+        # What artifact to check the differences of. Defaults to target.tar.bz2
+        # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for
+        # Android.
+        Optional("artifact"): str,
+        # Whether to unpack first. Diffoscope can normally work without unpacking,
+        # but when one needs to --exclude some contents, that doesn't work out well
+        # if said content is packed (e.g. in omni.ja).
+        Optional("unpack"): bool,
+        # Commands to run before performing the diff.
+        Optional("pre-diff-commands"): [str],
+        # Only run the task on a set of projects/branches.
+        Optional("run-on-projects"): task_description_schema["run-on-projects"],
+        Optional("optimization"): task_description_schema["optimization"],
+    }
+)
+
+transforms = TransformSequence()
+transforms.add_validate(diff_description_schema)
+
+
+@transforms.add
+def fill_template(config, tasks):
+    dummy_tasks = {}
+
+    for task in tasks:
+        name = task["name"]
+
+        deps = {}
+        urls = {}
+        previous_artifact = None
+        artifact = task.get("artifact")
+        for k in ("original", "new"):
+            value = task[k]
+            if isinstance(value, str):
+                deps[k] = value
+                dep_name = k
+                os_hint = value
+            else:
+                index = value["index-search"]
+                if index not in dummy_tasks:
+                    dummy_tasks[index] = {
+                        "label": "index-search-" + index,
+                        "description": index,
+                        "worker-type": "invalid/always-optimized",
+                        "run": {
+                            "using": "always-optimized",
+                        },
+                        "optimization": {
+                            "index-search": [index],
+                        },
+                    }
+                    yield dummy_tasks[index]
+                deps[index] = "index-search-" + index
+                dep_name = index
+                os_hint = index.split(".")[-1]
+            if artifact:
+                pass
+            elif "linux" in os_hint:
+                artifact = "target.tar.bz2"
+            elif "macosx" in os_hint:
+                artifact = "target.dmg"
+            elif "android" in os_hint:
+                artifact = "target.apk"
+            elif "win" in os_hint:
+                artifact = "target.zip"
+            else:
+                raise Exception(f"Cannot figure out the OS for {value!r}")
+            if previous_artifact is not None and previous_artifact != artifact:
+                raise Exception("Cannot compare builds from different OSes")
+            urls[k] = {
+                "artifact-reference": "<{}/{}>".format(
+                    dep_name, get_artifact_path(task, artifact)
+                ),
+            }
+            previous_artifact = artifact
+
+        taskdesc = {
+            "label": "diff-" + name,
+            "description": name,
+            "treeherder": {
+                "symbol": task["symbol"],
+                "platform": "diff/opt",
+                "kind": "other",
+                "tier": task["tier"],
+            },
+            "worker-type": "b-linux-gcp",
+            "worker": {
+                "docker-image": {"in-tree": "diffoscope"},
+                "artifacts": [
+                    {
+                        "type": "file",
+                        "path": f"/builds/worker/{f}",
+                        "name": f"public/{f}",
+                    }
+                    for f in (
+                        "diff.html",
+                        "diff.txt",
+                    )
+                ],
+                "env": {
+                    "ORIG_URL": urls["original"],
+                    "NEW_URL": urls["new"],
+                    "DIFFOSCOPE_ARGS": " ".join(
+                        task[k] for k in ("args", "extra-args") if k in task
+                    ),
+                    "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])),
+                },
+                "max-run-time": 1800,
+            },
+            "run": {
+                "using": "run-task",
+                "checkout": task.get("unpack", False),
+                "command": "/builds/worker/bin/get_and_diffoscope{}{}".format(
+                    " --unpack" if task.get("unpack") else "",
+                    " --fail" if task.get("fail-on-diff") else "",
+                ),
+            },
+            "dependencies": deps,
+            "optimization": task.get("optimization"),
+        }
+        if "run-on-projects" in task:
+            taskdesc["run-on-projects"] = task["run-on-projects"]
+
+        if artifact.endswith(".dmg"):
+            taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend(
+                [
+                    "linux64-cctools-port",
+                    "linux64-libdmg",
+                ]
+            )
+
+        yield taskdesc