Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 270 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py
new file mode 100644
index 0000000000..b480fcad02
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/source_test.py
@@ -0,0 +1,270 @@
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Source-test jobs can run on multiple platforms.  These transforms allow jobs
+with either `platform` or a list of `platforms`, and set the appropriate
+treeherder configuration and attributes for that platform.
+"""
+
+
+import copy
+import os
+
+import taskgraph
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import keymatch
+from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.treeherder import join_symbol, split_symbol
+from voluptuous import Any, Extra, Optional, Required
+
+from gecko_taskgraph.transforms.job import job_description_schema
+from gecko_taskgraph.util.hg import get_json_automationrelevance
+
+source_test_description_schema = Schema(
+    {
+        # most fields are passed directly through as job fields, and are not
+        # repeated here
+        Extra: object,
+        # The platform on which this task runs.  This will be used to set up attributes
+        # (for try selection) and treeherder metadata (for display).  If given as a list,
+        # the job will be "split" into multiple tasks, one with each platform.
+        Required("platform"): Any(str, [str]),
+        # Build labels required for the task. If this key is provided it must
+        # contain a build label for the task platform.
+        # The task will then depend on a build task, and the installer url will be
+        # saved to the GECKO_INSTALLER_URL environment variable.
+        Optional("require-build"): optionally_keyed_by("project", {str: str}),
+        # These fields can be keyed by "platform", and are otherwise identical to
+        # job descriptions.
+        Required("worker-type"): optionally_keyed_by(
+            "platform", job_description_schema["worker-type"]
+        ),
+        Required("worker"): optionally_keyed_by(
+            "platform", job_description_schema["worker"]
+        ),
+        Optional("python-version"): [int],
+        Optional("dependencies"): {
+            k: optionally_keyed_by("platform", v)
+            for k, v in job_description_schema["dependencies"].items()
+        },
+        # A list of artifacts to install from 'fetch' tasks.
+        Optional("fetches"): {
+            str: optionally_keyed_by(
+                "platform", job_description_schema["fetches"][str]
+            ),
+        },
+    }
+)
+
+transforms = TransformSequence()
+
+transforms.add_validate(source_test_description_schema)
+
+
+@transforms.add
+def set_job_name(config, jobs):
+    for job in jobs:
+        if "job-from" in job and job["job-from"] != "kind.yml":
+            from_name = os.path.splitext(job["job-from"])[0]
+            job["name"] = "{}-{}".format(from_name, job["name"])
+        yield job
+
+
+@transforms.add
+def expand_platforms(config, jobs):
+    for job in jobs:
+        if isinstance(job["platform"], str):
+            yield job
+            continue
+
+        for platform in job["platform"]:
+            pjob = copy.deepcopy(job)
+            pjob["platform"] = platform
+
+            if "name" in pjob:
+                pjob["name"] = "{}-{}".format(pjob["name"], platform)
+            else:
+                pjob["label"] = "{}-{}".format(pjob["label"], platform)
+            yield pjob
+
+
+@transforms.add
+def split_python(config, jobs):
+    for job in jobs:
+        key = "python-version"
+        versions = job.pop(key, [])
+        if not versions:
+            yield job
+            continue
+        for version in versions:
+            group = f"py{version}"
+            pyjob = copy.deepcopy(job)
+            if "name" in pyjob:
+                pyjob["name"] += f"-{group}"
+            else:
+                pyjob["label"] += f"-{group}"
+            symbol = split_symbol(pyjob["treeherder"]["symbol"])[1]
+            pyjob["treeherder"]["symbol"] = join_symbol(group, symbol)
+            pyjob["run"][key] = version
+            yield pyjob
+
+
+@transforms.add
+def split_jsshell(config, jobs):
+    all_shells = {"sm": "Spidermonkey", "v8": "Google V8"}
+
+    for job in jobs:
+        if not job["name"].startswith("jsshell"):
+            yield job
+            continue
+
+        test = job.pop("test")
+        for shell in job.get("shell", all_shells.keys()):
+            assert shell in all_shells
+
+            new_job = copy.deepcopy(job)
+            new_job["name"] = "{}-{}".format(new_job["name"], shell)
+            new_job["description"] = "{} on {}".format(
+                new_job["description"], all_shells[shell]
+            )
+            new_job["shell"] = shell
+
+            group = f"js-bench-{shell}"
+            symbol = split_symbol(new_job["treeherder"]["symbol"])[1]
+            new_job["treeherder"]["symbol"] = join_symbol(group, symbol)
+
+            run = new_job["run"]
+            run["mach"] = run["mach"].format(
+                shell=shell, SHELL=shell.upper(), test=test
+            )
+            yield new_job
+
+
+def add_build_dependency(config, job):
+    """
+    Add build dependency to the job and installer_url to env.
+    """
+    key = job["platform"]
+    build_labels = job.pop("require-build", {})
+    matches = keymatch(build_labels, key)
+    if not matches:
+        raise Exception(
+            "No build platform found. "
+            "Define 'require-build' for {} in the task config.".format(key)
+        )
+
+    if len(matches) > 1:
+        raise Exception(f"More than one build platform found for '{key}'.")
+
+    label = matches[0]
+    deps = job.setdefault("dependencies", {})
+    deps.update({"build": label})
+
+
+@transforms.add
+def handle_platform(config, jobs):
+    """
+    Handle the 'platform' property, setting up treeherder context as well as
+    try-related attributes.
+    """
+    fields = [
+        "always-target",
+        "fetches.toolchain",
+        "require-build",
+        "worker-type",
+        "worker",
+    ]
+
+    for job in jobs:
+        platform = job["platform"]
+
+        for field in fields:
+            resolve_keyed_by(
+                job, field, item_name=job["name"], project=config.params["project"]
+            )
+        for field in job.get("dependencies", {}):
+            resolve_keyed_by(
+                job,
+                f"dependencies.{field}",
+                item_name=job["name"],
+                project=config.params["project"],
+            )
+
+        if "treeherder" in job:
+            job["treeherder"].setdefault("platform", platform)
+
+        if "require-build" in job:
+            add_build_dependency(config, job)
+
+        del job["platform"]
+        yield job
+
+
+@transforms.add
+def handle_shell(config, jobs):
+    """
+    Handle the 'shell' property.
+    """
+    fields = [
+        "run-on-projects",
+        "worker.env",
+    ]
+
+    for job in jobs:
+        if not job.get("shell"):
+            yield job
+            continue
+
+        for field in fields:
+            resolve_keyed_by(job, field, item_name=job["name"])
+
+        del job["shell"]
+        yield job
+
+
+@transforms.add
+def set_code_review_env(config, jobs):
+    """
+    Add a CODE_REVIEW environment variable when running in code-review bot mode
+    """
+    is_code_review = config.params["target_tasks_method"] == "codereview"
+
+    for job in jobs:
+        attrs = job.get("attributes", {})
+        if is_code_review and attrs.get("code-review") is True:
+            env = job["worker"].setdefault("env", {})
+            env["CODE_REVIEW"] = "1"
+
+        yield job
+
+
+@transforms.add
+def set_base_revision_in_tgdiff(config, jobs):
+    # Don't attempt to download 'json-automation' locally as the revision may
+    # not exist in the repository.
+    if not os.environ.get("MOZ_AUTOMATION") or taskgraph.fast:
+        yield from jobs
+        return
+
+    data = get_json_automationrelevance(
+        config.params["head_repository"], config.params["head_rev"]
+    )
+    for job in jobs:
+        if job["name"] != "taskgraph-diff":
+            yield job
+            continue
+
+        job["run"]["command-context"] = {
+            "base_rev": data["changesets"][0]["parents"][0]
+        }
+        yield job
+
+
+@transforms.add
+def set_worker_exit_code(config, jobs):
+    for job in jobs:
+        worker = job["worker"]
+        worker.setdefault("retry-exit-status", [])
+        if 137 not in worker["retry-exit-status"]:
+            worker["retry-exit-status"].append(137)
+        yield job