Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 238 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py
new file mode 100644
index 0000000000..94a4d71b0b
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/build.py
@@ -0,0 +1,238 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+import logging
+
+from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.treeherder import add_suffix
+
+from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level
+from gecko_taskgraph.util.workertypes import worker_type_implementation
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, jobs):
+    """Set defaults, including those that differ per worker implementation"""
+    for job in jobs:
+        job["treeherder"].setdefault("kind", "build")
+        job["treeherder"].setdefault("tier", 1)
+        _, worker_os = worker_type_implementation(
+            config.graph_config, config.params, job["worker-type"]
+        )
+        worker = job.setdefault("worker", {})
+        worker.setdefault("env", {})
+        worker["chain-of-trust"] = True
+        if worker_os == "linux":
+            worker.setdefault("docker-image", {"in-tree": "debian11-amd64-build"})
+
+        yield job
+
+
+@transforms.add
+def stub_installer(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job,
+            "stub-installer",
+            item_name=job["name"],
+            project=config.params["project"],
+            **{
+                "release-type": config.params["release_type"],
+            },
+        )
+        job.setdefault("attributes", {})
+        if job.get("stub-installer"):
+            job["attributes"]["stub-installer"] = job["stub-installer"]
+            job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"})
+        if "stub-installer" in job:
+            del job["stub-installer"]
+        yield job
+
+
+@transforms.add
+def resolve_shipping_product(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job,
+            "shipping-product",
+            item_name=job["name"],
+            **{
+                "release-type": config.params["release_type"],
+            },
+        )
+        yield job
+
+
+@transforms.add
+def update_channel(config, jobs):
+    keys = [
+        "run.update-channel",
+        "run.mar-channel-id",
+        "run.accepted-mar-channel-ids",
+    ]
+    for job in jobs:
+        job["worker"].setdefault("env", {})
+        for key in keys:
+            resolve_keyed_by(
+                job,
+                key,
+                item_name=job["name"],
+                **{
+                    "project": config.params["project"],
+                    "release-type": config.params["release_type"],
+                },
+            )
+        update_channel = job["run"].pop("update-channel", None)
+        if update_channel:
+            job["run"].setdefault("extra-config", {})["update_channel"] = update_channel
+            job["attributes"]["update-channel"] = update_channel
+        mar_channel_id = job["run"].pop("mar-channel-id", None)
+        if mar_channel_id:
+            job["attributes"]["mar-channel-id"] = mar_channel_id
+            job["worker"]["env"]["MAR_CHANNEL_ID"] = mar_channel_id
+        accepted_mar_channel_ids = job["run"].pop("accepted-mar-channel-ids", None)
+        if accepted_mar_channel_ids:
+            job["attributes"]["accepted-mar-channel-ids"] = accepted_mar_channel_ids
+            job["worker"]["env"]["ACCEPTED_MAR_CHANNEL_IDS"] = accepted_mar_channel_ids
+
+        yield job
+
+
+@transforms.add
+def mozconfig(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job,
+            "run.mozconfig-variant",
+            item_name=job["name"],
+            **{
+                "release-type": config.params["release_type"],
+            },
+        )
+        mozconfig_variant = job["run"].pop("mozconfig-variant", None)
+        if mozconfig_variant:
+            job["run"].setdefault("extra-config", {})[
+                "mozconfig_variant"
+            ] = mozconfig_variant
+        yield job
+
+
+@transforms.add
+def use_artifact(config, jobs):
+    if is_try(config.params):
+        use_artifact = config.params["try_task_config"].get(
+            "use-artifact-builds", False
+        )
+    else:
+        use_artifact = False
+    for job in jobs:
+        if (
+            config.kind == "build"
+            and use_artifact
+            and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
+            # If tests aren't packaged, then we are not able to rebuild all the packages
+            and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1"
+        ):
+            job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a")
+            job["worker"]["env"]["USE_ARTIFACT"] = "1"
+            job["attributes"]["artifact-build"] = True
+        yield job
+
+
+@transforms.add
+def use_profile_data(config, jobs):
+    for job in jobs:
+        use_pgo = job.pop("use-pgo", False)
+        disable_pgo = config.params["try_task_config"].get("disable-pgo", False)
+        artifact_build = job["attributes"].get("artifact-build")
+        if not use_pgo or disable_pgo or artifact_build:
+            yield job
+            continue
+
+        # If use_pgo is True, the task uses the generate-profile task of the
+        # same name. Otherwise a task can specify a specific generate-profile
+        # task to use in the use_pgo field.
+        if use_pgo is True:
+            name = job["name"]
+        else:
+            name = use_pgo
+        dependencies = f"generate-profile-{name}"
+        job.setdefault("dependencies", {})["generate-profile"] = dependencies
+        job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"]
+        job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
+
+        _, worker_os = worker_type_implementation(
+            config.graph_config, config.params, job["worker-type"]
+        )
+        if worker_os == "linux":
+            # LTO linkage needs more open files than the default from run-task.
+            job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"})
+
+        if job.get("use-sccache"):
+            raise Exception(
+                "use-sccache is incompatible with use-pgo in {}".format(job["name"])
+            )
+
+        yield job
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job,
+            "use-sccache",
+            item_name=job["name"],
+            **{"release-level": release_level(config.params["project"])},
+        )
+        yield job
+
+
+@transforms.add
+def enable_full_crashsymbols(config, jobs):
+    """Enable full crashsymbols on jobs with
+    'enable-full-crashsymbols' set to True and on release branches, or
+    on try"""
+    branches = RELEASE_PROJECTS | {
+        "toolchains",
+        "try",
+    }
+    for job in jobs:
+        enable_full_crashsymbols = job["attributes"].get("enable-full-crashsymbols")
+        if enable_full_crashsymbols and config.params["project"] in branches:
+            logger.debug("Enabling full symbol generation for %s", job["name"])
+            job["worker"]["env"]["MOZ_ENABLE_FULL_SYMBOLS"] = "1"
+        else:
+            logger.debug("Disabling full symbol generation for %s", job["name"])
+            job["attributes"].pop("enable-full-crashsymbols", None)
+        yield job
+
+
+@transforms.add
+def set_expiry(config, jobs):
+    for job in jobs:
+        attributes = job["attributes"]
+        if (
+            "shippable" in attributes
+            and attributes["shippable"]
+            and config.kind
+            in {
+                "build",
+            }
+        ):
+            expiration_policy = "long"
+        else:
+            expiration_policy = "medium"
+
+        job["expiration-policy"] = expiration_policy
+        yield job