Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 275 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/loader/multi_dep.py b/taskcluster/gecko_taskgraph/loader/multi_dep.py
new file mode 100644
index 0000000000..04fda04415
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/loader/multi_dep.py
@@ -0,0 +1,275 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from taskgraph.task import Task
+from taskgraph.util.schema import Schema
+from voluptuous import Required
+
+from gecko_taskgraph.util.copy_task import copy_task
+
+from ..util.attributes import sorted_unique_list
+
+schema = Schema(
+    {
+        Required("primary-dependency", "primary dependency task"): Task,
+        Required(
+            "dependent-tasks",
+            "dictionary of dependent tasks, keyed by kind",
+        ): {str: Task},
+    }
+)
+
+
+# Define a collection of group_by functions
+GROUP_BY_MAP = {}
+
+
+def group_by(name):
+    def wrapper(func):
+        GROUP_BY_MAP[name] = func
+        return func
+
+    return wrapper
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Load tasks based on the jobs dependant kinds, designed for use as
+    multiple-dependent needs.
+
+    Required ``group-by-fn`` is used to define how we coalesce the
+    multiple deps together to pass to transforms, e.g. all kinds specified get
+    collapsed by platform with `platform`
+
+    Optional ``primary-dependency`` (ordered list or string) is used to determine
+    which upstream kind to inherit attrs from. See ``get_primary_dep``.
+
+    The `only-for-build-platforms` kind configuration, if specified, will limit
+    the build platforms for which a job will be created. Alternatively there is
+    'not-for-build-platforms' kind configuration which will be consulted only after
+    'only-for-build-platforms' is checked (if present), and omit any jobs where the
+    build platform matches.
+
+    Optional ``job-template`` kind configuration value, if specified, will be used to
+    pass configuration down to the specified transforms used.
+    """
+    job_template = config.get("job-template")
+
+    for dep_tasks in group_tasks(config, loaded_tasks):
+        job = {"dependent-tasks": dep_tasks}
+        job["primary-dependency"] = get_primary_dep(config, dep_tasks)
+        if job_template:
+            job.update(copy_task(job_template))
+        # copy shipping_product from upstream
+        product = job["primary-dependency"].attributes.get(
+            "shipping_product", job["primary-dependency"].task.get("shipping-product")
+        )
+        if product:
+            job.setdefault("shipping-product", product)
+        job.setdefault("attributes", {})["required_signoffs"] = sorted_unique_list(
+            *(
+                task.attributes.get("required_signoffs", [])
+                for task in dep_tasks.values()
+            )
+        )
+
+        yield job
+
+
+def skip_only_or_not(config, task):
+    """Return True if we should skip this task based on `only_` or `not_` config."""
+    only_platforms = config.get("only-for-build-platforms")
+    not_platforms = config.get("not-for-build-platforms")
+    only_attributes = config.get("only-for-attributes")
+    not_attributes = config.get("not-for-attributes")
+    task_attrs = task.attributes
+    if only_platforms or not_platforms:
+        platform = task_attrs.get("build_platform")
+        build_type = task_attrs.get("build_type")
+        if not platform or not build_type:
+            return True
+        combined_platform = f"{platform}/{build_type}"
+        if only_platforms and combined_platform not in only_platforms:
+            return True
+        if not_platforms and combined_platform in not_platforms:
+            return True
+    if only_attributes:
+        if not set(only_attributes) & set(task_attrs):
+            # make sure any attribute exists
+            return True
+    if not_attributes:
+        if set(not_attributes) & set(task_attrs):
+            return True
+    return False
+
+
+def group_tasks(config, tasks):
+    group_by_fn = GROUP_BY_MAP[config["group-by"]]
+
+    groups = group_by_fn(config, tasks)
+
+    for combinations in groups.values():
+        kinds = [f.kind for f in combinations]
+        assert_unique_members(
+            kinds,
+            error_msg=("Multi_dep.py should have filtered down to one task per kind"),
+        )
+        dependencies = {t.kind: copy_task(t) for t in combinations}
+        yield dependencies
+
+
+@group_by("platform")
+def platform_grouping(config, tasks):
+    groups = {}
+    for task in tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+        if skip_only_or_not(config, task):
+            continue
+        platform = task.attributes.get("build_platform")
+        build_type = task.attributes.get("build_type")
+        product = task.attributes.get(
+            "shipping_product", task.task.get("shipping-product")
+        )
+
+        groups.setdefault((platform, build_type, product), []).append(task)
+    return groups
+
+
+@group_by("single-locale")
+def single_locale_grouping(config, tasks):
+    """Split by a single locale (but also by platform, build-type, product)
+
+    The locale can be `None` (en-US build/signing/repackage), a single locale,
+    or multiple locales per task, e.g. for l10n chunking. In the case of a task
+    with, say, five locales, the task will show up in all five locale groupings.
+
+    This grouping is written for non-partner-repack beetmover, but might also
+    be useful elsewhere.
+
+    """
+    groups = {}
+
+    for task in tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+        if skip_only_or_not(config, task):
+            continue
+        platform = task.attributes.get("build_platform")
+        build_type = task.attributes.get("build_type")
+        product = task.attributes.get(
+            "shipping_product", task.task.get("shipping-product")
+        )
+        task_locale = task.attributes.get("locale")
+        chunk_locales = task.attributes.get("chunk_locales")
+        locales = chunk_locales or [task_locale]
+
+        for locale in locales:
+            locale_key = (platform, build_type, product, locale)
+            groups.setdefault(locale_key, [])
+            if task not in groups[locale_key]:
+                groups[locale_key].append(task)
+
+    return groups
+
+
+@group_by("chunk-locales")
+def chunk_locale_grouping(config, tasks):
+    """Split by a chunk_locale (but also by platform, build-type, product)
+
+    This grouping is written for mac signing with notarization, but might also
+    be useful elsewhere.
+
+    """
+    groups = {}
+
+    for task in tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+        if skip_only_or_not(config, task):
+            continue
+        platform = task.attributes.get("build_platform")
+        build_type = task.attributes.get("build_type")
+        product = task.attributes.get(
+            "shipping_product", task.task.get("shipping-product")
+        )
+        chunk_locales = tuple(sorted(task.attributes.get("chunk_locales", [])))
+
+        chunk_locale_key = (platform, build_type, product, chunk_locales)
+        groups.setdefault(chunk_locale_key, [])
+        if task not in groups[chunk_locale_key]:
+            groups[chunk_locale_key].append(task)
+
+    return groups
+
+
+@group_by("partner-repack-ids")
+def partner_repack_ids_grouping(config, tasks):
+    """Split by partner_repack_ids (but also by platform, build-type, product)
+
+    This grouping is written for release-{eme-free,partner}-repack-signing.
+
+    """
+    groups = {}
+
+    for task in tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+        if skip_only_or_not(config, task):
+            continue
+        platform = task.attributes.get("build_platform")
+        build_type = task.attributes.get("build_type")
+        product = task.attributes.get(
+            "shipping_product", task.task.get("shipping-product")
+        )
+        partner_repack_ids = tuple(
+            sorted(task.task.get("extra", {}).get("repack_ids", []))
+        )
+
+        partner_repack_ids_key = (platform, build_type, product, partner_repack_ids)
+        groups.setdefault(partner_repack_ids_key, [])
+        if task not in groups[partner_repack_ids_key]:
+            groups[partner_repack_ids_key].append(task)
+
+    return groups
+
+
+def assert_unique_members(kinds, error_msg=None):
+    if len(kinds) != len(set(kinds)):
+        raise Exception(error_msg)
+
+
+def get_primary_dep(config, dep_tasks):
+    """Find the dependent task to inherit attributes from.
+
+    If ``primary-dependency`` is defined in ``kind.yml`` and is a string,
+    then find the first dep with that task kind and return it. If it is
+    defined and is a list, the first kind in that list with a matching dep
+    is the primary dependency. If it's undefined, return the first dep.
+
+    """
+    primary_dependencies = config.get("primary-dependency")
+    if isinstance(primary_dependencies, str):
+        primary_dependencies = [primary_dependencies]
+    if not primary_dependencies:
+        assert len(dep_tasks) == 1, "Must define a primary-dependency!"
+        return list(dep_tasks.values())[0]
+    primary_dep = None
+    for primary_kind in primary_dependencies:
+        for dep_kind in dep_tasks:
+            if dep_kind == primary_kind:
+                assert (
+                    primary_dep is None
+                ), "Too many primary dependent tasks in dep_tasks: {}!".format(
+                    [t.label for t in dep_tasks]
+                )
+                primary_dep = dep_tasks[dep_kind]
+    if primary_dep is None:
+        raise Exception(
+            "Can't find dependency of {}: {}".format(
+                config["primary-dependency"], config
+            )
+        )
+    return primary_dep