Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 201 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/loader/__init__.py b/taskcluster/gecko_taskgraph/loader/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/loader/__init__.py
diff --git a/taskcluster/gecko_taskgraph/loader/test.py b/taskcluster/gecko_taskgraph/loader/test.py
new file mode 100644
index 0000000000..c97acecd1a
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/loader/test.py
@@ -0,0 +1,142 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from taskgraph.util.yaml import load_yaml
+
+from gecko_taskgraph.util.copy_task import copy_task
+
+from .transform import loader as transform_loader
+
+logger = logging.getLogger(__name__)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Generate tasks implementing Gecko tests.
+    """
+
+    builds_by_platform = get_builds_by_platform(
+        dep_kind="build", loaded_tasks=loaded_tasks
+    )
+    signed_builds_by_platform = get_builds_by_platform(
+        dep_kind="build-signing", loaded_tasks=loaded_tasks
+    )
+
+    # get the test platforms for those build tasks
+    test_platforms_cfg = load_yaml(path, "test-platforms.yml")
+    test_platforms = get_test_platforms(
+        test_platforms_cfg, builds_by_platform, signed_builds_by_platform
+    )
+
+    # expand the test sets for each of those platforms
+    test_sets_cfg = load_yaml(path, "test-sets.yml")
+    test_platforms = expand_tests(test_sets_cfg, test_platforms)
+
+    # load the test descriptions
+    tests = transform_loader(kind, path, config, params, loaded_tasks)
+    test_descriptions = {t.pop("name"): t for t in tests}
+
+    # generate all tests for all test platforms
+    for test_platform_name, test_platform in test_platforms.items():
+        for test_name in test_platform["test-names"]:
+            test = copy_task(test_descriptions[test_name])
+            test["build-platform"] = test_platform["build-platform"]
+            test["test-platform"] = test_platform_name
+            test["build-label"] = test_platform["build-label"]
+            if test_platform.get("build-signing-label", None):
+                test["build-signing-label"] = test_platform["build-signing-label"]
+
+            test["build-attributes"] = test_platform["build-attributes"]
+            test["test-name"] = test_name
+            if test_platform.get("shippable"):
+                test.setdefault("attributes", {})["shippable"] = True
+                test["attributes"]["shipping_product"] = test_platform[
+                    "shipping_product"
+                ]
+
+            logger.debug(
+                "Generating tasks for test {} on platform {}".format(
+                    test_name, test["test-platform"]
+                )
+            )
+            yield test
+
+
+def get_builds_by_platform(dep_kind, loaded_tasks):
+    """Find the build tasks on which tests will depend, keyed by
+    platform/type.  Returns a dictionary mapping build platform to task."""
+    builds_by_platform = {}
+    for task in loaded_tasks:
+        if task.kind != dep_kind:
+            continue
+
+        build_platform = task.attributes.get("build_platform")
+        build_type = task.attributes.get("build_type")
+        if not build_platform or not build_type:
+            continue
+        platform = f"{build_platform}/{build_type}"
+        if platform in builds_by_platform:
+            raise Exception("multiple build jobs for " + platform)
+        builds_by_platform[platform] = task
+    return builds_by_platform
+
+
+def get_test_platforms(
+    test_platforms_cfg, builds_by_platform, signed_builds_by_platform={}
+):
+    """Get the test platforms for which test tasks should be generated,
+    based on the available build platforms.  Returns a dictionary mapping
+    test platform to {test-set, build-platform, build-label}."""
+    test_platforms = {}
+    for test_platform, cfg in test_platforms_cfg.items():
+        build_platform = cfg["build-platform"]
+        if build_platform not in builds_by_platform:
+            logger.warning(
+                "No build task with platform {}; ignoring test platform {}".format(
+                    build_platform, test_platform
+                )
+            )
+            continue
+        test_platforms[test_platform] = {
+            "build-platform": build_platform,
+            "build-label": builds_by_platform[build_platform].label,
+            "build-attributes": builds_by_platform[build_platform].attributes,
+        }
+
+        if builds_by_platform[build_platform].attributes.get("shippable"):
+            test_platforms[test_platform]["shippable"] = builds_by_platform[
+                build_platform
+            ].attributes["shippable"]
+            test_platforms[test_platform]["shipping_product"] = builds_by_platform[
+                build_platform
+            ].attributes["shipping_product"]
+
+        test_platforms[test_platform].update(cfg)
+
+    return test_platforms
+
+
+def expand_tests(test_sets_cfg, test_platforms):
+    """Expand the test sets in `test_platforms` out to sets of test names.
+    Returns a dictionary like `get_test_platforms`, with an additional
+    `test-names` key for each test platform, containing a set of test
+    names."""
+    rv = {}
+    for test_platform, cfg in test_platforms.items():
+        test_sets = cfg["test-sets"]
+        if not set(test_sets) <= set(test_sets_cfg):
+            raise Exception(
+                "Test sets {} for test platform {} are not defined".format(
+                    ", ".join(test_sets), test_platform
+                )
+            )
+        test_names = set()
+        for test_set in test_sets:
+            test_names.update(test_sets_cfg[test_set])
+        rv[test_platform] = cfg.copy()
+        rv[test_platform]["test-names"] = test_names
+    return rv
diff --git a/taskcluster/gecko_taskgraph/loader/transform.py b/taskcluster/gecko_taskgraph/loader/transform.py
new file mode 100644
index 0000000000..1e513bcb73
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/loader/transform.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from taskgraph.util.yaml import load_yaml
+
+from ..util.templates import merge
+
+logger = logging.getLogger(__name__)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Get the input elements that will be transformed into tasks in a generic
+    way.  The elements themselves are free-form, and become the input to the
+    first transform.
+
+    By default, this reads jobs from the `jobs` key, or from yaml files
+    named by `jobs-from`.  The entities are read from mappings, and the
+    keys to those mappings are added in the `name` key of each entity.
+
+    If there is a `job-defaults` config, then every job is merged with it.
+    This provides a simple way to set default values for all jobs of a kind.
+    The `job-defaults` key can also be specified in a yaml file pointed to by
+    `jobs-from`. In this case it will only apply to tasks defined in the same
+    file.
+
+    Other kind implementations can use a different loader function to
+    produce inputs and hand them to `transform_inputs`.
+    """
+
+    def jobs():
+        defaults = config.get("job-defaults")
+        for name, job in config.get("jobs", {}).items():
+            if defaults:
+                job = merge(defaults, job)
+            job["job-from"] = "kind.yml"
+            yield name, job
+
+        for filename in config.get("jobs-from", []):
+            tasks = load_yaml(path, filename)
+
+            file_defaults = tasks.pop("job-defaults", None)
+            if defaults:
+                file_defaults = merge(defaults, file_defaults or {})
+
+            for name, job in tasks.items():
+                if file_defaults:
+                    job = merge(file_defaults, job)
+                job["job-from"] = filename
+                yield name, job
+
+    for name, job in jobs():
+        job["name"] = name
+        logger.debug(f"Generating tasks for {kind} {name}")
+        yield job