From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- taskcluster/gecko_taskgraph/loader/__init__.py | 0 taskcluster/gecko_taskgraph/loader/test.py | 142 ++++++++++++++++++++++++ taskcluster/gecko_taskgraph/loader/transform.py | 59 ++++++++++ 3 files changed, 201 insertions(+) create mode 100644 taskcluster/gecko_taskgraph/loader/__init__.py create mode 100644 taskcluster/gecko_taskgraph/loader/test.py create mode 100644 taskcluster/gecko_taskgraph/loader/transform.py (limited to 'taskcluster/gecko_taskgraph/loader') diff --git a/taskcluster/gecko_taskgraph/loader/__init__.py b/taskcluster/gecko_taskgraph/loader/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/taskcluster/gecko_taskgraph/loader/test.py b/taskcluster/gecko_taskgraph/loader/test.py new file mode 100644 index 0000000000..c97acecd1a --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/test.py @@ -0,0 +1,142 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph.util.copy_task import copy_task + +from .transform import loader as transform_loader + +logger = logging.getLogger(__name__) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Generate tasks implementing Gecko tests. + """ + + builds_by_platform = get_builds_by_platform( + dep_kind="build", loaded_tasks=loaded_tasks + ) + signed_builds_by_platform = get_builds_by_platform( + dep_kind="build-signing", loaded_tasks=loaded_tasks + ) + + # get the test platforms for those build tasks + test_platforms_cfg = load_yaml(path, "test-platforms.yml") + test_platforms = get_test_platforms( + test_platforms_cfg, builds_by_platform, signed_builds_by_platform + ) + + # expand the test sets for each of those platforms + test_sets_cfg = load_yaml(path, "test-sets.yml") + test_platforms = expand_tests(test_sets_cfg, test_platforms) + + # load the test descriptions + tests = transform_loader(kind, path, config, params, loaded_tasks) + test_descriptions = {t.pop("name"): t for t in tests} + + # generate all tests for all test platforms + for test_platform_name, test_platform in test_platforms.items(): + for test_name in test_platform["test-names"]: + test = copy_task(test_descriptions[test_name]) + test["build-platform"] = test_platform["build-platform"] + test["test-platform"] = test_platform_name + test["build-label"] = test_platform["build-label"] + if test_platform.get("build-signing-label", None): + test["build-signing-label"] = test_platform["build-signing-label"] + + test["build-attributes"] = test_platform["build-attributes"] + test["test-name"] = test_name + if test_platform.get("shippable"): + test.setdefault("attributes", {})["shippable"] = True + test["attributes"]["shipping_product"] = test_platform[ + "shipping_product" + ] + + logger.debug( + "Generating tasks for test {} on platform {}".format( + test_name, test["test-platform"] + ) + ) + yield test + + +def get_builds_by_platform(dep_kind, loaded_tasks): + """Find the build tasks on which tests will depend, keyed by + platform/type. Returns a dictionary mapping build platform to task.""" + builds_by_platform = {} + for task in loaded_tasks: + if task.kind != dep_kind: + continue + + build_platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + if not build_platform or not build_type: + continue + platform = f"{build_platform}/{build_type}" + if platform in builds_by_platform: + raise Exception("multiple build jobs for " + platform) + builds_by_platform[platform] = task + return builds_by_platform + + +def get_test_platforms( + test_platforms_cfg, builds_by_platform, signed_builds_by_platform={} +): + """Get the test platforms for which test tasks should be generated, + based on the available build platforms. Returns a dictionary mapping + test platform to {test-set, build-platform, build-label}.""" + test_platforms = {} + for test_platform, cfg in test_platforms_cfg.items(): + build_platform = cfg["build-platform"] + if build_platform not in builds_by_platform: + logger.warning( + "No build task with platform {}; ignoring test platform {}".format( + build_platform, test_platform + ) + ) + continue + test_platforms[test_platform] = { + "build-platform": build_platform, + "build-label": builds_by_platform[build_platform].label, + "build-attributes": builds_by_platform[build_platform].attributes, + } + + if builds_by_platform[build_platform].attributes.get("shippable"): + test_platforms[test_platform]["shippable"] = builds_by_platform[ + build_platform + ].attributes["shippable"] + test_platforms[test_platform]["shipping_product"] = builds_by_platform[ + build_platform + ].attributes["shipping_product"] + + test_platforms[test_platform].update(cfg) + + return test_platforms + + +def expand_tests(test_sets_cfg, test_platforms): + """Expand the test sets in `test_platforms` out to sets of test names. + Returns a dictionary like `get_test_platforms`, with an additional + `test-names` key for each test platform, containing a set of test + names.""" + rv = {} + for test_platform, cfg in test_platforms.items(): + test_sets = cfg["test-sets"] + if not set(test_sets) <= set(test_sets_cfg): + raise Exception( + "Test sets {} for test platform {} are not defined".format( + ", ".join(test_sets), test_platform + ) + ) + test_names = set() + for test_set in test_sets: + test_names.update(test_sets_cfg[test_set]) + rv[test_platform] = cfg.copy() + rv[test_platform]["test-names"] = test_names + return rv diff --git a/taskcluster/gecko_taskgraph/loader/transform.py b/taskcluster/gecko_taskgraph/loader/transform.py new file mode 100644 index 0000000000..1e513bcb73 --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/transform.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.yaml import load_yaml + +from ..util.templates import merge + +logger = logging.getLogger(__name__) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Get the input elements that will be transformed into tasks in a generic + way. The elements themselves are free-form, and become the input to the + first transform. + + By default, this reads jobs from the `jobs` key, or from yaml files + named by `jobs-from`. The entities are read from mappings, and the + keys to those mappings are added in the `name` key of each entity. + + If there is a `job-defaults` config, then every job is merged with it. + This provides a simple way to set default values for all jobs of a kind. + The `job-defaults` key can also be specified in a yaml file pointed to by + `jobs-from`. In this case it will only apply to tasks defined in the same + file. + + Other kind implementations can use a different loader function to + produce inputs and hand them to `transform_inputs`. + """ + + def jobs(): + defaults = config.get("job-defaults") + for name, job in config.get("jobs", {}).items(): + if defaults: + job = merge(defaults, job) + job["job-from"] = "kind.yml" + yield name, job + + for filename in config.get("jobs-from", []): + tasks = load_yaml(path, filename) + + file_defaults = tasks.pop("job-defaults", None) + if defaults: + file_defaults = merge(defaults, file_defaults or {}) + + for name, job in tasks.items(): + if file_defaults: + job = merge(file_defaults, job) + job["job-from"] = filename + yield name, job + + for name, job in jobs(): + job["name"] = name + logger.debug(f"Generating tasks for {kind} {name}") + yield job -- cgit v1.2.3