diff options
Diffstat (limited to 'third_party/python/taskcluster_taskgraph/taskgraph/loader')
3 files changed, 91 insertions, 0 deletions
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/loader/__init__.py b/third_party/python/taskcluster_taskgraph/taskgraph/loader/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/python/taskcluster_taskgraph/taskgraph/loader/__init__.py diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/loader/default.py b/third_party/python/taskcluster_taskgraph/taskgraph/loader/default.py new file mode 100644 index 0000000000..5b2c258917 --- /dev/null +++ b/third_party/python/taskcluster_taskgraph/taskgraph/loader/default.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from .transform import loader as transform_loader + +logger = logging.getLogger(__name__) + + +DEFAULT_TRANSFORMS = [ + "taskgraph.transforms.job:transforms", + "taskgraph.transforms.task:transforms", +] + + +def loader(kind, path, config, params, loaded_tasks): + """ + This default loader builds on the `transform` loader by providing sensible + default transforms that the majority of simple tasks will need. + Specifically, `job` and `task` transforms will be appended to the end of the + list of transforms in the kind being loaded. + """ + transform_refs = config.setdefault("transforms", []) + for t in DEFAULT_TRANSFORMS: + if t in config.get("transforms", ()): + raise KeyError( + f"Transform {t} is already present in the loader's default transforms; it must not be defined in the kind" + ) + transform_refs.extend(DEFAULT_TRANSFORMS) + return transform_loader(kind, path, config, params, loaded_tasks) diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/loader/transform.py b/third_party/python/taskcluster_taskgraph/taskgraph/loader/transform.py new file mode 100644 index 0000000000..a134ffd127 --- /dev/null +++ b/third_party/python/taskcluster_taskgraph/taskgraph/loader/transform.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.templates import merge +from taskgraph.util.yaml import load_yaml + +logger = logging.getLogger(__name__) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Get the input elements that will be transformed into tasks in a generic + way. The elements themselves are free-form, and become the input to the + first transform. + + By default, this reads tasks from the `tasks` key, or from yaml files + named by `tasks-from`. The entities are read from mappings, and the + keys to those mappings are added in the `name` key of each entity. + + If there is a `task-defaults` config, then every task is merged with it. + This provides a simple way to set default values for all tasks of a kind. + The `task-defaults` key can also be specified in a yaml file pointed to by + `tasks-from`. In this case it will only apply to tasks defined in the same + file. + + Other kind implementations can use a different loader function to + produce inputs and hand them to `transform_inputs`. + """ + + def generate_tasks(): + defaults = config.get("task-defaults") + for name, task in config.get("tasks", {}).items(): + if defaults: + task = merge(defaults, task) + task["task-from"] = "kind.yml" + yield name, task + + for filename in config.get("tasks-from", []): + tasks = load_yaml(path, filename) + + file_defaults = tasks.pop("task-defaults", None) + if defaults: + file_defaults = merge(defaults, file_defaults or {}) + + for name, task in tasks.items(): + if file_defaults: + task = merge(file_defaults, task) + task["task-from"] = filename + yield name, task + + for name, task in generate_tasks(): + task["name"] = name + logger.debug(f"Generating tasks for {kind} {name}") + yield task |