diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py | |
parent | Initial commit. (diff) | |
download | firefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py')
-rw-r--r-- | third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py | 283 |
1 files changed, 283 insertions, 0 deletions
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py new file mode 100644 index 0000000000..5911914f13 --- /dev/null +++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py @@ -0,0 +1,283 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import sys +from abc import ABC, abstractmethod + +import attr + +from taskgraph.config import GraphConfig +from taskgraph.parameters import Parameters +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.attributes import match_run_on_projects +from taskgraph.util.treeherder import join_symbol + +logger = logging.getLogger(__name__) + + +@attr.s(frozen=True) +class Verification(ABC): + func = attr.ib() + + @abstractmethod + def verify(self, **kwargs) -> None: + pass + + +@attr.s(frozen=True) +class InitialVerification(Verification): + """Verification that doesn't depend on any generation state.""" + + def verify(self): + self.func() + + +@attr.s(frozen=True) +class GraphVerification(Verification): + """Verification for a TaskGraph object.""" + + run_on_projects = attr.ib(default=None) + + def verify( + self, graph: TaskGraph, graph_config: GraphConfig, parameters: Parameters + ): + if self.run_on_projects and not match_run_on_projects( + parameters["project"], self.run_on_projects + ): + return + + scratch_pad = {} + graph.for_each_task( + self.func, + scratch_pad=scratch_pad, + graph_config=graph_config, + parameters=parameters, + ) + self.func( + None, + graph, + scratch_pad=scratch_pad, + graph_config=graph_config, + parameters=parameters, + ) + + +@attr.s(frozen=True) +class ParametersVerification(Verification): + """Verification for a set of parameters.""" + + def verify(self, parameters: Parameters): + self.func(parameters) + + +@attr.s(frozen=True) +class KindsVerification(Verification): + """Verification for kinds.""" + + def verify(self, kinds: dict): + self.func(kinds) + + +@attr.s(frozen=True) +class VerificationSequence: + """ + Container for a sequence of verifications over a TaskGraph. Each + verification is represented as a callable taking (task, taskgraph, + scratch_pad), called for each task in the taskgraph, and one more + time with no task but with the taskgraph and the same scratch_pad + that was passed for each task. + """ + + _verifications = attr.ib(factory=dict) + _verification_types = { + "graph": GraphVerification, + "initial": InitialVerification, + "kinds": KindsVerification, + "parameters": ParametersVerification, + } + + def __call__(self, name, *args, **kwargs): + for verification in self._verifications.get(name, []): + verification.verify(*args, **kwargs) + + def add(self, name, **kwargs): + cls = self._verification_types.get(name, GraphVerification) + + def wrap(func): + self._verifications.setdefault(name, []).append(cls(func, **kwargs)) + return func + + return wrap + + +verifications = VerificationSequence() + + +@verifications.add("full_task_graph") +def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function verifies that tuple + (collection.keys(), machine.platform, groupSymbol, symbol) is unique + for a target task graph. + """ + if task is None: + return + task_dict = task.task + if "extra" in task_dict: + extra = task_dict["extra"] + if "treeherder" in extra: + treeherder = extra["treeherder"] + + collection_keys = tuple(sorted(treeherder.get("collection", {}).keys())) + if len(collection_keys) != 1: + raise Exception( + "Task {} can't be in multiple treeherder collections " + "(the part of the platform after `/`): {}".format( + task.label, collection_keys + ) + ) + platform = treeherder.get("machine", {}).get("platform") + group_symbol = treeherder.get("groupSymbol") + symbol = treeherder.get("symbol") + + key = (platform, collection_keys[0], group_symbol, symbol) + if key in scratch_pad: + raise Exception( + "Duplicate treeherder platform and symbol in tasks " + "`{}`and `{}`: {} {}".format( + task.label, + scratch_pad[key], + f"{platform}/{collection_keys[0]}", + join_symbol(group_symbol, symbol), + ) + ) + else: + scratch_pad[key] = task.label + + +@verifications.add("full_task_graph") +def verify_trust_domain_v2_routes( + task, taskgraph, scratch_pad, graph_config, parameters +): + """ + This function ensures that any two tasks have distinct ``index.{trust-domain}.v2`` routes. + """ + if task is None: + return + route_prefix = "index.{}.v2".format(graph_config["trust-domain"]) + task_dict = task.task + routes = task_dict.get("routes", []) + + for route in routes: + if route.startswith(route_prefix): + if route in scratch_pad: + raise Exception( + "conflict between {}:{} for route: {}".format( + task.label, scratch_pad[route], route + ) + ) + else: + scratch_pad[route] = task.label + + +@verifications.add("full_task_graph") +def verify_routes_notification_filters( + task, taskgraph, scratch_pad, graph_config, parameters +): + """ + This function ensures that only understood filters for notifications are + specified. + + See: https://docs.taskcluster.net/reference/core/taskcluster-notify/docs/usage + """ + if task is None: + return + route_prefix = "notify." + valid_filters = ("on-any", "on-completed", "on-failed", "on-exception") + task_dict = task.task + routes = task_dict.get("routes", []) + + for route in routes: + if route.startswith(route_prefix): + # Get the filter of the route + route_filter = route.split(".")[-1] + if route_filter not in valid_filters: + raise Exception( + "{} has invalid notification filter ({})".format( + task.label, route_filter + ) + ) + + +@verifications.add("full_task_graph") +def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, parameters): + tiers = scratch_pad + if task is not None: + tiers[task.label] = ( + task.task.get("extra", {}).get("treeherder", {}).get("tier", sys.maxsize) + ) + else: + + def printable_tier(tier): + if tier == sys.maxsize: + return "unknown" + return tier + + for task in taskgraph.tasks.values(): + tier = tiers[task.label] + for d in task.dependencies.values(): + if taskgraph[d].task.get("workerType") == "always-optimized": + continue + if "dummy" in taskgraph[d].kind: + continue + if tier < tiers[d]: + raise Exception( + "{} (tier {}) cannot depend on {} (tier {})".format( + task.label, + printable_tier(tier), + d, + printable_tier(tiers[d]), + ) + ) + + +@verifications.add("full_task_graph") +def verify_toolchain_alias(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function verifies that toolchain aliases are not reused. + """ + if task is None: + return + attributes = task.attributes + if "toolchain-alias" in attributes: + keys = attributes["toolchain-alias"] + if not keys: + keys = [] + elif isinstance(keys, str): + keys = [keys] + for key in keys: + if key in scratch_pad: + raise Exception( + "Duplicate toolchain-alias in tasks " + "`{}`and `{}`: {}".format( + task.label, + scratch_pad[key], + key, + ) + ) + else: + scratch_pad[key] = task.label + + +@verifications.add("optimized_task_graph") +def verify_always_optimized(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function ensures that always-optimized tasks have been optimized. + """ + if task is None: + return + if task.task.get("workerType") == "always-optimized": + raise Exception(f"Could not optimize the task {task.label!r}") |