From 2aa4a82499d4becd2284cdb482213d541b8804dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 16:29:10 +0200 Subject: Adding upstream version 86.0.1. Signed-off-by: Daniel Baumann --- taskcluster/mach_commands.py | 701 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 701 insertions(+) create mode 100644 taskcluster/mach_commands.py (limited to 'taskcluster/mach_commands.py') diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py new file mode 100644 index 0000000000..d72530fc2c --- /dev/null +++ b/taskcluster/mach_commands.py @@ -0,0 +1,701 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import json +import logging +import os +from six import text_type +import six +import sys +import time +import traceback +import re + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + SubCommand, +) + +from mozbuild.base import MachCommandBase + + +def strtobool(value): + """Convert string to boolean. + + Wraps "distutils.util.strtobool", deferring the import of the package + in case it's not installed. Otherwise, we have a "chicken and egg problem" where + |mach bootstrap| would install the required package to enable "distutils.util", but + it can't because mach fails to interpret this file. + """ + from distutils.util import strtobool + + return bool(strtobool(value)) + + +class ShowTaskGraphSubCommand(SubCommand): + """A SubCommand with TaskGraph-specific arguments""" + + def __call__(self, func): + after = SubCommand.__call__(self, func) + args = [ + CommandArgument( + "--root", + "-r", + help="root of the taskgraph definition relative to topsrcdir", + ), + CommandArgument( + "--quiet", "-q", action="store_true", help="suppress all logging output" + ), + CommandArgument( + "--verbose", + "-v", + action="store_true", + help="include debug-level logging output", + ), + CommandArgument( + "--json", + "-J", + action="store_const", + dest="format", + const="json", + help="Output task graph as a JSON object", + ), + CommandArgument( + "--labels", + "-L", + action="store_const", + dest="format", + const="labels", + help="Output the label for each task in the task graph (default)", + ), + CommandArgument( + "--parameters", + "-p", + default="project=mozilla-central", + help="parameters file (.yml or .json; see " + "`taskcluster/docs/parameters.rst`)`", + ), + CommandArgument( + "--no-optimize", + dest="optimize", + action="store_false", + default="true", + help="do not remove tasks from the graph that are found in the " + "index (a.k.a. optimize the graph)", + ), + CommandArgument( + "--tasks-regex", + "--tasks", + default=None, + help="only return tasks with labels matching this regular " + "expression.", + ), + CommandArgument( + "--target-kind", + default=None, + help="only return tasks that are of the given kind, " + "or their dependencies.", + ), + CommandArgument( + "-F", + "--fast", + dest="fast", + default=False, + action="store_true", + help="enable fast task generation for local debugging.", + ), + CommandArgument( + "-o", + "--output-file", + default=None, + help="file path to store generated output.", + ), + ] + for arg in args: + after = arg(after) + return after + + +@CommandProvider +class MachCommands(MachCommandBase): + @Command( + "taskgraph", + category="ci", + description="Manipulate TaskCluster task graphs defined in-tree", + ) + def taskgraph(self): + """The taskgraph subcommands all relate to the generation of task graphs + for Gecko continuous integration. A task graph is a set of tasks linked + by dependencies: for example, a binary must be built before it is tested, + and that build may further depend on various toolchains, libraries, etc. + """ + + @ShowTaskGraphSubCommand( + "taskgraph", "tasks", description="Show all tasks in the taskgraph" + ) + def taskgraph_tasks(self, **options): + return self.show_taskgraph("full_task_set", options) + + @ShowTaskGraphSubCommand("taskgraph", "full", description="Show the full taskgraph") + def taskgraph_full(self, **options): + return self.show_taskgraph("full_task_graph", options) + + @ShowTaskGraphSubCommand( + "taskgraph", "target", description="Show the target task set" + ) + def taskgraph_target(self, **options): + return self.show_taskgraph("target_task_set", options) + + @ShowTaskGraphSubCommand( + "taskgraph", "target-graph", description="Show the target taskgraph" + ) + def taskgraph_target_taskgraph(self, **options): + return self.show_taskgraph("target_task_graph", options) + + @ShowTaskGraphSubCommand( + "taskgraph", "optimized", description="Show the optimized taskgraph" + ) + def taskgraph_optimized(self, **options): + return self.show_taskgraph("optimized_task_graph", options) + + @ShowTaskGraphSubCommand( + "taskgraph", "morphed", description="Show the morphed taskgraph" + ) + def taskgraph_morphed(self, **options): + return self.show_taskgraph("morphed_task_graph", options) + + @SubCommand("taskgraph", "actions", description="Write actions.json to stdout") + @CommandArgument( + "--root", "-r", help="root of the taskgraph definition relative to topsrcdir" + ) + @CommandArgument( + "--quiet", "-q", action="store_true", help="suppress all logging output" + ) + @CommandArgument( + "--verbose", + "-v", + action="store_true", + help="include debug-level logging output", + ) + @CommandArgument( + "--parameters", + "-p", + default="project=mozilla-central", + help="parameters file (.yml or .json; see " + "`taskcluster/docs/parameters.rst`)`", + ) + def taskgraph_actions(self, **options): + return self.show_actions(options) + + @SubCommand("taskgraph", "decision", description="Run the decision task") + @CommandArgument( + "--root", + "-r", + type=text_type, + help="root of the taskgraph definition relative to topsrcdir", + ) + @CommandArgument( + "--base-repository", + type=text_type, + required=True, + help='URL for "base" repository to clone', + ) + @CommandArgument( + "--head-repository", + type=text_type, + required=True, + help='URL for "head" repository to fetch revision from', + ) + @CommandArgument( + "--head-ref", + type=text_type, + required=True, + help="Reference (this is same as rev usually for hg)", + ) + @CommandArgument( + "--head-rev", + type=text_type, + required=True, + help="Commit revision to use from head repository", + ) + @CommandArgument( + "--comm-base-repository", + type=text_type, + required=False, + help='URL for "base" comm-* repository to clone', + ) + @CommandArgument( + "--comm-head-repository", + type=text_type, + required=False, + help='URL for "head" comm-* repository to fetch revision from', + ) + @CommandArgument( + "--comm-head-ref", + type=text_type, + required=False, + help="comm-* Reference (this is same as rev usually for hg)", + ) + @CommandArgument( + "--comm-head-rev", + type=text_type, + required=False, + help="Commit revision to use from head comm-* repository", + ) + @CommandArgument( + "--project", + type=text_type, + required=True, + help="Project to use for creating task graph. Example: --project=try", + ) + @CommandArgument( + "--pushlog-id", type=text_type, dest="pushlog_id", required=True, default="0" + ) + @CommandArgument("--pushdate", dest="pushdate", required=True, type=int, default=0) + @CommandArgument( + "--owner", + type=text_type, + required=True, + help="email address of who owns this graph", + ) + @CommandArgument( + "--level", type=text_type, required=True, help="SCM level of this repository" + ) + @CommandArgument( + "--target-tasks-method", + type=text_type, + help="method for selecting the target tasks to generate", + ) + @CommandArgument( + "--optimize-target-tasks", + type=lambda flag: strtobool(flag), + nargs="?", + const="true", + help="If specified, this indicates whether the target " + "tasks are eligible for optimization. Otherwise, " + "the default for the project is used.", + ) + @CommandArgument( + "--try-task-config-file", + type=text_type, + help="path to try task configuration file", + ) + @CommandArgument( + "--tasks-for", + type=text_type, + required=True, + help="the tasks_for value used to generate this task", + ) + @CommandArgument( + "--include-push-tasks", + action="store_true", + help="Whether tasks from the on-push graph should be re-used " + "in this graph. This allows cron graphs to avoid rebuilding " + "jobs that were built on-push.", + ) + @CommandArgument( + "--rebuild-kind", + dest="rebuild_kinds", + action="append", + default=argparse.SUPPRESS, + help="Kinds that should not be re-used from the on-push graph.", + ) + def taskgraph_decision(self, **options): + """Run the decision task: generate a task graph and submit to + TaskCluster. This is only meant to be called within decision tasks, + and requires a great many arguments. Commands like `mach taskgraph + optimized` are better suited to use on the command line, and can take + the parameters file generated by a decision task.""" + + import taskgraph.decision + + try: + self.setup_logging() + start = time.monotonic() + ret = taskgraph.decision.taskgraph_decision(options) + end = time.monotonic() + if os.environ.get("MOZ_AUTOMATION") == "1": + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [ + { + "name": "decision", + "value": end - start, + "lowerIsBetter": True, + "shouldAlert": True, + "subtests": [], + } + ], + } + print( + "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)), + file=sys.stderr, + ) + return ret + except Exception: + traceback.print_exc() + sys.exit(1) + + @SubCommand( + "taskgraph", + "cron", + description="Provide a pointer to the new `.cron.yml` handler.", + ) + def taskgraph_cron(self, **options): + print( + 'Handling of ".cron.yml" files has move to ' + "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision." + ) + sys.exit(1) + + @SubCommand( + "taskgraph", + "action-callback", + description="Run action callback used by action tasks", + ) + @CommandArgument( + "--root", + "-r", + default="taskcluster/ci", + help="root of the taskgraph definition relative to topsrcdir", + ) + def action_callback(self, **options): + from taskgraph.actions import trigger_action_callback + from taskgraph.actions.util import get_parameters + + try: + self.setup_logging() + + # the target task for this action (or null if it's a group action) + task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null")) + # the target task group for this action + task_group_id = os.environ.get("ACTION_TASK_GROUP_ID", None) + input = json.loads(os.environ.get("ACTION_INPUT", "null")) + callback = os.environ.get("ACTION_CALLBACK", None) + root = options["root"] + + parameters = get_parameters(task_group_id) + + return trigger_action_callback( + task_group_id=task_group_id, + task_id=task_id, + input=input, + callback=callback, + parameters=parameters, + root=root, + test=False, + ) + except Exception: + traceback.print_exc() + sys.exit(1) + + @SubCommand( + "taskgraph", + "test-action-callback", + description="Run an action callback in a testing mode", + ) + @CommandArgument( + "--root", + "-r", + default="taskcluster/ci", + help="root of the taskgraph definition relative to topsrcdir", + ) + @CommandArgument( + "--parameters", + "-p", + default="project=mozilla-central", + help="parameters file (.yml or .json; see " + "`taskcluster/docs/parameters.rst`)`", + ) + @CommandArgument( + "--task-id", default=None, help="TaskId to which the action applies" + ) + @CommandArgument( + "--task-group-id", default=None, help="TaskGroupId to which the action applies" + ) + @CommandArgument("--input", default=None, help="Action input (.yml or .json)") + @CommandArgument( + "callback", default=None, help="Action callback name (Python function name)" + ) + def test_action_callback(self, **options): + import taskgraph.parameters + import taskgraph.actions + from taskgraph.util import yaml + + def load_data(filename): + with open(filename) as f: + if filename.endswith(".yml"): + return yaml.load_stream(f) + elif filename.endswith(".json"): + return json.load(f) + else: + raise Exception("unknown filename {}".format(filename)) + + try: + self.setup_logging() + task_id = options["task_id"] + + if options["input"]: + input = load_data(options["input"]) + else: + input = None + + parameters = taskgraph.parameters.load_parameters_file( + options["parameters"], + strict=False, + # FIXME: There should be a way to parameterize this. + trust_domain="gecko", + ) + parameters.check() + + root = options["root"] + + return taskgraph.actions.trigger_action_callback( + task_group_id=options["task_group_id"], + task_id=task_id, + input=input, + callback=options["callback"], + parameters=parameters, + root=root, + test=True, + ) + except Exception: + traceback.print_exc() + sys.exit(1) + + def setup_logging(self, quiet=False, verbose=True): + """ + Set up Python logging for all loggers, sending results to stderr (so + that command output can be redirected easily) and adding the typical + mach timestamp. + """ + # remove the old terminal handler + old = self.log_manager.replace_terminal_handler(None) + + # re-add it, with level and fh set appropriately + if not quiet: + level = logging.DEBUG if verbose else logging.INFO + self.log_manager.add_terminal_logging( + fh=sys.stderr, + level=level, + write_interval=old.formatter.write_interval, + write_times=old.formatter.write_times, + ) + + # all of the taskgraph logging is unstructured logging + self.log_manager.enable_unstructured() + + def show_taskgraph(self, graph_attr, options): + import taskgraph.parameters + import taskgraph.generator + import taskgraph + + if options["fast"]: + taskgraph.fast = True + + try: + self.setup_logging(quiet=options["quiet"], verbose=options["verbose"]) + parameters = taskgraph.parameters.parameters_loader( + options["parameters"], + overrides={"target-kind": options.get("target_kind")}, + strict=False, + ) + + tgg = taskgraph.generator.TaskGraphGenerator( + root_dir=options.get("root"), + parameters=parameters, + ) + + tg = getattr(tgg, graph_attr) + + show_method = getattr( + self, "show_taskgraph_" + (options["format"] or "labels") + ) + tg = self.get_filtered_taskgraph(tg, options["tasks_regex"]) + + fh = options["output_file"] + if fh: + fh = open(fh, "w") + show_method(tg, file=fh) + except Exception: + traceback.print_exc() + sys.exit(1) + + def show_taskgraph_labels(self, taskgraph, file=None): + for index in taskgraph.graph.visit_postorder(): + print(taskgraph.tasks[index].label, file=file) + + def show_taskgraph_json(self, taskgraph, file=None): + print( + json.dumps( + taskgraph.to_json(), sort_keys=True, indent=2, separators=(",", ": ") + ), + file=file, + ) + + def get_filtered_taskgraph(self, taskgraph, tasksregex): + from taskgraph.graph import Graph + from taskgraph.taskgraph import TaskGraph + + """ + This class method filters all the tasks on basis of a regular expression + and returns a new TaskGraph object + """ + # return original taskgraph if no regular expression is passed + if not tasksregex: + return taskgraph + named_links_dict = taskgraph.graph.named_links_dict() + filteredtasks = {} + filterededges = set() + regexprogram = re.compile(tasksregex) + + for key in taskgraph.graph.visit_postorder(): + task = taskgraph.tasks[key] + if regexprogram.match(task.label): + filteredtasks[key] = task + for depname, dep in six.iteritems(named_links_dict[key]): + if regexprogram.match(dep): + filterededges.add((key, dep, depname)) + filtered_taskgraph = TaskGraph( + filteredtasks, Graph(set(filteredtasks), filterededges) + ) + return filtered_taskgraph + + def show_actions(self, options): + import taskgraph.parameters + import taskgraph.generator + import taskgraph + import taskgraph.actions + + try: + self.setup_logging(quiet=options["quiet"], verbose=options["verbose"]) + parameters = taskgraph.parameters.parameters_loader(options["parameters"]) + + tgg = taskgraph.generator.TaskGraphGenerator( + root_dir=options.get("root"), + parameters=parameters, + ) + + actions = taskgraph.actions.render_actions_json( + tgg.parameters, + tgg.graph_config, + decision_task_id="DECISION-TASK", + ) + print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": "))) + except Exception: + traceback.print_exc() + sys.exit(1) + + +@CommandProvider +class TaskClusterImagesProvider(MachCommandBase): + @Command( + "taskcluster-load-image", + category="ci", + description="Load a pre-built Docker image. Note that you need to " + "have docker installed and running for this to work.", + ) + @CommandArgument( + "--task-id", + help="Load the image at public/image.tar.zst in this task, " + "rather than searching the index", + ) + @CommandArgument( + "-t", + "--tag", + help="tag that the image should be loaded as. If not " + "image will be loaded with tag from the tarball", + metavar="name:tag", + ) + @CommandArgument( + "image_name", + nargs="?", + help="Load the image of this name based on the current " + "contents of the tree (as built for mozilla-central " + "or mozilla-inbound)", + ) + def load_image(self, image_name, task_id, tag): + from taskgraph.docker import load_image_by_name, load_image_by_task_id + + if not image_name and not task_id: + print("Specify either IMAGE-NAME or TASK-ID") + sys.exit(1) + try: + if task_id: + ok = load_image_by_task_id(task_id, tag) + else: + ok = load_image_by_name(image_name, tag) + if not ok: + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + @Command( + "taskcluster-build-image", category="ci", description="Build a Docker image" + ) + @CommandArgument("image_name", help="Name of the image to build") + @CommandArgument( + "-t", "--tag", help="tag that the image should be built as.", metavar="name:tag" + ) + @CommandArgument( + "--context-only", + help="File name the context tarball should be written to." + "with this option it will only build the context.tar.", + metavar="context.tar", + ) + def build_image(self, image_name, tag, context_only): + from taskgraph.docker import build_image, build_context + + try: + if context_only is None: + build_image(image_name, tag, os.environ) + else: + build_context(image_name, context_only, os.environ) + except Exception: + traceback.print_exc() + sys.exit(1) + + +@CommandProvider +class TaskClusterPartialsData(MachCommandBase): + @Command( + "release-history", + category="ci", + description="Query balrog for release history used by enable partials generation", + ) + @CommandArgument( + "-b", + "--branch", + help="The gecko project branch used in balrog, such as " + "mozilla-central, release, maple", + ) + @CommandArgument( + "--product", default="Firefox", help="The product identifier, such as 'Firefox'" + ) + def generate_partials_builds(self, product, branch): + from taskgraph.util.partials import populate_release_history + + try: + import yaml + + release_history = { + "release_history": populate_release_history(product, branch) + } + print( + yaml.safe_dump( + release_history, allow_unicode=True, default_flow_style=False + ) + ) + except Exception: + traceback.print_exc() + sys.exit(1) -- cgit v1.2.3