summaryrefslogtreecommitdiffstats
path: root/taskcluster/mach_commands.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /taskcluster/mach_commands.py
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/mach_commands.py')
-rw-r--r--taskcluster/mach_commands.py416
1 files changed, 416 insertions, 0 deletions
diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py
new file mode 100644
index 0000000000..b752cb7d16
--- /dev/null
+++ b/taskcluster/mach_commands.py
@@ -0,0 +1,416 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+import traceback
+from functools import partial
+
+import gecko_taskgraph.main
+from gecko_taskgraph.main import commands as taskgraph_commands
+from mach.decorators import Command, CommandArgument, SettingsProvider, SubCommand
+
+logger = logging.getLogger("taskcluster")
+
+
+@SettingsProvider
+class TaskgraphConfig(object):
+ @classmethod
+ def config_settings(cls):
+ return [
+ (
+ "taskgraph.diffcmd",
+ "string",
+ "The command to run with `./mach taskgraph --diff`",
+ "diff --report-identical-files "
+ "--label={attr}@{base} --label={attr}@{cur} -U20",
+ {},
+ )
+ ]
+
+
+def strtobool(value):
+ """Convert string to boolean.
+
+ Wraps "distutils.util.strtobool", deferring the import of the package
+ in case it's not installed. Otherwise, we have a "chicken and egg problem" where
+ |mach bootstrap| would install the required package to enable "distutils.util", but
+ it can't because mach fails to interpret this file.
+ """
+ from distutils.util import strtobool
+
+ return bool(strtobool(value))
+
+
+def get_taskgraph_command_parser(name):
+ """Given a command name, obtain its argument parser.
+
+ Args:
+ name (str): Name of the command.
+
+ Returns:
+ ArgumentParser: An ArgumentParser instance.
+ """
+ command = taskgraph_commands[name]
+ parser = argparse.ArgumentParser()
+ for arg in command.func.args:
+ parser.add_argument(*arg[0], **arg[1])
+
+ parser.set_defaults(func=command.func, **command.defaults)
+ return parser
+
+
+def get_taskgraph_decision_parser():
+ parser = get_taskgraph_command_parser("decision")
+
+ extra_args = [
+ (
+ ["--optimize-target-tasks"],
+ {
+ "type": lambda flag: strtobool(flag),
+ "nargs": "?",
+ "const": "true",
+ "help": "If specified, this indicates whether the target "
+ "tasks are eligible for optimization. Otherwise, the default "
+ "for the project is used.",
+ },
+ ),
+ (
+ ["--include-push-tasks"],
+ {
+ "action": "store_true",
+ "help": "Whether tasks from the on-push graph should be re-used "
+ "in this graph. This allows cron graphs to avoid rebuilding "
+ "jobs that were built on-push.",
+ },
+ ),
+ (
+ ["--rebuild-kind"],
+ {
+ "dest": "rebuild_kinds",
+ "action": "append",
+ "default": argparse.SUPPRESS,
+ "help": "Kinds that should not be re-used from the on-push graph.",
+ },
+ ),
+ ]
+ for arg in extra_args:
+ parser.add_argument(*arg[0], **arg[1])
+
+ return parser
+
+
+@Command(
+ "taskgraph",
+ category="ci",
+ description="Manipulate TaskCluster task graphs defined in-tree",
+)
+def taskgraph_command(command_context):
+ """The taskgraph subcommands all relate to the generation of task graphs
+ for Gecko continuous integration. A task graph is a set of tasks linked
+ by dependencies: for example, a binary must be built before it is tested,
+ and that build may further depend on various toolchains, libraries, etc.
+ """
+
+
+@SubCommand(
+ "taskgraph",
+ "tasks",
+ description="Show all tasks in the taskgraph",
+ parser=partial(get_taskgraph_command_parser, "tasks"),
+)
+def taskgraph_tasks(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+@SubCommand(
+ "taskgraph",
+ "full",
+ description="Show the full taskgraph",
+ parser=partial(get_taskgraph_command_parser, "full"),
+)
+def taskgraph_full(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+@SubCommand(
+ "taskgraph",
+ "target",
+ description="Show the target task set",
+ parser=partial(get_taskgraph_command_parser, "target"),
+)
+def taskgraph_target(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+@SubCommand(
+ "taskgraph",
+ "target-graph",
+ description="Show the target taskgraph",
+ parser=partial(get_taskgraph_command_parser, "target-graph"),
+)
+def taskgraph_target_graph(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+@SubCommand(
+ "taskgraph",
+ "optimized",
+ description="Show the optimized taskgraph",
+ parser=partial(get_taskgraph_command_parser, "optimized"),
+)
+def taskgraph_optimized(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+@SubCommand(
+ "taskgraph",
+ "morphed",
+ description="Show the morphed taskgraph",
+ parser=partial(get_taskgraph_command_parser, "morphed"),
+)
+def taskgraph_morphed(command_context, **options):
+ return run_show_taskgraph(command_context, **options)
+
+
+def run_show_taskgraph(command_context, **options):
+ # There are cases where we don't want to set up mach logging (e.g logs
+ # are being redirected to disk). By monkeypatching the 'setup_logging'
+ # function we can let 'taskgraph.main' decide whether or not to log to
+ # the terminal.
+ gecko_taskgraph.main.setup_logging = partial(
+ setup_logging,
+ command_context,
+ quiet=options["quiet"],
+ verbose=options["verbose"],
+ )
+ show_taskgraph = options.pop("func")
+ return show_taskgraph(options)
+
+
+@SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
+@CommandArgument(
+ "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
+)
+@CommandArgument(
+ "--quiet", "-q", action="store_true", help="suppress all logging output"
+)
+@CommandArgument(
+ "--verbose",
+ "-v",
+ action="store_true",
+ help="include debug-level logging output",
+)
+@CommandArgument(
+ "--parameters",
+ "-p",
+ default="project=mozilla-central",
+ help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
+)
+def taskgraph_actions(command_context, **options):
+ return show_actions(command_context, options)
+
+
+@SubCommand(
+ "taskgraph",
+ "decision",
+ description="Run the decision task",
+ parser=get_taskgraph_decision_parser,
+)
+def taskgraph_decision(command_context, **options):
+ """Run the decision task: generate a task graph and submit to
+ TaskCluster. This is only meant to be called within decision tasks,
+ and requires a great many arguments. Commands like `mach taskgraph
+ optimized` are better suited to use on the command line, and can take
+ the parameters file generated by a decision task."""
+ try:
+ setup_logging(command_context)
+ start = time.monotonic()
+ ret = taskgraph_commands["decision"].func(options)
+ end = time.monotonic()
+ if os.environ.get("MOZ_AUTOMATION") == "1":
+ perfherder_data = {
+ "framework": {"name": "build_metrics"},
+ "suites": [
+ {
+ "name": "decision",
+ "value": end - start,
+ "lowerIsBetter": True,
+ "shouldAlert": True,
+ "subtests": [],
+ }
+ ],
+ }
+ print(
+ "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
+ file=sys.stderr,
+ )
+ return ret
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+
+@SubCommand(
+ "taskgraph",
+ "cron",
+ description="Provide a pointer to the new `.cron.yml` handler.",
+)
+def taskgraph_cron(command_context, **options):
+ print(
+ 'Handling of ".cron.yml" files has move to '
+ "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
+ )
+ sys.exit(1)
+
+
+@SubCommand(
+ "taskgraph",
+ "action-callback",
+ description="Run action callback used by action tasks",
+ parser=partial(get_taskgraph_command_parser, "action-callback"),
+)
+def action_callback(command_context, **options):
+ setup_logging(command_context)
+ taskgraph_commands["action-callback"].func(options)
+
+
+@SubCommand(
+ "taskgraph",
+ "test-action-callback",
+ description="Run an action callback in a testing mode",
+ parser=partial(get_taskgraph_command_parser, "test-action-callback"),
+)
+def test_action_callback(command_context, **options):
+ setup_logging(command_context)
+
+ if not options["parameters"]:
+ options["parameters"] = "project=mozilla-central"
+
+ taskgraph_commands["test-action-callback"].func(options)
+
+
+def setup_logging(command_context, quiet=False, verbose=True):
+ """
+ Set up Python logging for all loggers, sending results to stderr (so
+ that command output can be redirected easily) and adding the typical
+ mach timestamp.
+ """
+ # remove the old terminal handler
+ old = command_context.log_manager.replace_terminal_handler(None)
+
+ # re-add it, with level and fh set appropriately
+ if not quiet:
+ level = logging.DEBUG if verbose else logging.INFO
+ command_context.log_manager.add_terminal_logging(
+ fh=sys.stderr,
+ level=level,
+ write_interval=old.formatter.write_interval,
+ write_times=old.formatter.write_times,
+ )
+
+ # all of the taskgraph logging is unstructured logging
+ command_context.log_manager.enable_unstructured()
+
+
+def show_actions(command_context, options):
+ import gecko_taskgraph
+ import gecko_taskgraph.actions
+ from taskgraph.generator import TaskGraphGenerator
+ from taskgraph.parameters import parameters_loader
+
+ try:
+ setup_logging(
+ command_context, quiet=options["quiet"], verbose=options["verbose"]
+ )
+ parameters = parameters_loader(options["parameters"])
+
+ tgg = TaskGraphGenerator(
+ root_dir=options.get("root"),
+ parameters=parameters,
+ )
+
+ actions = gecko_taskgraph.actions.render_actions_json(
+ tgg.parameters,
+ tgg.graph_config,
+ decision_task_id="DECISION-TASK",
+ )
+ print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+
+@Command(
+ "taskcluster-load-image",
+ category="ci",
+ description="Load a pre-built Docker image. Note that you need to "
+ "have docker installed and running for this to work.",
+ parser=partial(get_taskgraph_command_parser, "load-image"),
+)
+def load_image(command_context, **kwargs):
+ taskgraph_commands["load-image"].func(kwargs)
+
+
+@Command(
+ "taskcluster-build-image",
+ category="ci",
+ description="Build a Docker image",
+ parser=partial(get_taskgraph_command_parser, "build-image"),
+)
+def build_image(command_context, **kwargs):
+ try:
+ taskgraph_commands["build-image"].func(kwargs)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+
+@Command(
+ "taskcluster-image-digest",
+ category="ci",
+ description="Print the digest of the image of this name based on the "
+ "current contents of the tree.",
+ parser=partial(get_taskgraph_command_parser, "build-image"),
+)
+def image_digest(command_context, **kwargs):
+ taskgraph_commands["image-digest"].func(kwargs)
+
+
+@Command(
+ "release-history",
+ category="ci",
+ description="Query balrog for release history used by enable partials generation",
+)
+@CommandArgument(
+ "-b",
+ "--branch",
+ help="The gecko project branch used in balrog, such as "
+ "mozilla-central, release, maple",
+)
+@CommandArgument(
+ "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
+)
+def generate_partials_builds(command_context, product, branch):
+ from gecko_taskgraph.util.partials import populate_release_history
+
+ try:
+ import yaml
+
+ release_history = {"release_history": populate_release_history(product, branch)}
+ print(
+ yaml.safe_dump(
+ release_history, allow_unicode=True, default_flow_style=False
+ )
+ )
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)