summaryrefslogtreecommitdiffstats
path: root/tools/mach_commands.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tools/mach_commands.py594
1 files changed, 594 insertions, 0 deletions
diff --git a/tools/mach_commands.py b/tools/mach_commands.py
new file mode 100644
index 0000000000..9b96047f71
--- /dev/null
+++ b/tools/mach_commands.py
@@ -0,0 +1,594 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import logging
+import subprocess
+import sys
+from datetime import datetime, timedelta
+from operator import itemgetter
+
+from mach.decorators import Command, CommandArgument, SubCommand
+from mozbuild.base import MozbuildObject
+
+
+def _get_busted_bugs(payload):
+ import requests
+
+ payload = dict(payload)
+ payload["include_fields"] = "id,summary,last_change_time,resolution"
+ payload["blocks"] = 1543241
+ response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
+ response.raise_for_status()
+ return response.json().get("bugs", [])
+
+
+@Command(
+ "busted",
+ category="misc",
+ description="Query known bugs in our tooling, and file new ones.",
+)
+def busted_default(command_context):
+ unresolved = _get_busted_bugs({"resolution": "---"})
+ creation_time = datetime.now() - timedelta(days=15)
+ creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
+ resolved = _get_busted_bugs({"creation_time": creation_time})
+ resolved = [bug for bug in resolved if bug["resolution"]]
+ all_bugs = sorted(
+ unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
+ )
+ if all_bugs:
+ for bug in all_bugs:
+ print(
+ "[%s] Bug %s - %s"
+ % (
+ "UNRESOLVED"
+ if not bug["resolution"]
+ else "RESOLVED - %s" % bug["resolution"],
+ bug["id"],
+ bug["summary"],
+ )
+ )
+ else:
+ print("No known tooling issues found.")
+
+
+@SubCommand("busted", "file", description="File a bug for busted tooling.")
+@CommandArgument(
+ "against",
+ help=(
+ "The specific mach command that is busted (i.e. if you encountered "
+ "an error with `mach build`, run `mach busted file build`). If "
+ "the issue is not connected to any particular mach command, you "
+ "can also run `mach busted file general`."
+ ),
+)
+def busted_file(command_context, against):
+ import webbrowser
+
+ if (
+ against != "general"
+ and against not in command_context._mach_context.commands.command_handlers
+ ):
+ print(
+ "%s is not a valid value for `against`. `against` must be "
+ "the name of a `mach` command, or else the string "
+ '"general".' % against
+ )
+ return 1
+
+ if against == "general":
+ product = "Firefox Build System"
+ component = "General"
+ else:
+ import inspect
+
+ import mozpack.path as mozpath
+
+ # Look up the file implementing that command, then cross-refernce
+ # moz.build files to get the product/component.
+ handler = command_context._mach_context.commands.command_handlers[against]
+ sourcefile = mozpath.relpath(
+ inspect.getsourcefile(handler.func), command_context.topsrcdir
+ )
+ reader = command_context.mozbuild_reader(config_mode="empty")
+ try:
+ res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
+ product, component = res.product, res.component
+ except TypeError:
+ # The file might not have a bug set.
+ product = "Firefox Build System"
+ component = "General"
+
+ uri = (
+ "https://bugzilla.mozilla.org/enter_bug.cgi?"
+ "product=%s&component=%s&blocked=1543241" % (product, component)
+ )
+ webbrowser.open_new_tab(uri)
+
+
+MACH_PASTEBIN_DURATIONS = {
+ "onetime": "onetime",
+ "hour": "3600",
+ "day": "86400",
+ "week": "604800",
+ "month": "2073600",
+}
+
+EXTENSION_TO_HIGHLIGHTER = {
+ ".hgrc": "ini",
+ "Dockerfile": "docker",
+ "Makefile": "make",
+ "applescript": "applescript",
+ "arduino": "arduino",
+ "bash": "bash",
+ "bat": "bat",
+ "c": "c",
+ "clojure": "clojure",
+ "cmake": "cmake",
+ "coffee": "coffee-script",
+ "console": "console",
+ "cpp": "cpp",
+ "cs": "csharp",
+ "css": "css",
+ "cu": "cuda",
+ "cuda": "cuda",
+ "dart": "dart",
+ "delphi": "delphi",
+ "diff": "diff",
+ "django": "django",
+ "docker": "docker",
+ "elixir": "elixir",
+ "erlang": "erlang",
+ "go": "go",
+ "h": "c",
+ "handlebars": "handlebars",
+ "haskell": "haskell",
+ "hs": "haskell",
+ "html": "html",
+ "ini": "ini",
+ "ipy": "ipythonconsole",
+ "ipynb": "ipythonconsole",
+ "irc": "irc",
+ "j2": "django",
+ "java": "java",
+ "js": "js",
+ "json": "json",
+ "jsx": "jsx",
+ "kt": "kotlin",
+ "less": "less",
+ "lisp": "common-lisp",
+ "lsp": "common-lisp",
+ "lua": "lua",
+ "m": "objective-c",
+ "make": "make",
+ "matlab": "matlab",
+ "md": "_markdown",
+ "nginx": "nginx",
+ "numpy": "numpy",
+ "patch": "diff",
+ "perl": "perl",
+ "php": "php",
+ "pm": "perl",
+ "postgresql": "postgresql",
+ "py": "python",
+ "rb": "rb",
+ "rs": "rust",
+ "rst": "rst",
+ "sass": "sass",
+ "scss": "scss",
+ "sh": "bash",
+ "sol": "sol",
+ "sql": "sql",
+ "swift": "swift",
+ "tex": "tex",
+ "typoscript": "typoscript",
+ "vim": "vim",
+ "xml": "xml",
+ "xslt": "xslt",
+ "yaml": "yaml",
+ "yml": "yaml",
+}
+
+
+def guess_highlighter_from_path(path):
+ """Return a known highlighter from a given path
+
+ Attempt to select a highlighter by checking the file extension in the mapping
+ of extensions to highlighter. If that fails, attempt to pass the basename of
+ the file. Return `_code` as the default highlighter if that fails.
+ """
+ import os
+
+ _name, ext = os.path.splitext(path)
+
+ if ext.startswith("."):
+ ext = ext[1:]
+
+ if ext in EXTENSION_TO_HIGHLIGHTER:
+ return EXTENSION_TO_HIGHLIGHTER[ext]
+
+ basename = os.path.basename(path)
+
+ return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code")
+
+
+PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
+
+PASTEMO_URL = "https://paste.mozilla.org/api/"
+
+
+@Command(
+ "pastebin",
+ category="misc",
+ description="Command line interface to paste.mozilla.org.",
+)
+@CommandArgument(
+ "--list-highlighters",
+ action="store_true",
+ help="List known highlighters and exit",
+)
+@CommandArgument(
+ "--highlighter", default=None, help="Syntax highlighting to use for paste"
+)
+@CommandArgument(
+ "--expires",
+ default="week",
+ choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
+ help="Expire paste after given time duration (default: %(default)s)",
+)
+@CommandArgument(
+ "--verbose",
+ action="store_true",
+ help="Print extra info such as selected syntax highlighter",
+)
+@CommandArgument(
+ "path",
+ nargs="?",
+ default=None,
+ help="Path to file for upload to paste.mozilla.org",
+)
+def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path):
+ """Command line interface to `paste.mozilla.org`.
+
+ Takes either a filename whose content should be pasted, or reads
+ content from standard input. If a highlighter is specified it will
+ be used, otherwise the file name will be used to determine an
+ appropriate highlighter.
+ """
+
+ import requests
+
+ def verbose_print(*args, **kwargs):
+ """Print a string if `--verbose` flag is set"""
+ if verbose:
+ print(*args, **kwargs)
+
+ # Show known highlighters and exit.
+ if list_highlighters:
+ lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
+ print("Available lexers:\n - %s" % "\n - ".join(sorted(lexers)))
+ return 0
+
+ # Get a correct expiry value.
+ try:
+ verbose_print("Setting expiry from %s" % expires)
+ expires = MACH_PASTEBIN_DURATIONS[expires]
+ verbose_print("Using %s as expiry" % expires)
+ except KeyError:
+ print(
+ "%s is not a valid duration.\n"
+ "(hint: try one of %s)"
+ % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys()))
+ )
+ return 1
+
+ data = {
+ "format": "json",
+ "expires": expires,
+ }
+
+ # Get content to be pasted.
+ if path:
+ verbose_print("Reading content from %s" % path)
+ try:
+ with open(path, "r") as f:
+ content = f.read()
+ except IOError:
+ print("ERROR. No such file %s" % path)
+ return 1
+
+ lexer = guess_highlighter_from_path(path)
+ if lexer:
+ data["lexer"] = lexer
+ else:
+ verbose_print("Reading content from stdin")
+ content = sys.stdin.read()
+
+ # Assert the length of content to be posted does not exceed the maximum.
+ content_length = len(content)
+ verbose_print("Checking size of content is okay (%d)" % content_length)
+ if content_length > PASTEMO_MAX_CONTENT_LENGTH:
+ print(
+ "Paste content is too large (%d, maximum %d)"
+ % (content_length, PASTEMO_MAX_CONTENT_LENGTH)
+ )
+ return 1
+
+ data["content"] = content
+
+ # Highlight as specified language, overwriting value set from filename.
+ if highlighter:
+ verbose_print("Setting %s as highlighter" % highlighter)
+ data["lexer"] = highlighter
+
+ try:
+ verbose_print("Sending request to %s" % PASTEMO_URL)
+ resp = requests.post(PASTEMO_URL, data=data)
+
+ # Error code should always be 400.
+ # Response content will include a helpful error message,
+ # so print it here (for example, if an invalid highlighter is
+ # provided, it will return a list of valid highlighters).
+ if resp.status_code >= 400:
+ print("Error code %d: %s" % (resp.status_code, resp.content))
+ return 1
+
+ verbose_print("Pasted successfully")
+
+ response_json = resp.json()
+
+ verbose_print("Paste highlighted as %s" % response_json["lexer"])
+ print(response_json["url"])
+
+ return 0
+ except Exception as e:
+ print("ERROR. Paste failed.")
+ print("%s" % e)
+ return 1
+
+
+class PypiBasedTool:
+ """
+ Helper for loading a tool that is hosted on pypi. The package is expected
+ to expose a `mach_interface` module which has `new_release_on_pypi`,
+ `parser`, and `run` functions.
+ """
+
+ def __init__(self, module_name, pypi_name=None):
+ self.name = module_name
+ self.pypi_name = pypi_name or module_name
+
+ def _import(self):
+ # Lazy loading of the tools mach interface.
+ # Note that only the mach_interface module should be used from this file.
+ import importlib
+
+ try:
+ return importlib.import_module("%s.mach_interface" % self.name)
+ except ImportError:
+ return None
+
+ def create_parser(self, subcommand=None):
+ # Create the command line parser.
+ # If the tool is not installed, or not up to date, it will
+ # first be installed.
+ cmd = MozbuildObject.from_environment()
+ cmd.activate_virtualenv()
+ tool = self._import()
+ if not tool:
+ # The tool is not here at all, install it
+ cmd.virtualenv_manager.install_pip_package(self.pypi_name)
+ print(
+ "%s was installed. please re-run your"
+ " command. If you keep getting this message please "
+ " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
+ )
+ else:
+ # Check if there is a new release available
+ release = tool.new_release_on_pypi()
+ if release:
+ print(release)
+ # there is one, so install it. Note that install_pip_package
+ # does not work here, so just run pip directly.
+ subprocess.check_call(
+ [
+ cmd.virtualenv_manager.python_path,
+ "-m",
+ "pip",
+ "install",
+ f"{self.pypi_name}=={release}",
+ ]
+ )
+ print(
+ "%s was updated to version %s. please"
+ " re-run your command." % (self.pypi_name, release)
+ )
+ else:
+ # Tool is up to date, return the parser.
+ if subcommand:
+ return tool.parser(subcommand)
+ else:
+ return tool.parser()
+ # exit if we updated or installed mozregression because
+ # we may have already imported mozregression and running it
+ # as this may cause issues.
+ sys.exit(0)
+
+ def run(self, **options):
+ tool = self._import()
+ tool.run(options)
+
+
+def mozregression_create_parser():
+ # Create the mozregression command line parser.
+ # if mozregression is not installed, or not up to date, it will
+ # first be installed.
+ loader = PypiBasedTool("mozregression")
+ return loader.create_parser()
+
+
+@Command(
+ "mozregression",
+ category="misc",
+ description="Regression range finder for nightly and inbound builds.",
+ parser=mozregression_create_parser,
+)
+def run(command_context, **options):
+ command_context.activate_virtualenv()
+ mozregression = PypiBasedTool("mozregression")
+ mozregression.run(**options)
+
+
+@Command(
+ "node",
+ category="devenv",
+ description="Run the NodeJS interpreter used for building.",
+)
+@CommandArgument("args", nargs=argparse.REMAINDER)
+def node(command_context, args):
+ from mozbuild.nodeutil import find_node_executable
+
+ # Avoid logging the command
+ command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+
+ node_path, _ = find_node_executable()
+
+ return command_context.run_process(
+ [node_path] + args,
+ pass_thru=True, # Allow user to run Node interactively.
+ ensure_exit_code=False, # Don't throw on non-zero exit code.
+ )
+
+
+@Command(
+ "npm",
+ category="devenv",
+ description="Run the npm executable from the NodeJS used for building.",
+)
+@CommandArgument("args", nargs=argparse.REMAINDER)
+def npm(command_context, args):
+ from mozbuild.nodeutil import find_npm_executable
+
+ # Avoid logging the command
+ command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+
+ import os
+
+ # Add node and npm from mozbuild to front of system path
+ #
+ # This isn't pretty, but npm currently executes itself with
+ # `#!/usr/bin/env node`, which means it just uses the node in the
+ # current PATH. As a result, stuff gets built wrong and installed
+ # in the wrong places and probably other badness too without this:
+ npm_path, _ = find_npm_executable()
+ if not npm_path:
+ exit(-1, "could not find npm executable")
+ path = os.path.abspath(os.path.dirname(npm_path))
+ os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
+
+ # karma-firefox-launcher needs the path to firefox binary.
+ firefox_bin = command_context.get_binary_path(validate_exists=False)
+ if os.path.exists(firefox_bin):
+ os.environ["FIREFOX_BIN"] = firefox_bin
+
+ return command_context.run_process(
+ [npm_path, "--scripts-prepend-node-path=auto"] + args,
+ pass_thru=True, # Avoid eating npm output/error messages
+ ensure_exit_code=False, # Don't throw on non-zero exit code.
+ )
+
+
+def logspam_create_parser(subcommand):
+ # Create the logspam command line parser.
+ # if logspam is not installed, or not up to date, it will
+ # first be installed.
+ loader = PypiBasedTool("logspam", "mozilla-log-spam")
+ return loader.create_parser(subcommand)
+
+
+from functools import partial
+
+
+@Command(
+ "logspam",
+ category="misc",
+ description="Warning categorizer for treeherder test runs.",
+)
+def logspam(command_context):
+ pass
+
+
+@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
+def report(command_context, **options):
+ command_context.activate_virtualenv()
+ logspam = PypiBasedTool("logspam")
+ logspam.run(command="report", **options)
+
+
+@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
+def bisect(command_context, **options):
+ command_context.activate_virtualenv()
+ logspam = PypiBasedTool("logspam")
+ logspam.run(command="bisect", **options)
+
+
+@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
+def create(command_context, **options):
+ command_context.activate_virtualenv()
+ logspam = PypiBasedTool("logspam")
+ logspam.run(command="file", **options)
+
+
+# mots_loader will be used when running commands and subcommands, as well as
+# when creating the parsers.
+mots_loader = PypiBasedTool("mots")
+
+
+def mots_create_parser(subcommand=None):
+ return mots_loader.create_parser(subcommand)
+
+
+def mots_run_subcommand(command, command_context, **options):
+ command_context.activate_virtualenv()
+ mots_loader.run(command=command, **options)
+
+
+class motsSubCommand(SubCommand):
+ """A helper subclass that reduces repitition when defining subcommands."""
+
+ def __init__(self, subcommand):
+ super().__init__(
+ "mots",
+ subcommand,
+ parser=partial(mots_create_parser, subcommand),
+ )
+
+
+@Command(
+ "mots",
+ category="misc",
+ description="Manage module information in-tree using the mots CLI.",
+ parser=mots_create_parser,
+)
+def mots(command_context, **options):
+ """The main mots command call."""
+ command_context.activate_virtualenv()
+ mots_loader.run(**options)
+
+
+# Define subcommands that will be proxied through mach.
+for sc in (
+ "clean",
+ "check-hashes",
+ "export",
+ "export-and-clean",
+ "module",
+ "query",
+ "settings",
+ "user",
+ "validate",
+):
+ # Pass through args and kwargs, but add the subcommand string as the first argument.
+ motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw))