diff options
Diffstat (limited to 'tools/mach_commands.py')
| -rw-r--r-- | tools/mach_commands.py | 594 |
1 files changed, 594 insertions, 0 deletions
diff --git a/tools/mach_commands.py b/tools/mach_commands.py new file mode 100644 index 0000000000..9b96047f71 --- /dev/null +++ b/tools/mach_commands.py @@ -0,0 +1,594 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import subprocess +import sys +from datetime import datetime, timedelta +from operator import itemgetter + +from mach.decorators import Command, CommandArgument, SubCommand +from mozbuild.base import MozbuildObject + + +def _get_busted_bugs(payload): + import requests + + payload = dict(payload) + payload["include_fields"] = "id,summary,last_change_time,resolution" + payload["blocks"] = 1543241 + response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload) + response.raise_for_status() + return response.json().get("bugs", []) + + +@Command( + "busted", + category="misc", + description="Query known bugs in our tooling, and file new ones.", +) +def busted_default(command_context): + unresolved = _get_busted_bugs({"resolution": "---"}) + creation_time = datetime.now() - timedelta(days=15) + creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ") + resolved = _get_busted_bugs({"creation_time": creation_time}) + resolved = [bug for bug in resolved if bug["resolution"]] + all_bugs = sorted( + unresolved + resolved, key=itemgetter("last_change_time"), reverse=True + ) + if all_bugs: + for bug in all_bugs: + print( + "[%s] Bug %s - %s" + % ( + "UNRESOLVED" + if not bug["resolution"] + else "RESOLVED - %s" % bug["resolution"], + bug["id"], + bug["summary"], + ) + ) + else: + print("No known tooling issues found.") + + +@SubCommand("busted", "file", description="File a bug for busted tooling.") +@CommandArgument( + "against", + help=( + "The specific mach command that is busted (i.e. if you encountered " + "an error with `mach build`, run `mach busted file build`). If " + "the issue is not connected to any particular mach command, you " + "can also run `mach busted file general`." + ), +) +def busted_file(command_context, against): + import webbrowser + + if ( + against != "general" + and against not in command_context._mach_context.commands.command_handlers + ): + print( + "%s is not a valid value for `against`. `against` must be " + "the name of a `mach` command, or else the string " + '"general".' % against + ) + return 1 + + if against == "general": + product = "Firefox Build System" + component = "General" + else: + import inspect + + import mozpack.path as mozpath + + # Look up the file implementing that command, then cross-refernce + # moz.build files to get the product/component. + handler = command_context._mach_context.commands.command_handlers[against] + sourcefile = mozpath.relpath( + inspect.getsourcefile(handler.func), command_context.topsrcdir + ) + reader = command_context.mozbuild_reader(config_mode="empty") + try: + res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"] + product, component = res.product, res.component + except TypeError: + # The file might not have a bug set. + product = "Firefox Build System" + component = "General" + + uri = ( + "https://bugzilla.mozilla.org/enter_bug.cgi?" + "product=%s&component=%s&blocked=1543241" % (product, component) + ) + webbrowser.open_new_tab(uri) + + +MACH_PASTEBIN_DURATIONS = { + "onetime": "onetime", + "hour": "3600", + "day": "86400", + "week": "604800", + "month": "2073600", +} + +EXTENSION_TO_HIGHLIGHTER = { + ".hgrc": "ini", + "Dockerfile": "docker", + "Makefile": "make", + "applescript": "applescript", + "arduino": "arduino", + "bash": "bash", + "bat": "bat", + "c": "c", + "clojure": "clojure", + "cmake": "cmake", + "coffee": "coffee-script", + "console": "console", + "cpp": "cpp", + "cs": "csharp", + "css": "css", + "cu": "cuda", + "cuda": "cuda", + "dart": "dart", + "delphi": "delphi", + "diff": "diff", + "django": "django", + "docker": "docker", + "elixir": "elixir", + "erlang": "erlang", + "go": "go", + "h": "c", + "handlebars": "handlebars", + "haskell": "haskell", + "hs": "haskell", + "html": "html", + "ini": "ini", + "ipy": "ipythonconsole", + "ipynb": "ipythonconsole", + "irc": "irc", + "j2": "django", + "java": "java", + "js": "js", + "json": "json", + "jsx": "jsx", + "kt": "kotlin", + "less": "less", + "lisp": "common-lisp", + "lsp": "common-lisp", + "lua": "lua", + "m": "objective-c", + "make": "make", + "matlab": "matlab", + "md": "_markdown", + "nginx": "nginx", + "numpy": "numpy", + "patch": "diff", + "perl": "perl", + "php": "php", + "pm": "perl", + "postgresql": "postgresql", + "py": "python", + "rb": "rb", + "rs": "rust", + "rst": "rst", + "sass": "sass", + "scss": "scss", + "sh": "bash", + "sol": "sol", + "sql": "sql", + "swift": "swift", + "tex": "tex", + "typoscript": "typoscript", + "vim": "vim", + "xml": "xml", + "xslt": "xslt", + "yaml": "yaml", + "yml": "yaml", +} + + +def guess_highlighter_from_path(path): + """Return a known highlighter from a given path + + Attempt to select a highlighter by checking the file extension in the mapping + of extensions to highlighter. If that fails, attempt to pass the basename of + the file. Return `_code` as the default highlighter if that fails. + """ + import os + + _name, ext = os.path.splitext(path) + + if ext.startswith("."): + ext = ext[1:] + + if ext in EXTENSION_TO_HIGHLIGHTER: + return EXTENSION_TO_HIGHLIGHTER[ext] + + basename = os.path.basename(path) + + return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code") + + +PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024 + +PASTEMO_URL = "https://paste.mozilla.org/api/" + + +@Command( + "pastebin", + category="misc", + description="Command line interface to paste.mozilla.org.", +) +@CommandArgument( + "--list-highlighters", + action="store_true", + help="List known highlighters and exit", +) +@CommandArgument( + "--highlighter", default=None, help="Syntax highlighting to use for paste" +) +@CommandArgument( + "--expires", + default="week", + choices=sorted(MACH_PASTEBIN_DURATIONS.keys()), + help="Expire paste after given time duration (default: %(default)s)", +) +@CommandArgument( + "--verbose", + action="store_true", + help="Print extra info such as selected syntax highlighter", +) +@CommandArgument( + "path", + nargs="?", + default=None, + help="Path to file for upload to paste.mozilla.org", +) +def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path): + """Command line interface to `paste.mozilla.org`. + + Takes either a filename whose content should be pasted, or reads + content from standard input. If a highlighter is specified it will + be used, otherwise the file name will be used to determine an + appropriate highlighter. + """ + + import requests + + def verbose_print(*args, **kwargs): + """Print a string if `--verbose` flag is set""" + if verbose: + print(*args, **kwargs) + + # Show known highlighters and exit. + if list_highlighters: + lexers = set(EXTENSION_TO_HIGHLIGHTER.values()) + print("Available lexers:\n - %s" % "\n - ".join(sorted(lexers))) + return 0 + + # Get a correct expiry value. + try: + verbose_print("Setting expiry from %s" % expires) + expires = MACH_PASTEBIN_DURATIONS[expires] + verbose_print("Using %s as expiry" % expires) + except KeyError: + print( + "%s is not a valid duration.\n" + "(hint: try one of %s)" + % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys())) + ) + return 1 + + data = { + "format": "json", + "expires": expires, + } + + # Get content to be pasted. + if path: + verbose_print("Reading content from %s" % path) + try: + with open(path, "r") as f: + content = f.read() + except IOError: + print("ERROR. No such file %s" % path) + return 1 + + lexer = guess_highlighter_from_path(path) + if lexer: + data["lexer"] = lexer + else: + verbose_print("Reading content from stdin") + content = sys.stdin.read() + + # Assert the length of content to be posted does not exceed the maximum. + content_length = len(content) + verbose_print("Checking size of content is okay (%d)" % content_length) + if content_length > PASTEMO_MAX_CONTENT_LENGTH: + print( + "Paste content is too large (%d, maximum %d)" + % (content_length, PASTEMO_MAX_CONTENT_LENGTH) + ) + return 1 + + data["content"] = content + + # Highlight as specified language, overwriting value set from filename. + if highlighter: + verbose_print("Setting %s as highlighter" % highlighter) + data["lexer"] = highlighter + + try: + verbose_print("Sending request to %s" % PASTEMO_URL) + resp = requests.post(PASTEMO_URL, data=data) + + # Error code should always be 400. + # Response content will include a helpful error message, + # so print it here (for example, if an invalid highlighter is + # provided, it will return a list of valid highlighters). + if resp.status_code >= 400: + print("Error code %d: %s" % (resp.status_code, resp.content)) + return 1 + + verbose_print("Pasted successfully") + + response_json = resp.json() + + verbose_print("Paste highlighted as %s" % response_json["lexer"]) + print(response_json["url"]) + + return 0 + except Exception as e: + print("ERROR. Paste failed.") + print("%s" % e) + return 1 + + +class PypiBasedTool: + """ + Helper for loading a tool that is hosted on pypi. The package is expected + to expose a `mach_interface` module which has `new_release_on_pypi`, + `parser`, and `run` functions. + """ + + def __init__(self, module_name, pypi_name=None): + self.name = module_name + self.pypi_name = pypi_name or module_name + + def _import(self): + # Lazy loading of the tools mach interface. + # Note that only the mach_interface module should be used from this file. + import importlib + + try: + return importlib.import_module("%s.mach_interface" % self.name) + except ImportError: + return None + + def create_parser(self, subcommand=None): + # Create the command line parser. + # If the tool is not installed, or not up to date, it will + # first be installed. + cmd = MozbuildObject.from_environment() + cmd.activate_virtualenv() + tool = self._import() + if not tool: + # The tool is not here at all, install it + cmd.virtualenv_manager.install_pip_package(self.pypi_name) + print( + "%s was installed. please re-run your" + " command. If you keep getting this message please " + " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name) + ) + else: + # Check if there is a new release available + release = tool.new_release_on_pypi() + if release: + print(release) + # there is one, so install it. Note that install_pip_package + # does not work here, so just run pip directly. + subprocess.check_call( + [ + cmd.virtualenv_manager.python_path, + "-m", + "pip", + "install", + f"{self.pypi_name}=={release}", + ] + ) + print( + "%s was updated to version %s. please" + " re-run your command." % (self.pypi_name, release) + ) + else: + # Tool is up to date, return the parser. + if subcommand: + return tool.parser(subcommand) + else: + return tool.parser() + # exit if we updated or installed mozregression because + # we may have already imported mozregression and running it + # as this may cause issues. + sys.exit(0) + + def run(self, **options): + tool = self._import() + tool.run(options) + + +def mozregression_create_parser(): + # Create the mozregression command line parser. + # if mozregression is not installed, or not up to date, it will + # first be installed. + loader = PypiBasedTool("mozregression") + return loader.create_parser() + + +@Command( + "mozregression", + category="misc", + description="Regression range finder for nightly and inbound builds.", + parser=mozregression_create_parser, +) +def run(command_context, **options): + command_context.activate_virtualenv() + mozregression = PypiBasedTool("mozregression") + mozregression.run(**options) + + +@Command( + "node", + category="devenv", + description="Run the NodeJS interpreter used for building.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def node(command_context, args): + from mozbuild.nodeutil import find_node_executable + + # Avoid logging the command + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + + node_path, _ = find_node_executable() + + return command_context.run_process( + [node_path] + args, + pass_thru=True, # Allow user to run Node interactively. + ensure_exit_code=False, # Don't throw on non-zero exit code. + ) + + +@Command( + "npm", + category="devenv", + description="Run the npm executable from the NodeJS used for building.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def npm(command_context, args): + from mozbuild.nodeutil import find_npm_executable + + # Avoid logging the command + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + + import os + + # Add node and npm from mozbuild to front of system path + # + # This isn't pretty, but npm currently executes itself with + # `#!/usr/bin/env node`, which means it just uses the node in the + # current PATH. As a result, stuff gets built wrong and installed + # in the wrong places and probably other badness too without this: + npm_path, _ = find_npm_executable() + if not npm_path: + exit(-1, "could not find npm executable") + path = os.path.abspath(os.path.dirname(npm_path)) + os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"]) + + # karma-firefox-launcher needs the path to firefox binary. + firefox_bin = command_context.get_binary_path(validate_exists=False) + if os.path.exists(firefox_bin): + os.environ["FIREFOX_BIN"] = firefox_bin + + return command_context.run_process( + [npm_path, "--scripts-prepend-node-path=auto"] + args, + pass_thru=True, # Avoid eating npm output/error messages + ensure_exit_code=False, # Don't throw on non-zero exit code. + ) + + +def logspam_create_parser(subcommand): + # Create the logspam command line parser. + # if logspam is not installed, or not up to date, it will + # first be installed. + loader = PypiBasedTool("logspam", "mozilla-log-spam") + return loader.create_parser(subcommand) + + +from functools import partial + + +@Command( + "logspam", + category="misc", + description="Warning categorizer for treeherder test runs.", +) +def logspam(command_context): + pass + + +@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report")) +def report(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="report", **options) + + +@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect")) +def bisect(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="bisect", **options) + + +@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file")) +def create(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="file", **options) + + +# mots_loader will be used when running commands and subcommands, as well as +# when creating the parsers. +mots_loader = PypiBasedTool("mots") + + +def mots_create_parser(subcommand=None): + return mots_loader.create_parser(subcommand) + + +def mots_run_subcommand(command, command_context, **options): + command_context.activate_virtualenv() + mots_loader.run(command=command, **options) + + +class motsSubCommand(SubCommand): + """A helper subclass that reduces repitition when defining subcommands.""" + + def __init__(self, subcommand): + super().__init__( + "mots", + subcommand, + parser=partial(mots_create_parser, subcommand), + ) + + +@Command( + "mots", + category="misc", + description="Manage module information in-tree using the mots CLI.", + parser=mots_create_parser, +) +def mots(command_context, **options): + """The main mots command call.""" + command_context.activate_virtualenv() + mots_loader.run(**options) + + +# Define subcommands that will be proxied through mach. +for sc in ( + "clean", + "check-hashes", + "export", + "export-and-clean", + "module", + "query", + "settings", + "user", + "validate", +): + # Pass through args and kwargs, but add the subcommand string as the first argument. + motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw)) |