# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import argparse import ast import difflib import errno import shlex import sys import types import uuid from collections.abc import Iterable from pathlib import Path from typing import Dict, Optional, Union from mozfile import load_source from .base import MissingFileError, UnknownCommandError INVALID_ENTRY_POINT = r""" Entry points should return a list of command providers or directories containing command providers. The following entry point is invalid: %s You are seeing this because there is an error in an external module attempting to implement a mach command. Please fix the error, or uninstall the module from your system. """.lstrip() class MachCommandReference: """A reference to a mach command. Holds the metadata for a mach command. """ module: Path def __init__( self, module: Union[str, Path], command_dependencies: Optional[list] = None, ): self.module = Path(module) self.command_dependencies = command_dependencies or [] MACH_COMMANDS = { "addstory": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "addtest": MachCommandReference("testing/mach_commands.py"), "addwidget": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "android": MachCommandReference("mobile/android/mach_commands.py"), "android-emulator": MachCommandReference("mobile/android/mach_commands.py"), "artifact": MachCommandReference( "python/mozbuild/mozbuild/artifact_commands.py", ), "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"), "bootstrap": MachCommandReference( "python/mozboot/mozboot/mach_commands.py", ), "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"), "build": MachCommandReference( "python/mozbuild/mozbuild/build_commands.py", ), "build-backend": MachCommandReference( "python/mozbuild/mozbuild/build_commands.py", ), "buildsymbols": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "buildtokens": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "busted": MachCommandReference("tools/mach_commands.py"), "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "clang-format": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "clang-tidy": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "compare-locales": MachCommandReference("tools/compare-locales/mach_commands.py"), "compileflags": MachCommandReference( "python/mozbuild/mozbuild/compilation/codecomplete.py" ), "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"), "cppunittest": MachCommandReference("testing/mach_commands.py"), "cramtest": MachCommandReference("testing/mach_commands.py"), "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"), "data-review": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"), "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "environment": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "eslint": MachCommandReference("tools/lint/mach_commands.py"), "event-into-legacy": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "fetch-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), "file-info": MachCommandReference( "python/mozbuild/mozbuild/frontend/mach_commands.py" ), "firefox-ui-functional": MachCommandReference( "testing/firefox-ui/mach_commands.py" ), "fluent-migration-test": MachCommandReference("testing/mach_commands.py"), "format": MachCommandReference("tools/lint/mach_commands.py"), "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"), "geckoview-junit": MachCommandReference( "testing/mochitest/mach_commands.py", ["test"] ), "gen-use-counter-metrics": MachCommandReference("dom/base/mach_commands.py"), "generate-test-certs": MachCommandReference( "security/manager/tools/mach_commands.py" ), "gradle": MachCommandReference("mobile/android/mach_commands.py"), "gradle-install": MachCommandReference("mobile/android/mach_commands.py"), "gtest": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py", ["test"] ), "hazards": MachCommandReference("js/src/devtools/rootAnalysis/mach_commands.py"), "ide": MachCommandReference("python/mozbuild/mozbuild/backend/mach_commands.py"), "import-pr": MachCommandReference("tools/vcs/mach_commands.py"), "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"), "jit-test": MachCommandReference("testing/mach_commands.py"), "jsapi-tests": MachCommandReference("testing/mach_commands.py"), "jsshell-bench": MachCommandReference("testing/mach_commands.py"), "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"), "jstests": MachCommandReference("testing/mach_commands.py"), "l10n-cross-channel": MachCommandReference( "tools/compare-locales/mach_commands.py" ), "lint": MachCommandReference("tools/lint/mach_commands.py"), "logspam": MachCommandReference("tools/mach_commands.py"), "mach-commands": MachCommandReference("python/mach/mach/commands/commandinfo.py"), "mach-completion": MachCommandReference("python/mach/mach/commands/commandinfo.py"), "mach-debug-commands": MachCommandReference( "python/mach/mach/commands/commandinfo.py" ), "macos-sign": MachCommandReference("tools/signing/macos/mach_commands.py"), "manifest": MachCommandReference("testing/mach_commands.py"), "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"), "mochitest": MachCommandReference("testing/mochitest/mach_commands.py", ["test"]), "mots": MachCommandReference("tools/mach_commands.py"), "mozbuild-reference": MachCommandReference( "python/mozbuild/mozbuild/frontend/mach_commands.py", ), "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"), "mozregression": MachCommandReference("tools/mach_commands.py"), "node": MachCommandReference("tools/mach_commands.py"), "npm": MachCommandReference("tools/mach_commands.py"), "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "package-multi-locale": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "pastebin": MachCommandReference("tools/mach_commands.py"), "perf-data-review": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "perftest": MachCommandReference("python/mozperftest/mozperftest/mach_commands.py"), "perftest-test": MachCommandReference( "python/mozperftest/mozperftest/mach_commands.py", ), "perftest-tools": MachCommandReference( "python/mozperftest/mozperftest/mach_commands.py" ), "power": MachCommandReference("tools/power/mach_commands.py"), "prettier-format": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "puppeteer-test": MachCommandReference("remote/mach_commands.py"), "python": MachCommandReference("python/mach_commands.py"), "python-test": MachCommandReference("python/mach_commands.py"), "raptor": MachCommandReference("testing/raptor/mach_commands.py"), "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"), "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"), "release": MachCommandReference("python/mozrelease/mozrelease/mach_commands.py"), "release-history": MachCommandReference("taskcluster/mach_commands.py"), "remote": MachCommandReference("remote/mach_commands.py"), "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "resource-usage": MachCommandReference( "python/mozbuild/mozbuild/build_commands.py", ), "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), "rusttests": MachCommandReference("testing/mach_commands.py"), "settings": MachCommandReference("python/mach/mach/commands/settings.py"), "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "static-analysis": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "storybook": MachCommandReference( "browser/components/storybook/mach_commands.py", ["run"] ), "talos-test": MachCommandReference("testing/talos/mach_commands.py"), "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"), "taskcluster-image-digest": MachCommandReference("taskcluster/mach_commands.py"), "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"), "taskgraph": MachCommandReference("taskcluster/mach_commands.py"), "telemetry-tests-client": MachCommandReference( "toolkit/components/telemetry/tests/marionette/mach_commands.py" ), "test": MachCommandReference("testing/mach_commands.py"), "test-info": MachCommandReference("testing/mach_commands.py"), "test-interventions": MachCommandReference( "testing/webcompat/mach_commands.py", ), "tps-build": MachCommandReference("testing/tps/mach_commands.py"), "try": MachCommandReference("tools/tryselect/mach_commands.py"), "ts": MachCommandReference("tools/ts/mach_commands.py"), "uniffi": MachCommandReference( "toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py" ), "update-glean": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "update-glean-tags": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"), "vcs-setup": MachCommandReference( "python/mozboot/mozboot/mach_commands.py", ), "vendor": MachCommandReference( "python/mozbuild/mozbuild/vendor/mach_commands.py", ), "warnings-list": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "warnings-summary": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "watch": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py", ), "web-platform-tests": MachCommandReference( "testing/web-platform/mach_commands.py", ), "web-platform-tests-update": MachCommandReference( "testing/web-platform/mach_commands.py", ), "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"), "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"), "wpt": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-fetch-logs": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-fission-regressions": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-interop-score": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-manifest-update": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-metadata-merge": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-metadata-summary": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"), "xpcshell": MachCommandReference("js/xpconnect/mach_commands.py"), "xpcshell-test": MachCommandReference( "testing/xpcshell/mach_commands.py", ["test"] ), } class DecoratorVisitor(ast.NodeVisitor): def __init__(self): self.results = {} def visit_FunctionDef(self, node): # We only care about `Command` and `SubCommand` decorators, since # they are the only ones that can specify virtualenv_name decorators = [ decorator for decorator in node.decorator_list if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and decorator.func.id in ["SubCommand", "Command"] ] relevant_kwargs = ["command", "subcommand", "virtualenv_name"] for decorator in decorators: kwarg_dict = {} for name, arg in zip(["command", "subcommand"], decorator.args): kwarg_dict[name] = arg.s for keyword in decorator.keywords: if keyword.arg not in relevant_kwargs: # We only care about these 3 kwargs, so we can safely skip the rest continue kwarg_dict[keyword.arg] = getattr(keyword.value, "s", "") command = kwarg_dict.pop("command") self.results.setdefault(command, {}) sub_command = kwarg_dict.pop("subcommand", None) virtualenv_name = kwarg_dict.pop("virtualenv_name", None) if sub_command: self.results[command].setdefault("subcommands", {}) sub_command_dict = self.results[command]["subcommands"].setdefault( sub_command, {} ) if virtualenv_name: sub_command_dict["virtualenv_name"] = virtualenv_name elif virtualenv_name: # If there is no `subcommand` we are in the `@Command` # decorator, and need to store the virtualenv_name for # the 'command'. self.results[command]["virtualenv_name"] = virtualenv_name self.generic_visit(node) def command_virtualenv_info_for_module(module_path): with module_path.open("r") as file: content = file.read() tree = ast.parse(content) visitor = DecoratorVisitor() visitor.visit(tree) return visitor.results class DetermineCommandVenvAction(argparse.Action): def __init__( self, option_strings, dest, topsrcdir, required=True, ): self.topsrcdir = topsrcdir argparse.Action.__init__( self, option_strings, dest, required=required, help=argparse.SUPPRESS, nargs=argparse.REMAINDER, ) def __call__(self, parser, namespace, values, option_string=None): if len(values) == 0: return command = values[0] aliases = namespace.mach_command_aliases if command in aliases: alias = aliases[command] arg_string = shlex.split(alias) command = arg_string.pop(0) # the "help" command does not have a module file, it's handled # a bit later and should be skipped here. if command == "help": return command_reference = MACH_COMMANDS.get(command) if not command_reference: # Try to find similarly named commands, may raise UnknownCommandError. suggested_command = suggest_command(command) sys.stderr.write( f"We're assuming the '{command}' command is '{suggested_command}' and we're executing it for you.\n\n" ) command = suggested_command command_reference = MACH_COMMANDS.get(command) setattr(namespace, "command_name", command) if len(values) > 1: potential_sub_command_name = values[1] else: potential_sub_command_name = None module_path = Path(self.topsrcdir) / command_reference.module module_dict = command_virtualenv_info_for_module(module_path) command_dict = module_dict.get(command, {}) if not command_dict: return site = command_dict.get("virtualenv_name", "common") if potential_sub_command_name and not potential_sub_command_name.startswith( "-" ): all_sub_commands_dict = command_dict.get("subcommands", {}) if all_sub_commands_dict: sub_command_dict = all_sub_commands_dict.get( potential_sub_command_name, {} ) if sub_command_dict: site = sub_command_dict.get("virtualenv_name", "common") setattr(namespace, "site_name", site) def suggest_command(command): names = MACH_COMMANDS.keys() # We first try to look for a valid command that is very similar to the given command. suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8) # If we find more than one matching command, or no command at all, # we give command suggestions instead (with a lower matching threshold). # All commands that start with the given command (for instance: # 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-') # are also included. if len(suggested_commands) != 1: suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5)) suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} raise UnknownCommandError(command, "run", suggested_commands) return suggested_commands[0] def load_commands_from_directory(path: Path): """Scan for mach commands from modules in a directory. This takes a path to a directory, loads the .py files in it, and registers and found mach command providers with this mach instance. """ for f in sorted(path.iterdir()): if not f.suffix == ".py" or f.name == "__init__.py": continue full_path = path / f module_name = f"mach.commands.{str(f)[0:-3]}" load_commands_from_file(full_path, module_name=module_name) def load_commands_from_file(path: Union[str, Path], module_name=None): """Scan for mach commands from a file. This takes a path to a file and loads it as a Python module under the module name specified. If no name is specified, a random one will be chosen. """ if module_name is None: # Ensure parent module is present otherwise we'll (likely) get # an error due to unknown parent. if "mach.commands" not in sys.modules: mod = types.ModuleType("mach.commands") sys.modules["mach.commands"] = mod module_name = f"mach.commands.{uuid.uuid4().hex}" try: load_source(module_name, str(path)) except IOError as e: if e.errno != errno.ENOENT: raise raise MissingFileError(f"{path} does not exist") def load_commands_from_spec( spec: Dict[str, MachCommandReference], topsrcdir: str, missing_ok=False ): """Load mach commands based on the given spec. Takes a dictionary mapping command names to their metadata. """ modules = set(spec[command].module for command in spec) for path in modules: try: load_commands_from_file(Path(topsrcdir) / path) except MissingFileError: if not missing_ok: raise def load_commands_from_entry_point(group="mach.providers"): """Scan installed packages for mach command provider entry points. An entry point is a function that returns a list of paths to files or directories containing command providers. This takes an optional group argument which specifies the entry point group to use. If not specified, it defaults to 'mach.providers'. """ try: import pkg_resources except ImportError: print( "Could not find setuptools, ignoring command entry points", file=sys.stderr, ) return for entry in pkg_resources.iter_entry_points(group=group, name=None): paths = [Path(path) for path in entry.load()()] if not isinstance(paths, Iterable): print(INVALID_ENTRY_POINT % entry) sys.exit(1) for path in paths: if path.is_file(): load_commands_from_file(path) elif path.is_dir(): load_commands_from_directory(path) else: print(f"command provider '{path}' does not exist") def load_command_module_from_command_name(command_name: str, topsrcdir: str): command_reference = MACH_COMMANDS.get(command_name) load_commands_from_spec( {command_name: command_reference}, topsrcdir, missing_ok=False )