diff options
Diffstat (limited to 'python/mach')
57 files changed, 6351 insertions, 0 deletions
diff --git a/python/mach/README.rst b/python/mach/README.rst new file mode 100644 index 0000000000..7c2e00becb --- /dev/null +++ b/python/mach/README.rst @@ -0,0 +1,13 @@ +==== +mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +To learn more, read the docs in ``docs/``. diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh new file mode 100644 index 0000000000..13935cf88c --- /dev/null +++ b/python/mach/bash-completion.sh @@ -0,0 +1,18 @@ +function _mach() +{ + local cur targets + COMPREPLY=() + + # Calling `mach-completion` with -h/--help would result in the + # help text being used as the completion targets. + if [[ $COMP_LINE == *"-h"* || $COMP_LINE == *"--help"* ]]; then + return 0 + fi + + # Load the list of targets + targets=`"${COMP_WORDS[0]}" mach-completion ${COMP_LINE}` + cur="${COMP_WORDS[COMP_CWORD]}" + COMPREPLY=( $(compgen -W "$targets" -- ${cur}) ) + return 0 +} +complete -o default -F _mach mach diff --git a/python/mach/docs/commands.rst b/python/mach/docs/commands.rst new file mode 100644 index 0000000000..2d014af9f7 --- /dev/null +++ b/python/mach/docs/commands.rst @@ -0,0 +1,145 @@ +.. _mach_commands: + +===================== +Implementing Commands +===================== + +Mach commands are defined via Python decorators. + +All the relevant decorators are defined in the *mach.decorators* module. +The important decorators are as follows: + +:py:func:`CommandProvider <mach.decorators.CommandProvider>` + A class decorator that denotes that a class contains mach + commands. The decorator takes no arguments. + +:py:func:`Command <mach.decorators.Command>` + A method decorator that denotes that the method should be called when + the specified command is requested. The decorator takes a command name + as its first argument and a number of additional arguments to + configure the behavior of the command. + +:py:func:`CommandArgument <mach.decorators.CommandArgument>` + A method decorator that defines an argument to the command. Its + arguments are essentially proxied to ArgumentParser.add_argument() + +:py:func:`SubCommand <mach.decorators.SubCommand>` + A method decorator that denotes that the method should be a + sub-command to an existing ``@Command``. The decorator takes the + parent command name as its first argument and the sub-command name + as its second argument. + + ``@CommandArgument`` can be used on ``@SubCommand`` instances just + like they can on ``@Command`` instances. + +Classes with the ``@CommandProvider`` decorator **must** subclass +``MachCommandBase`` and have a compatible ``__init__`` method. + +Here is a complete example: + +.. code-block:: python + + from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + ) + from mozbuild.base import MachCommandBase + + @CommandProvider + class MyClass(MachCommandBase): + @Command('doit', help='Do ALL OF THE THINGS.') + @CommandArgument('--force', '-f', action='store_true', + help='Force doing it.') + def doit(self, force=False): + # Do stuff here. + +When the module is loaded, the decorators tell mach about all handlers. +When mach runs, it takes the assembled metadata from these handlers and +hooks it up to the command line driver. Under the hood, arguments passed +to the decorators are being used to help mach parse command arguments, +formulate arguments to the methods, etc. See the documentation in the +:py:mod:`mach.base` module for more. + +The Python modules defining mach commands do not need to live inside the +main mach source tree. + +Conditionally Filtering Commands +================================ + +Sometimes it might only make sense to run a command given a certain +context. For example, running tests only makes sense if the product +they are testing has been built, and said build is available. To make +sure a command is only runnable from within a correct context, you can +define a series of conditions on the +:py:func:`Command <mach.decorators.Command>` decorator. + +A condition is simply a function that takes an instance of the +:py:func:`mach.decorators.CommandProvider` class as an argument, and +returns ``True`` or ``False``. If any of the conditions defined on a +command return ``False``, the command will not be runnable. The +docstring of a condition function is used in error messages, to explain +why the command cannot currently be run. + +Here is an example: + +.. code-block:: python + + from mach.decorators import ( + CommandProvider, + Command, + ) + + def build_available(cls): + """The build needs to be available.""" + return cls.build_path is not None + + @CommandProvider + class MyClass(MachCommandBase): + def __init__(self, *args, **kwargs): + super(MyClass, self).__init__(*args, **kwargs) + self.build_path = ... + + @Command('run_tests', conditions=[build_available]) + def run_tests(self): + # Do stuff here. + +It is important to make sure that any state needed by the condition is +available to instances of the command provider. + +By default all commands without any conditions applied will be runnable, +but it is possible to change this behaviour by setting +``require_conditions`` to ``True``: + +.. code-block:: python + + m = mach.main.Mach() + m.require_conditions = True + +Minimizing Code in Commands +=========================== + +Mach command modules, classes, and methods work best when they are +minimal dispatchers. The reason is import bloat. Currently, the mach +core needs to import every Python file potentially containing mach +commands for every command invocation. If you have dozens of commands or +commands in modules that import a lot of Python code, these imports +could slow mach down and waste memory. + +It is thus recommended that mach modules, classes, and methods do as +little work as possible. Ideally the module should only import from +the :py:mod:`mach` package. If you need external modules, you should +import them from within the command method. + +To keep code size small, the body of a command method should be limited +to: + +1. Obtaining user input (parsing arguments, prompting, etc) +2. Calling into some other Python package +3. Formatting output + +Of course, these recommendations can be ignored if you want to risk +slower performance. + +In the future, the mach driver may cache the dispatching information or +have it intelligently loaded to facilitate lazy loading. diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst new file mode 100644 index 0000000000..8a2a99a2f5 --- /dev/null +++ b/python/mach/docs/driver.rst @@ -0,0 +1,32 @@ +.. _mach_driver: + +======= +Drivers +======= + +Entry Points +============ + +It is possible to use setuptools' entry points to load commands +directly from python packages. A mach entry point is a function which +returns a list of files or directories containing mach command +providers. e.g.: + +.. code-block:: python + + def list_providers(): + providers = [] + here = os.path.abspath(os.path.dirname(__file__)) + for p in os.listdir(here): + if p.endswith('.py'): + providers.append(os.path.join(here, p)) + return providers + +See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins +for more information on creating an entry point. To search for entry +point plugins, you can call +:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.: + +.. code-block:: python + + mach.load_commands_from_entry_point("mach.external.providers") diff --git a/python/mach/docs/faq.rst b/python/mach/docs/faq.rst new file mode 100644 index 0000000000..bf53612fa0 --- /dev/null +++ b/python/mach/docs/faq.rst @@ -0,0 +1,85 @@ +.. _mach_faq: + +========================== +Frequently Asked Questions +========================== + +How do I report bugs? +--------------------- + +Bugs against the ``mach`` core can be filed in Bugzilla in the `Firefox +Build System::Mach +Core <https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Mach%20Core>`__ component. + +.. note:: + + Most ``mach`` bugs are bugs in individual commands, not bugs in the core + ``mach`` code. Bugs for individual commands should be filed against the + component that command is related to. For example, bugs in the + *build* command should be filed against *Firefox Build System :: + General*. Bugs against testing commands should be filed somewhere in + the *Testing* product. + + +Is ``mach`` a build system? +--------------------------- + +No. ``mach`` is just a generic command dispatching tool that happens to have +a few commands that interact with the real build system. Historically, +``mach`` *was* born to become a better interface to the build system. +However, its potential beyond just build system interaction was quickly +realized and ``mach`` grew to fit those needs. + +How do I add features to ``mach``? +---------------------------------- +If you would like to add a new feature to ``mach`` that cannot be implemented as +a ``mach`` command, the first step is to file a bug in the +``Firefox Build System :: Mach Core`` component. + +Should I implement X as a ``mach`` command? +------------------------------------------- + +There are no hard or fast rules. Generally speaking, if you have some +piece of functionality or action that is useful to multiple people +(especially if it results in productivity wins), then you should +consider implementing a ``mach`` command for it. + +Some other cases where you should consider implementing something as a +mach command: + +- When your tool is a random script in the tree. Random scripts are + hard to find and may not conform to coding conventions or best + practices. Mach provides a framework in which your tool can live that + will put it in a better position to succeed than if it were on its + own. +- When the alternative is a ``make`` target. The build team generally does + not like one-off ``make`` targets that aren't part of building (read: + compiling) the tree. This includes things related to testing and + packaging. These weigh down ``Makefiles`` and add to the burden of + maintaining the build system. Instead, you are encouraged to + implement ancillary functionality in Python. If you do implement something + in Python, hooking it up to ``mach`` is often trivial. + + +How does ``mach`` fit into the modules system? +---------------------------------------------- + +Mozilla operates with a `modules governance +system <https://www.mozilla.org/about/governance/policies/module-ownership/>`__ where +there are different components with different owners. There is not +currently a ``mach`` module. There may or may never be one; currently ``mach`` +is owned by the build team. + +Even if a ``mach`` module were established, ``mach`` command modules would +likely never belong to it. Instead, ``mach`` command modules are owne by the +team/module that owns the system they interact with. In other words, ``mach`` +is not a power play to consolidate authority for tooling. Instead, it aims to +expose that tooling through a common, shared interface. + + +Who do I contact for help or to report issues? +---------------------------------------------- + +You can ask questions in +`#build <https://chat.mozilla.org/#/room/#build:mozilla.org>`__. + diff --git a/python/mach/docs/index.rst b/python/mach/docs/index.rst new file mode 100644 index 0000000000..0b5639cc64 --- /dev/null +++ b/python/mach/docs/index.rst @@ -0,0 +1,88 @@ +==== +Mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +.. raw:: html + + <h2>Features</h2> + +---- + +On a high level, mach is similar to using argparse with subparsers (for +command handling). When you dig deeper, mach offers a number of +additional features: + +Distributed command definitions + With optparse/argparse, you have to define your commands on a central + parser instance. With mach, you annotate your command methods with + decorators and mach finds and dispatches to them automatically. + +Command categories + Mach commands can be grouped into categories when displayed in help. + This is currently not possible with argparse. + +Logging management + Mach provides a facility for logging (both classical text and + structured) that is available to any command handler. + +Settings files + Mach provides a facility for reading settings from an ini-like file + format. + +.. raw:: html + + <h2>Components</h2> + +---- + +Mach is conceptually composed of the following components: + +core + The mach core is the core code powering mach. This is a Python package + that contains all the business logic that makes mach work. The mach + core is common to all mach deployments. + +commands + These are what mach dispatches to. Commands are simply Python methods + registered as command names. The set of commands is unique to the + environment mach is deployed in. + +driver + The *driver* is the entry-point to mach. It is simply an executable + script that loads the mach core, tells it where commands can be found, + then asks the mach core to handle the current request. The driver is + unique to the deployed environment. But, it's usually based on an + example from this source tree. + +.. raw:: html + + <h2> Project State</h2> + +---- + +mach was originally written as a command dispatching framework to aid +Firefox development. While the code is mostly generic, there are still +some pieces that closely tie it to Mozilla/Firefox. The goal is for +these to eventually be removed and replaced with generic features so +mach is suitable for anybody to use. Until then, mach may not be the +best fit for you. + +.. toctree:: + :maxdepth: 1 + :hidden: + + usage + commands + driver + logging + settings + telemetry + faq diff --git a/python/mach/docs/logging.rst b/python/mach/docs/logging.rst new file mode 100644 index 0000000000..ff245cf032 --- /dev/null +++ b/python/mach/docs/logging.rst @@ -0,0 +1,100 @@ +.. _mach_logging: + +======= +Logging +======= + +Mach configures a built-in logging facility so commands can easily log +data. + +What sets the logging facility apart from most loggers you've seen is +that it encourages structured logging. Instead of conventional logging +where simple strings are logged, the internal logging mechanism logs all +events with the following pieces of information: + +* A string *action* +* A dict of log message fields +* A formatting string + +Essentially, instead of assembling a human-readable string at +logging-time, you create an object holding all the pieces of data that +will constitute your logged event. For each unique type of logged event, +you assign an *action* name. + +Depending on how logging is configured, your logged event could get +written a couple of different ways. + +JSON Logging +============ + +Where machines are the intended target of the logging data, a JSON +logger is configured. The JSON logger assembles an array consisting of +the following elements: + +* Decimal wall clock time in seconds since UNIX epoch +* String *action* of message +* Object with structured message data + +The JSON-serialized array is written to a configured file handle. +Consumers of this logging stream can just perform a readline() then feed +that into a JSON deserializer to reconstruct the original logged +message. They can key off the *action* element to determine how to +process individual events. There is no need to invent a parser. +Convenient, isn't it? + +Logging for Humans +================== + +Where humans are the intended consumer of a log message, the structured +log message are converted to more human-friendly form. This is done by +utilizing the *formatting* string provided at log time. The logger +simply calls the *format* method of the formatting string, passing the +dict containing the message's fields. + +When *mach* is used in a terminal that supports it, the logging facility +also supports terminal features such as colorization. This is done +automatically in the logging layer - there is no need to control this at +logging time. + +In addition, messages intended for humans typically prepends every line +with the time passed since the application started. + +Logging HOWTO +============= + +Structured logging piggybacks on top of Python's built-in logging +infrastructure provided by the *logging* package. We accomplish this by +taking advantage of *logging.Logger.log()*'s *extra* argument. To this +argument, we pass a dict with the fields *action* and *params*. These +are the string *action* and dict of message fields, respectively. The +formatting string is passed as the *msg* argument, like normal. + +If you were logging to a logger directly, you would do something like: + +.. code-block:: python + + logger.log(logging.INFO, 'My name is {name}', + extra={'action': 'my_name', 'params': {'name': 'Gregory'}}) + +The JSON logging would produce something like:: + + [1339985554.306338, "my_name", {"name": "Gregory"}] + +Human logging would produce something like:: + + 0.52 My name is Gregory + +Since there is a lot of complexity using logger.log directly, it is +recommended to go through a wrapping layer that hides part of the +complexity for you. The easiest way to do this is by utilizing the +LoggingMixin: + +.. code-block:: python + + import logging + from mach.mixin.logging import LoggingMixin + + class MyClass(LoggingMixin): + def foo(self): + self.log(logging.INFO, 'foo_start', {'bar': True}, + 'Foo performed. Bar: {bar}') diff --git a/python/mach/docs/metrics.md b/python/mach/docs/metrics.md new file mode 100644 index 0000000000..da1bd5cf3f --- /dev/null +++ b/python/mach/docs/metrics.md @@ -0,0 +1,53 @@ +<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. --> + +# Metrics +This document enumerates the metrics collected by this project. +This project may depend on other projects which also collect metrics. +This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project. + +# Pings + + - [usage](#usage) + + +## usage + +Sent when the mach invocation is completed (regardless of result). Contains information about the mach invocation that was made, its result, and some details about the current environment and hardware. + + +This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section). + +**Data reviews for this ping:** + +- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34> + +**Bugs related to this ping:** + +- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053> + +The following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| mach.argv |[string_list](https://mozilla.github.io/glean/book/user/metrics/string_list.html) |Parameters provided to mach. Absolute paths are sanitized to be relative to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" would be replaced with "$topsrcdir/python/mozbuild". If a valid replacement base path cannot be found, the path is replaced with "<path omitted>". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.command |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the mach command that was invoked, such as "build", "doc", or "try". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.duration |[timespan](https://mozilla.github.io/glean/book/user/metrics/timespan.html) |How long it took for the command to complete. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.success |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the mach invocation succeeded. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.cpu_brand |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |CPU brand string from CPUID. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.logical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of logical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.memory |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |Amount of system memory. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.physical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of physical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.artifact |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-artifact-builds`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.ccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--with-ccache`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.clobber |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the build was a clobber/full build. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15)||never | | +| mozbuild.compiler |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The compiler type in use (CC_TYPE), such as "clang" or "gcc". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.debug |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-debug`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.icecream |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if icecream in use. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.opt |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-optimize`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.sccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if ccache in use is sccache. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | + + +Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection). + +<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. --> + diff --git a/python/mach/docs/settings.rst b/python/mach/docs/settings.rst new file mode 100644 index 0000000000..b8da739036 --- /dev/null +++ b/python/mach/docs/settings.rst @@ -0,0 +1,143 @@ +.. _mach_settings: + +======== +Settings +======== + +Mach can read settings in from a set of configuration files. These +configuration files are either named ``machrc`` or ``.machrc`` and +are specified by the bootstrap script. In mozilla-central, these files +can live in ``~/.mozbuild`` and/or ``topsrcdir``. + +Settings can be specified anywhere, and used both by mach core or +individual commands. + + +Core Settings +============= + +These settings are implemented by mach core. + +* alias - Create a command alias. This is useful if you want to alias a command to something else, optionally including some defaults. It can either be used to create an entire new command, or provide defaults for an existing one. For example: + +.. parsed-literal:: + + [alias] + mochitest = mochitest -f browser + browser-test = mochitest -f browser + + +Defining Settings +================= + +Settings need to be explicitly defined, along with their type, +otherwise mach will throw when trying to access them. + +To define settings, use the :func:`~decorators.SettingsProvider` +decorator in an existing mach command module. E.g: + +.. code-block:: python + + from mach.decorators import SettingsProvider + from mozbuild.base import MachCommandBase + + @SettingsProvider + class ArbitraryClassName(MachCommandBase): + config_settings = [ + ('foo.bar', 'string', "A helpful description"), + ('foo.baz', 'int', "Another description", 0, {'choices': set([0,1,2])}), + ] + +``@SettingsProvider``'s must specify a variable called ``config_settings`` +that returns a list of tuples. Alternatively, it can specify a function +called ``config_settings`` that returns a list of tuples. + +Each tuple is of the form: + +.. code-block:: python + + ('<section>.<option>', '<type>', '<description>', default, extra) + +``type`` is a string and can be one of: +string, boolean, int, pos_int, path + +``description`` is a string explaining how to define the settings and +where they get used. Descriptions should ideally be multi-line paragraphs +where the first line acts as a short description. + +``default`` is optional, and provides a default value in case none was +specified by any of the configuration files. + +``extra`` is also optional and is a dict containing additional key/value +pairs to add to the setting's metadata. The following keys may be specified +in the ``extra`` dict: + * ``choices`` - A set of allowed values for the setting. + +Wildcards +--------- + +Sometimes a section should allow arbitrarily defined options from the user, such +as the ``alias`` section mentioned above. To define a section like this, use ``*`` +as the option name. For example: + +.. parsed-literal:: + + ('foo.*', 'string', 'desc') + +This allows configuration files like this: + +.. parsed-literal:: + + [foo] + arbitrary1 = some string + arbitrary2 = some other string + + +Finding Settings +================ + +You can see which settings are available as well as their description and +expected values by running: + +.. parsed-literal:: + + ./mach settings # or + ./mach settings --list + + +Accessing Settings +================== + +Now that the settings are defined and documented, they're accessible from +individual mach commands from the mach command context. +For example: + +.. code-block:: python + + from mach.decorators import ( + Command, + CommandProvider, + SettingsProvider, + ) + from mozbuild.base import MachCommandBase + + @SettingsProvider + class ExampleSettings(object): + config_settings = [ + ('a.b', 'string', 'desc', 'default'), + ('foo.bar', 'string', 'desc',), + ('foo.baz', 'int', 'desc', 0, {'choices': set([0,1,2])}), + ] + + @CommandProvider + class Commands(MachCommandBase): + def __init__(self, *args, **kwargs): + super(Commands, self).__init__(*args, **kwargs) + self.settings = self._mach_context.settings + + @Command('command', category='misc', + description='Prints a setting') + def command(self): + print(self.settings.a.b) + for option in self.settings.foo: + print(self.settings.foo[option]) diff --git a/python/mach/docs/telemetry.rst b/python/mach/docs/telemetry.rst new file mode 100644 index 0000000000..b55881e89b --- /dev/null +++ b/python/mach/docs/telemetry.rst @@ -0,0 +1,39 @@ +.. _mach_telemetry: + +============== +Mach Telemetry +============== + +`Glean <https://mozilla.github.io/glean/>`_ is used to collect telemetry, and uses the metrics +defined in the ``metrics.yaml`` files in-tree. +These files are all documented in a single :ref:`generated file here<metrics>`. + +.. toctree:: + :maxdepth: 1 + + metrics + +Adding Metrics to a new Command +=============================== + +If you would like to submit telemetry metrics from your mach ``@Command``, you should take two steps: + +#. Parameterize your class's ``@CommandProvider`` annotation with ``metrics_path``. +#. Use the ``self.metrics`` handle provided by ``MachCommandBase`` + +For example:: + + METRICS_PATH = os.path.abspath(os.path.join(__file__, '..', '..', 'metrics.yaml')) + + @CommandProvider(metrics_path=METRICS_PATH) + class CustomCommand(MachCommandBase): + @Command('custom-command') + def custom_command(self): + self.metrics.custom.foo.set('bar') + +Updating Generated Metrics Docs +=============================== + +When a ``metrics.yaml`` is added/changed/removed, :ref:`the metrics document<metrics>` will need to be updated:: + + ./mach doc mach-telemetry diff --git a/python/mach/docs/usage.rst b/python/mach/docs/usage.rst new file mode 100644 index 0000000000..ce10ca3d24 --- /dev/null +++ b/python/mach/docs/usage.rst @@ -0,0 +1,157 @@ +.. _mach_usage: + +========== +User Guide +========== + +Mach is the central entry point for most operations that can be performed in +mozilla-central. + + +Command Help +------------ + +To see an overview of all the available commands, run: + +.. code-block:: shell + + $ ./mach help + +For more detailed information on a specific command, run: + +.. code-block:: shell + + $ ./mach help <command> + +If a command has subcommands listed, you can see more details on the subcommand +by running: + +.. code-block:: shell + + $ ./mach help <command> <subcommand> + +Alternatively, you can pass ``-h/--help``. For example, all of the +following are valid: + +.. code-block:: shell + + $ ./mach help try + $ ./mach help try fuzzy + $ ./mach try -h + $ ./mach try fuzzy --help + + +Tab Completion +-------------- + +There are commands built-in to ``mach`` that can generate a fast tab completion +script for various shells. Supported shells are currently ``bash``, ``zsh`` and +``fish``. These generated scripts will slowly become out of date over time, so +you may want to create a cron task to periodically re-generate them. + +See below for installation instructions: + +Bash +~~~~ + +.. code-block:: shell + + $ mach mach-completion bash -f _mach + $ sudo mv _mach /etc/bash_completion.d + +Bash (homebrew) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ mach mach-completion bash -f $(brew --prefix)/etc/bash_completion.d/mach.bash-completion + +Zsh +~~~ + +.. code-block:: shell + + $ mkdir ~/.zfunc + $ mach mach-completion zsh -f ~/.zfunc/_mach + +then edit ~/.zshrc and add: + +.. code-block:: shell + + fpath+=~/.zfunc + autoload -U compinit && compinit + +You can use any directory of your choosing. + +Zsh (oh-my-zsh) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ mkdir $ZSH/plugins/mach + $ mach mach-completion zsh -f $ZSH/plugins/mach/_mach + +then edit ~/.zshrc and add 'mach' to your enabled plugins: + +.. code-block:: shell + + plugins(mach ...) + +Zsh (prezto) +~~~~~~~~~~~~ + +.. code-block:: shell + + $ mach mach-completion zsh -f ~/.zprezto/modules/completion/external/src/_mach + +Fish +~~~~ + +.. code-block:: shell + + $ ./mach mach-completion fish -f ~/.config/fish/completions/mach.fish + +Fish (homebrew) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ ./mach mach-completion fish -f (brew --prefix)/share/fish/vendor_completions.d/mach.fish + + +User Settings +------------- + +Some mach commands can read configuration from a ``machrc`` file. The default +location for this file is ``~/.mozbuild/machrc`` (you'll need to create it). +This can also be set to a different location by setting the ``MACHRC`` +environment variable. + +For a list of all the available settings, run: + +.. code-block:: shell + + $ ./mach settings + +The settings file follows the ``ini`` format, e.g: + +.. code-block:: ini + + [alias] + eslint = lint -l eslint + + [build] + telemetry = true + + [try] + default = fuzzy + +Adding ``mach`` to your ``PATH`` +-------------------------------- + +If you don't like having to type ``./mach``, you can add your source directory +to your ``PATH``. DO NOT copy the script to a directory already in your +``PATH``. + + +.. _bash completion: https://searchfox.org/mozilla-central/source/python/mach/bash-completion.sh diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/__init__.py diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py new file mode 100644 index 0000000000..a53aa552d2 --- /dev/null +++ b/python/mach/mach/base.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from mach.telemetry import NoopTelemetry + + +class CommandContext(object): + """Holds run-time state so it can easily be passed to command providers.""" + + def __init__( + self, + cwd=None, + settings=None, + log_manager=None, + commands=None, + telemetry=NoopTelemetry(False), + **kwargs + ): + self.cwd = cwd + self.settings = settings + self.log_manager = log_manager + self.commands = commands + self.telemetry = telemetry + self.command_attrs = {} + + for k, v in kwargs.items(): + setattr(self, k, v) + + +class MachError(Exception): + """Base class for all errors raised by mach itself.""" + + +class NoCommandError(MachError): + """No command was passed into mach.""" + + def __init__(self, namespace): + MachError.__init__(self) + self.namespace = namespace + + +class UnknownCommandError(MachError): + """Raised when we attempted to execute an unknown command.""" + + def __init__(self, command, verb, suggested_commands=None): + MachError.__init__(self) + + self.command = command + self.verb = verb + self.suggested_commands = suggested_commands or [] + + +class UnrecognizedArgumentError(MachError): + """Raised when an unknown argument is passed to mach.""" + + def __init__(self, command, arguments): + MachError.__init__(self) + + self.command = command + self.arguments = arguments + + +class FailedCommandError(Exception): + """Raised by commands to signal a handled failure to be printed by mach + + When caught by mach a FailedCommandError will print message and exit + with ''exit_code''. The optional ''reason'' is a string in cases where + other scripts may wish to handle the exception, though this is generally + intended to communicate failure to mach. + """ + + def __init__(self, message, exit_code=1, reason=""): + Exception.__init__(self, message) + self.exit_code = exit_code + self.reason = reason + + +class MissingFileError(MachError): + """Attempted to load a mach commands file that doesn't exist.""" diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/commands/__init__.py diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py new file mode 100644 index 0000000000..ac5e657b26 --- /dev/null +++ b/python/mach/mach/commands/commandinfo.py @@ -0,0 +1,494 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os +import re +import subprocess +import sys +from itertools import chain + +import attr + +from mach.decorators import ( + CommandProvider, + Command, + CommandArgument, + SubCommand, +) +from mozbuild.base import MachCommandBase +from mozbuild.util import memoize, memoized_property + +here = os.path.abspath(os.path.dirname(__file__)) +COMPLETION_TEMPLATES_DIR = os.path.join(here, "completion_templates") + + +@attr.s +class CommandInfo(object): + name = attr.ib(type=str) + description = attr.ib(type=str) + subcommands = attr.ib(type=list) + options = attr.ib(type=dict) + subcommand = attr.ib(type=str, default=None) + + +def render_template(shell, context): + filename = "{}.template".format(shell) + with open(os.path.join(COMPLETION_TEMPLATES_DIR, filename)) as fh: + template = fh.read() + return template % context + + +@CommandProvider +class BuiltinCommands(MachCommandBase): + @memoized_property + def command_handlers(self): + """A dictionary of command handlers keyed by command name.""" + return self._mach_context.commands.command_handlers + + @memoized_property + def commands(self): + """A sorted list of all command names.""" + return sorted(self.command_handlers) + + def _get_parser_options(self, parser): + options = {} + for action in parser._actions: + # ignore positional args + if not action.option_strings: + continue + + # ignore suppressed args + if action.help == argparse.SUPPRESS: + continue + + options[tuple(action.option_strings)] = action.help or "" + return options + + @memoized_property + def global_options(self): + """Return a dict of global options. + + Of the form `{("-o", "--option"): "description"}`. + """ + for group in self._mach_context.global_parser._action_groups: + if group.title == "Global Arguments": + return self._get_parser_options(group) + + @memoize + def _get_handler_options(self, handler): + """Return a dict of options for the given handler. + + Of the form `{("-o", "--option"): "description"}`. + """ + options = {} + for option_strings, val in handler.arguments: + # ignore positional args + if option_strings[0][0] != "-": + continue + + options[tuple(option_strings)] = val.get("help", "") + + if handler._parser: + options.update(self._get_parser_options(handler.parser)) + + return options + + def _get_handler_info(self, handler): + try: + options = self._get_handler_options(handler) + except (Exception, SystemExit): + # We don't want misbehaving commands to break tab completion, + # ignore any exceptions. + options = {} + + subcommands = [] + for sub in sorted(handler.subcommand_handlers): + subcommands.append(self._get_handler_info(handler.subcommand_handlers[sub])) + + return CommandInfo( + name=handler.name, + description=handler.description or "", + options=options, + subcommands=subcommands, + subcommand=handler.subcommand, + ) + + @memoized_property + def commands_info(self): + """Return a list of CommandInfo objects for each command.""" + commands_info = [] + # Loop over self.commands rather than self.command_handlers.items() for + # alphabetical order. + for c in self.commands: + commands_info.append(self._get_handler_info(self.command_handlers[c])) + return commands_info + + @Command("mach-commands", category="misc", description="List all mach commands.") + def run_commands(self): + print("\n".join(self.commands)) + + @Command( + "mach-debug-commands", + category="misc", + description="Show info about available mach commands.", + ) + @CommandArgument( + "match", + metavar="MATCH", + default=None, + nargs="?", + help="Only display commands containing given substring.", + ) + def run_debug_commands(self, match=None): + import inspect + + for command, handler in self.command_handlers.items(): + if match and match not in command: + continue + + cls = handler.cls + method = getattr(cls, getattr(handler, "method")) + + print(command) + print("=" * len(command)) + print("") + print("File: %s" % inspect.getsourcefile(method)) + print("Class: %s" % cls.__name__) + print("Method: %s" % handler.method) + print("") + + @Command( + "mach-completion", + category="misc", + description="Prints a list of completion strings for the specified command.", + ) + @CommandArgument( + "args", default=None, nargs=argparse.REMAINDER, help="Command to complete." + ) + def run_completion(self, args): + if not args: + print("\n".join(self.commands)) + return + + is_help = "help" in args + command = None + for i, arg in enumerate(args): + if arg in self.commands: + command = arg + args = args[i + 1 :] + break + + # If no command is typed yet, just offer the commands. + if not command: + print("\n".join(self.commands)) + return + + handler = self.command_handlers[command] + # If a subcommand was typed, update the handler. + for arg in args: + if arg in handler.subcommand_handlers: + handler = handler.subcommand_handlers[arg] + break + + targets = sorted(handler.subcommand_handlers.keys()) + if is_help: + print("\n".join(targets)) + return + + targets.append("help") + targets.extend(chain(*self._get_handler_options(handler).keys())) + print("\n".join(targets)) + + def _zsh_describe(self, value, description=None): + value = '"' + value.replace(":", "\\:") + if description: + description = re.sub( + r'(["\'#&;`|*?~<>^()\[\]{}$\\\x0A\xFF])', r"\\\1", description + ) + value += ":{}".format(subprocess.list2cmdline([description]).strip('"')) + + value += '"' + + return value + + @SubCommand( + "mach-completion", + "bash", + description="Print mach completion script for bash shell", + ) + @CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", + ) + def completion_bash(self, outfile): + commands_subcommands = [] + case_options = [] + case_subcommands = [] + for i, cmd in enumerate(self.commands_info): + # Build case statement for options. + options = [] + for opt_strs, description in cmd.options.items(): + for opt in opt_strs: + options.append(self._zsh_describe(opt, None).strip('"')) + + if options: + case_options.append( + "\n".join( + [ + " ({})".format(cmd.name), + ' opts="${{opts}} {}"'.format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommand options. + for sub in cmd.subcommands: + options = [] + for opt_strs, description in sub.options.items(): + for opt in opt_strs: + options.append(self._zsh_describe(opt, None)) + + if options: + case_options.append( + "\n".join( + [ + ' ("{} {}")'.format( + sub.name, sub.subcommand + ), + ' opts="${{opts}} {}"'.format( + " ".join(options) + ), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommands. + subcommands = [ + self._zsh_describe(s.subcommand, None) for s in cmd.subcommands + ] + if subcommands: + commands_subcommands.append( + '[{}]=" {} "'.format( + cmd.name, " ".join([h.subcommand for h in cmd.subcommands]) + ) + ) + + case_subcommands.append( + "\n".join( + [ + " ({})".format(cmd.name), + ' subs="${{subs}} {}"'.format( + " ".join(subcommands) + ), + " ;;", + "", + ] + ) + ) + + globalopts = [opt for opt_strs in self.global_options for opt in opt_strs] + context = { + "case_options": "\n".join(case_options), + "case_subcommands": "\n".join(case_subcommands), + "commands": " ".join(self.commands), + "commands_subcommands": " ".join(sorted(commands_subcommands)), + "globalopts": " ".join(sorted(globalopts)), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("bash", context), file=outfile) + + @SubCommand( + "mach-completion", + "zsh", + description="Print mach completion script for zsh shell", + ) + @CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", + ) + def completion_zsh(self, outfile): + commands_descriptions = [] + commands_subcommands = [] + case_options = [] + case_subcommands = [] + for i, cmd in enumerate(self.commands_info): + commands_descriptions.append(self._zsh_describe(cmd.name, cmd.description)) + + # Build case statement for options. + options = [] + for opt_strs, description in cmd.options.items(): + for opt in opt_strs: + options.append(self._zsh_describe(opt, description)) + + if options: + case_options.append( + "\n".join( + [ + " ({})".format(cmd.name), + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommand options. + for sub in cmd.subcommands: + options = [] + for opt_strs, description in sub.options.items(): + for opt in opt_strs: + options.append(self._zsh_describe(opt, description)) + + if options: + case_options.append( + "\n".join( + [ + " ({} {})".format(sub.name, sub.subcommand), + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommands. + subcommands = [ + self._zsh_describe(s.subcommand, s.description) for s in cmd.subcommands + ] + if subcommands: + commands_subcommands.append( + '[{}]=" {} "'.format( + cmd.name, " ".join([h.subcommand for h in cmd.subcommands]) + ) + ) + + case_subcommands.append( + "\n".join( + [ + " ({})".format(cmd.name), + " subs+=({})".format(" ".join(subcommands)), + " ;;", + "", + ] + ) + ) + + globalopts = [] + for opt_strings, description in self.global_options.items(): + for opt in opt_strings: + globalopts.append(self._zsh_describe(opt, description)) + + context = { + "case_options": "\n".join(case_options), + "case_subcommands": "\n".join(case_subcommands), + "commands": " ".join(sorted(commands_descriptions)), + "commands_subcommands": " ".join(sorted(commands_subcommands)), + "globalopts": " ".join(sorted(globalopts)), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("zsh", context), file=outfile) + + @SubCommand( + "mach-completion", + "fish", + description="Print mach completion script for fish shell", + ) + @CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", + ) + def completion_fish(self, outfile): + def _append_opt_strs(comp, opt_strs): + for opt in opt_strs: + if opt.startswith("--"): + comp += " -l {}".format(opt[2:]) + elif opt.startswith("-"): + comp += " -s {}".format(opt[1:]) + return comp + + globalopts = [] + for opt_strs, description in self.global_options.items(): + comp = ( + "complete -c mach -n '__fish_mach_complete_no_command' " + "-d '{}'".format(description.replace("'", "\\'")) + ) + comp = _append_opt_strs(comp, opt_strs) + globalopts.append(comp) + + cmds = [] + cmds_opts = [] + for i, cmd in enumerate(self.commands_info): + cmds.append( + "complete -c mach -f -n '__fish_mach_complete_no_command' " + "-a {} -d '{}'".format( + cmd.name, + cmd.description.replace("'", "\\'"), + ) + ) + + cmds_opts += ["# {}".format(cmd.name)] + + subcommands = " ".join([s.subcommand for s in cmd.subcommands]) + for opt_strs, description in cmd.options.items(): + comp = ( + "complete -c mach -A -n '__fish_mach_complete_command {} {}' " + "-d '{}'".format( + cmd.name, subcommands, description.replace("'", "\\'") + ) + ) + comp = _append_opt_strs(comp, opt_strs) + cmds_opts.append(comp) + + for sub in cmd.subcommands: + + for opt_strs, description in sub.options.items(): + comp = ( + "complete -c mach -A -n '__fish_mach_complete_subcommand {} {}' " + "-d '{}'".format( + sub.name, sub.subcommand, description.replace("'", "\\'") + ) + ) + comp = _append_opt_strs(comp, opt_strs) + cmds_opts.append(comp) + + description = sub.description or "" + description = description.replace("'", "\\'") + comp = ( + "complete -c mach -A -n '__fish_mach_complete_command {} {}' " + "-d '{}' -a {}".format( + cmd.name, subcommands, description, sub.subcommand + ) + ) + cmds_opts.append(comp) + + if i < len(self.commands) - 1: + cmds_opts.append("") + + context = { + "commands": " ".join(self.commands), + "command_completions": "\n".join(cmds), + "command_option_completions": "\n".join(cmds_opts), + "global_option_completions": "\n".join(globalopts), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("fish", context), file=outfile) diff --git a/python/mach/mach/commands/completion_templates/bash.template b/python/mach/mach/commands/completion_templates/bash.template new file mode 100644 index 0000000000..5372308702 --- /dev/null +++ b/python/mach/mach/commands/completion_templates/bash.template @@ -0,0 +1,62 @@ +_mach_complete() +{ + local com coms comsubs cur opts script sub subs + COMPREPLY=() + declare -A comsubs=( %(commands_subcommands)s ) + + _get_comp_words_by_ref -n : cur words + # for an alias, get the real script behind it + if [[ $(type -t ${words[0]}) == "alias" ]]; then + script=$(alias ${words[0]} | sed -E "s/alias ${words[0]}='(.*)'/\\1/") + else + script=${words[0]} + fi + # lookup for command and subcommand + for word in ${words[@]:1}; do + if [[ $word == -* ]]; then + continue + fi + + if [[ -z $com ]]; then + com=$word + elif [[ "${comsubs[$com]}" == *" $word "* ]]; then + sub=$word + break + fi + done + # completing for an option + if [[ ${cur} == -* ]] ; then + if [[ -n $com ]]; then + if [[ -n $sub ]]; then + optkey="$com $sub" + else + optkey="$com" + fi + case $optkey in +%(case_options)s + esac + else + # no command, complete global options + opts="%(globalopts)s" + fi + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + __ltrim_colon_completions "$cur" + return 0; + # completing for a command + elif [[ $cur == $com ]]; then + coms="%(commands)s" + COMPREPLY=($(compgen -W "${coms}" -- ${cur})) + __ltrim_colon_completions "$cur" + return 0 + else + if [[ -z $sub ]]; then + case "$com" in +%(case_subcommands)s + esac + COMPREPLY=($(compgen -W "${subs}" -- ${cur})) + __ltrim_colon_completions "$cur" + fi + return 0 + fi +} +complete -o default -F _mach_complete mach diff --git a/python/mach/mach/commands/completion_templates/fish.template b/python/mach/mach/commands/completion_templates/fish.template new file mode 100644 index 0000000000..8373ee4080 --- /dev/null +++ b/python/mach/mach/commands/completion_templates/fish.template @@ -0,0 +1,64 @@ +function __fish_mach_complete_no_command + for i in (commandline -opc) + if contains -- $i %(commands)s + return 1 + end + end + return 0 +end + +function __fish_mach_complete_command_matches + for i in (commandline -opc) + if contains -- $i %(commands)s + set com $i + break + end + end + + if not set -q com + return 1 + end + + if test "$com" != "$argv" + return 1 + end + return 0 +end + +function __fish_mach_complete_command + __fish_mach_complete_command_matches $argv[1] + if test $status -ne 0 + return 1 + end + + # If a subcommand is already entered, don't complete, we should defer to + # '__fish_mach_complete_subcommand'. + for i in (commandline -opc) + if contains -- $i $argv[2..-1] + return 1 + end + end + return 0 +end + +function __fish_mach_complete_subcommand + __fish_mach_complete_command_matches $argv[1] + if test $status -ne 0 + return 1 + end + + # Command matches, now check for subcommand + for i in (commandline -opc) + if contains -- $i $argv[2] + return 0 + end + end + return 1 +end + +# global options +%(global_option_completions)s +# commands +%(command_completions)s +# command options +%(command_option_completions)s diff --git a/python/mach/mach/commands/completion_templates/zsh.template b/python/mach/mach/commands/completion_templates/zsh.template new file mode 100644 index 0000000000..21677841ef --- /dev/null +++ b/python/mach/mach/commands/completion_templates/zsh.template @@ -0,0 +1,62 @@ +#compdef mach +_mach_complete() +{ + local com coms comsubs cur optkey opts state sub subs + cur=${words[${#words[@]}]} + typeset -A comsubs + comsubs=( %(commands_subcommands)s ) + + # lookup for command and subcommand + for word in ${words[@]:1}; do + if [[ $word == -* ]]; then + continue + fi + + if [[ -z $com ]]; then + com=$word + elif [[ ${comsubs[$com]} == *" $word "* ]]; then + sub=$word + break + fi + done + + # check for a subcommand + if [[ $cur == $com ]]; then + state="command" + coms=(%(commands)s) + elif [[ ${cur} == -* ]]; then + state="option" + if [[ -z $com ]]; then + # no command, use global options + opts=(%(globalopts)s) + fi + fi + case $state in + (command) + _describe 'command' coms + ;; + (option) + if [[ -n $sub ]]; then + optkey="$com $sub" + else + optkey="$com" + fi + case $optkey in +%(case_options)s + esac + _describe 'option' opts + ;; + *) + if [[ -z $sub ]]; then + # if we're completing a command with subcommands, add them here + case "$com" in +%(case_subcommands)s + esac + _describe 'subcommand' subs + fi + # also fallback to file completion + _arguments '*:file:_files' + esac +} +_mach_complete "$@" +compdef _mach_complete mach diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py new file mode 100644 index 0000000000..22e8ba9c42 --- /dev/null +++ b/python/mach/mach/commands/settings.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from textwrap import TextWrapper + +from mach.config import TYPE_CLASSES +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase + + +@CommandProvider +class Settings(MachCommandBase): + """Interact with settings for mach. + + Currently, we only provide functionality to view what settings are + available. In the future, this module will be used to modify settings, help + people create configs via a wizard, etc. + """ + + def __init__(self, *args, **kwargs): + super(Settings, self).__init__(*args, **kwargs) + self._settings = self._mach_context.settings + + @Command( + "settings", category="devenv", description="Show available config settings." + ) + @CommandArgument( + "-l", + "--list", + dest="short", + action="store_true", + help="Show settings in a concise list", + ) + def run_settings(self, short=None): + """List available settings.""" + types = {v: k for k, v in TYPE_CLASSES.items()} + wrapper = TextWrapper(initial_indent="# ", subsequent_indent="# ") + for i, section in enumerate(sorted(self._settings)): + if not short: + print("%s[%s]" % ("" if i == 0 else "\n", section)) + + for option in sorted(self._settings[section]._settings): + meta = self._settings[section].get_meta(option) + desc = meta["description"] + + if short: + print("%s.%s -- %s" % (section, option, desc.splitlines()[0])) + continue + + if option == "*": + option = "<option>" + + if "choices" in meta: + value = "{%s}" % ", ".join(meta["choices"]) + else: + value = "<%s>" % types[meta["type_cls"]] + + print(wrapper.fill(desc)) + print(";%s=%s" % (option, value)) diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py new file mode 100644 index 0000000000..0551784d90 --- /dev/null +++ b/python/mach/mach/config.py @@ -0,0 +1,434 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +r""" +This file defines classes for representing config data/settings. + +Config data is modeled as key-value pairs. Keys are grouped together into named +sections. Individual config settings (options) have metadata associated with +them. This metadata includes type, default value, valid values, etc. + +The main interface to config data is the ConfigSettings class. 1 or more +ConfigProvider classes are associated with ConfigSettings and define what +settings are available. +""" + +from __future__ import absolute_import, unicode_literals + +import collections +import os +import sys +import six +from functools import wraps +from six.moves.configparser import RawConfigParser, NoSectionError +from six import string_types + + +class ConfigException(Exception): + pass + + +class ConfigType(object): + """Abstract base class for config values.""" + + @staticmethod + def validate(value): + """Validates a Python value conforms to this type. + + Raises a TypeError or ValueError if it doesn't conform. Does not do + anything if the value is valid. + """ + + @staticmethod + def from_config(config, section, option): + """Obtain the value of this type from a RawConfigParser. + + Receives a RawConfigParser instance, a str section name, and the str + option in that section to retrieve. + + The implementation may assume the option exists in the RawConfigParser + instance. + + Implementations are not expected to validate the value. But, they + should return the appropriate Python type. + """ + + @staticmethod + def to_config(value): + return value + + +class StringType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, string_types): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +class BooleanType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, bool): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getboolean(section, option) + + @staticmethod + def to_config(value): + return "true" if value else "false" + + +class IntegerType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getint(section, option) + + +class PositiveIntegerType(IntegerType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + if value < 0: + raise ValueError() + + +class PathType(StringType): + @staticmethod + def validate(value): + if not isinstance(value, string_types): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +TYPE_CLASSES = { + "string": StringType, + "boolean": BooleanType, + "int": IntegerType, + "pos_int": PositiveIntegerType, + "path": PathType, +} + + +class DefaultValue(object): + pass + + +def reraise_attribute_error(func): + """Used to make sure __getattr__ wrappers around __getitem__ + raise AttributeError instead of KeyError. + """ + + @wraps(func) + def _(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyError: + exc_class, exc, tb = sys.exc_info() + six.reraise(AttributeError().__class__, exc, tb) + + return _ + + +class ConfigSettings(collections.Mapping): + """Interface for configuration settings. + + This is the main interface to the configuration. + + A configuration is a collection of sections. Each section contains + key-value pairs. + + When an instance is created, the caller first registers ConfigProvider + instances with it. This tells the ConfigSettings what individual settings + are available and defines extra metadata associated with those settings. + This is used for validation, etc. + + Once ConfigProvider instances are registered, a config is populated. It can + be loaded from files or populated by hand. + + ConfigSettings instances are accessed like dictionaries or by using + attributes. e.g. the section "foo" is accessed through either + settings.foo or settings['foo']. + + Sections are modeled by the ConfigSection class which is defined inside + this one. They look just like dicts or classes with attributes. To access + the "bar" option in the "foo" section: + + value = settings.foo.bar + value = settings['foo']['bar'] + value = settings.foo['bar'] + + Assignment is similar: + + settings.foo.bar = value + settings['foo']['bar'] = value + settings['foo'].bar = value + + You can even delete user-assigned values: + + del settings.foo.bar + del settings['foo']['bar'] + + If there is a default, it will be returned. + + When settings are mutated, they are validated against the registered + providers. Setting unknown settings or setting values to illegal values + will result in exceptions being raised. + """ + + class ConfigSection(collections.MutableMapping, object): + """Represents an individual config section.""" + + def __init__(self, config, name, settings): + object.__setattr__(self, "_config", config) + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_settings", settings) + + wildcard = any(s == "*" for s in self._settings) + object.__setattr__(self, "_wildcard", wildcard) + + @property + def options(self): + try: + return self._config.options(self._name) + except NoSectionError: + return [] + + def get_meta(self, option): + if option in self._settings: + return self._settings[option] + if self._wildcard: + return self._settings["*"] + raise KeyError("Option not registered with provider: %s" % option) + + def _validate(self, option, value): + meta = self.get_meta(option) + meta["type_cls"].validate(value) + + if "choices" in meta and value not in meta["choices"]: + raise ValueError( + "Value '%s' must be one of: %s" + % (value, ", ".join(sorted(meta["choices"]))) + ) + + # MutableMapping interface + def __len__(self): + return len(self.options) + + def __iter__(self): + return iter(self.options) + + def __contains__(self, k): + return self._config.has_option(self._name, k) + + def __getitem__(self, k): + meta = self.get_meta(k) + + if self._config.has_option(self._name, k): + v = meta["type_cls"].from_config(self._config, self._name, k) + else: + v = meta.get("default", DefaultValue) + + if v == DefaultValue: + raise KeyError("No default value registered: %s" % k) + + self._validate(k, v) + return v + + def __setitem__(self, k, v): + self._validate(k, v) + meta = self.get_meta(k) + + if not self._config.has_section(self._name): + self._config.add_section(self._name) + + self._config.set(self._name, k, meta["type_cls"].to_config(v)) + + def __delitem__(self, k): + self._config.remove_option(self._name, k) + + # Prune empty sections. + if not len(self._config.options(self._name)): + self._config.remove_section(self._name) + + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) + + @reraise_attribute_error + def __setattr__(self, k, v): + self.__setitem__(k, v) + + @reraise_attribute_error + def __delattr__(self, k): + self.__delitem__(k) + + def __init__(self): + self._config = RawConfigParser() + self._config.optionxform = str + + self._settings = {} + self._sections = {} + self._finalized = False + self.loaded_files = set() + + def load_file(self, filename): + self.load_files([filename]) + + def load_files(self, filenames): + """Load a config from files specified by their paths. + + Files are loaded in the order given. Subsequent files will overwrite + values from previous files. If a file does not exist, it will be + ignored. + """ + filtered = [f for f in filenames if os.path.exists(f)] + + fps = [open(f, "rt") for f in filtered] + self.load_fps(fps) + self.loaded_files.update(set(filtered)) + for fp in fps: + fp.close() + + def load_fps(self, fps): + """Load config data by reading file objects.""" + + for fp in fps: + self._config.readfp(fp) + + def write(self, fh): + """Write the config to a file object.""" + self._config.write(fh) + + @classmethod + def _format_metadata( + cls, + provider, + section, + option, + type_cls, + description, + default=DefaultValue, + extra=None, + ): + """Formats and returns the metadata for a setting. + + Each setting must have: + + section -- str section to which the setting belongs. This is how + settings are grouped. + + option -- str id for the setting. This must be unique within the + section it appears. + + type -- a ConfigType-derived type defining the type of the setting. + + description -- str describing how to use the setting and where it + applies. + + Each setting has the following optional parameters: + + default -- The default value for the setting. If None (the default) + there is no default. + + extra -- A dict of additional key/value pairs to add to the + setting metadata. + """ + if isinstance(type_cls, string_types): + type_cls = TYPE_CLASSES[type_cls] + + meta = { + "description": description, + "type_cls": type_cls, + } + + if default != DefaultValue: + meta["default"] = default + + if extra: + meta.update(extra) + + return meta + + def register_provider(self, provider): + """Register a SettingsProvider with this settings interface.""" + + if self._finalized: + raise ConfigException("Providers cannot be registered after finalized.") + + settings = provider.config_settings + if callable(settings): + settings = settings() + + config_settings = collections.defaultdict(dict) + for setting in settings: + section, option = setting[0].split(".") + + if option in config_settings[section]: + raise ConfigException( + "Setting has already been registered: %s.%s" % (section, option) + ) + + meta = self._format_metadata(provider, section, option, *setting[1:]) + config_settings[section][option] = meta + + for section_name, settings in config_settings.items(): + section = self._settings.get(section_name, {}) + + for k, v in settings.items(): + if k in section: + raise ConfigException( + "Setting already registered: %s.%s" % (section_name, k) + ) + + section[k] = v + + self._settings[section_name] = section + + def _finalize(self): + if self._finalized: + return + + for section, settings in self._settings.items(): + s = ConfigSettings.ConfigSection(self._config, section, settings) + self._sections[section] = s + + self._finalized = True + + # Mapping interface. + def __len__(self): + return len(self._settings) + + def __iter__(self): + self._finalize() + + return iter(self._sections.keys()) + + def __contains__(self, k): + return k in self._settings + + def __getitem__(self, k): + self._finalize() + + return self._sections[k] + + # Allow attribute access because it looks nice. + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py new file mode 100644 index 0000000000..ded7eb7369 --- /dev/null +++ b/python/mach/mach/decorators.py @@ -0,0 +1,371 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import argparse +import collections + +from .base import MachError +from .registrar import Registrar +from mozbuild.base import MachCommandBase + + +class _MachCommand(object): + """Container for mach command metadata.""" + + __slots__ = ( + # Content from decorator arguments to define the command. + "name", + "subcommand", + "category", + "description", + "conditions", + "_parser", + "arguments", + "argument_group_names", + "virtualenv_name", + "ok_if_tests_disabled", + # By default, subcommands will be sorted. If this is set to + # 'declaration', they will be left in declaration order. + "order", + # Describes how dispatch is performed. + # The Python class providing the command. This is the class type not + # an instance of the class. Mach will instantiate a new instance of + # the class if the command is executed. + "cls", + # The path to the `metrics.yaml` file that describes data that telemetry will + # gather for this command. This path is optional. + "metrics_path", + # The name of the method providing the command. In other words, this + # is the str name of the attribute on the class type corresponding to + # the name of the function. + "method", + # Dict of string to _MachCommand defining sub-commands for this + # command. + "subcommand_handlers", + # For subcommands, the global order that the subcommand's declaration + # was seen. + "decl_order", + ) + + def __init__( + self, + name=None, + subcommand=None, + category=None, + description=None, + conditions=None, + parser=None, + order=None, + virtualenv_name=None, + ok_if_tests_disabled=False, + ): + self.name = name + self.subcommand = subcommand + self.category = category + self.description = description + self.conditions = conditions or [] + self._parser = parser + self.arguments = [] + self.argument_group_names = [] + self.virtualenv_name = virtualenv_name + self.order = order + if ok_if_tests_disabled and category != "testing": + raise ValueError( + "ok_if_tests_disabled should only be set for " "`testing` mach commands" + ) + self.ok_if_tests_disabled = ok_if_tests_disabled + + self.cls = None + self.metrics_path = None + self.method = None + self.subcommand_handlers = {} + self.decl_order = None + + def create_instance(self, context, virtualenv_name): + metrics = None + if self.metrics_path: + metrics = context.telemetry.metrics(self.metrics_path) + return self.cls(context, virtualenv_name=virtualenv_name, metrics=metrics) + + @property + def parser(self): + # Creating CLI parsers at command dispatch time can be expensive. Make + # it possible to lazy load them by using functions. + if callable(self._parser): + self._parser = self._parser() + + return self._parser + + @property + def docstring(self): + return self.cls.__dict__[self.method].__doc__ + + def __ior__(self, other): + if not isinstance(other, _MachCommand): + raise ValueError("can only operate on _MachCommand instances") + + for a in self.__slots__: + if not getattr(self, a): + setattr(self, a, getattr(other, a)) + + return self + + +def CommandProvider(_cls=None, metrics_path=None): + def finalize(cls): + if not issubclass(cls, MachCommandBase): + raise MachError( + "Mach command provider class %s must be a subclass of " + "mozbuild.base.MachComandBase" % cls.__name__ + ) + + seen_commands = set() + + # We scan __dict__ because we only care about the classes' own attributes, + # not inherited ones. If we did inherited attributes, we could potentially + # define commands multiple times. We also sort keys so commands defined in + # the same class are grouped in a sane order. + command_methods = sorted( + [ + (name, value._mach_command) + for name, value in cls.__dict__.items() + if hasattr(value, "_mach_command") + ] + ) + + for method, command in command_methods: + # Ignore subcommands for now: we handle them later. + if command.subcommand: + continue + + seen_commands.add(command.name) + + if not command.conditions and Registrar.require_conditions: + continue + + msg = ( + "Mach command '%s' implemented incorrectly. " + + "Conditions argument must take a list " + + "of functions. Found %s instead." + ) + + if not isinstance(command.conditions, collections.Iterable): + msg = msg % (command.name, type(command.conditions)) + raise MachError(msg) + + for c in command.conditions: + if not hasattr(c, "__call__"): + msg = msg % (command.name, type(c)) + raise MachError(msg) + + command.cls = cls + command.metrics_path = metrics_path + command.method = method + + Registrar.register_command_handler(command) + + # Now do another pass to get sub-commands. We do this in two passes so + # we can check the parent command existence without having to hold + # state and reconcile after traversal. + for method, command in command_methods: + # It is a regular command. + if not command.subcommand: + continue + + if command.name not in seen_commands: + raise MachError( + "Command referenced by sub-command does not " + "exist: %s" % command.name + ) + + if command.name not in Registrar.command_handlers: + continue + + command.cls = cls + command.metrics_path = metrics_path + command.method = method + parent = Registrar.command_handlers[command.name] + + if command.subcommand in parent.subcommand_handlers: + raise MachError("sub-command already defined: %s" % command.subcommand) + + parent.subcommand_handlers[command.subcommand] = command + + return cls + + if _cls: + # The CommandProvider was used without parameters, e.g.: + # + # @CommandProvider + # class Example: + # ... + # Invoke finalize() immediately + return finalize(_cls) + else: + # The CommandProvider was used with parameters, e.g.: + # + # @CommandProvider(metrics_path='...') + # class Example: + # ... + # Return a callback which will be parameterized with the decorated class + return finalize + + +class Command(object): + """Decorator for functions or methods that provide a mach command. + + The decorator accepts arguments that define basic attributes of the + command. The following arguments are recognized: + + category -- The string category to which this command belongs. Mach's + help will group commands by category. + + description -- A brief description of what the command does. + + parser -- an optional argparse.ArgumentParser instance or callable + that returns an argparse.ArgumentParser instance to use as the + basis for the command arguments. + + For example: + + @Command('foo', category='misc', description='Run the foo action') + def foo(self): + pass + """ + + def __init__(self, name, **kwargs): + self._mach_command = _MachCommand(name=name, **kwargs) + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + + return func + + +class SubCommand(object): + """Decorator for functions or methods that provide a sub-command. + + Mach commands can have sub-commands. e.g. ``mach command foo`` or + ``mach command bar``. Each sub-command has its own parser and is + effectively its own mach command. + + The decorator accepts arguments that define basic attributes of the + sub command: + + command -- The string of the command this sub command should be + attached to. + + subcommand -- The string name of the sub command to register. + + description -- A textual description for this sub command. + """ + + global_order = 0 + + def __init__(self, command, subcommand, description=None, parser=None): + self._mach_command = _MachCommand( + name=command, subcommand=subcommand, description=description, parser=parser + ) + self._mach_command.decl_order = SubCommand.global_order + SubCommand.global_order += 1 + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + + return func + + +class CommandArgument(object): + """Decorator for additional arguments to mach subcommands. + + This decorator should be used to add arguments to mach commands. Arguments + to the decorator are proxied to ArgumentParser.add_argument(). + + For example: + + @Command('foo', help='Run the foo action') + @CommandArgument('-b', '--bar', action='store_true', default=False, + help='Enable bar mode.') + def foo(self): + pass + """ + + def __init__(self, *args, **kwargs): + if kwargs.get("nargs") == argparse.REMAINDER: + # These are the assertions we make in dispatcher.py about + # those types of CommandArguments. + assert len(args) == 1 + assert all( + k in ("default", "nargs", "help", "group", "metavar") for k in kwargs + ) + self._command_args = (args, kwargs) + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command.arguments.insert(0, self._command_args) + + return func + + +class CommandArgumentGroup(object): + """Decorator for additional argument groups to mach commands. + + This decorator should be used to add arguments groups to mach commands. + Arguments to the decorator are proxied to + ArgumentParser.add_argument_group(). + + For example: + + @Command('foo', helps='Run the foo action') + @CommandArgumentGroup('group1') + @CommandArgument('-b', '--bar', group='group1', action='store_true', + default=False, help='Enable bar mode.') + def foo(self): + pass + + The name should be chosen so that it makes sense as part of the phrase + 'Command Arguments for <name>' because that's how it will be shown in the + help message. + """ + + def __init__(self, group_name): + self._group_name = group_name + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command.argument_group_names.insert(0, self._group_name) + + return func + + +def SettingsProvider(cls): + """Class decorator to denote that this class provides Mach settings. + + When this decorator is encountered, the underlying class will automatically + be registered with the Mach registrar and will (likely) be hooked up to the + mach driver. + """ + if not hasattr(cls, "config_settings"): + raise MachError( + "@SettingsProvider must contain a config_settings attribute. It " + "may either be a list of tuples, or a callable that returns a list " + "of tuples. Each tuple must be of the form:\n" + "(<section>.<option>, <type_cls>, <description>, <default>, <choices>)\n" + "as specified by ConfigSettings._format_metadata." + ) + + Registrar.register_settings_provider(cls) + return cls diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py new file mode 100644 index 0000000000..8574401deb --- /dev/null +++ b/python/mach/mach/dispatcher.py @@ -0,0 +1,518 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function +from __future__ import absolute_import, unicode_literals + +import argparse +import difflib +import shlex +import sys + +from operator import itemgetter + +from .base import ( + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) +from .decorators import SettingsProvider + + +@SettingsProvider +class DispatchSettings: + config_settings = [ + ( + "alias.*", + "string", + """ +Create a command alias of the form `<alias>=<command> <args>`. +Aliases can also be used to set default arguments: +<command>=<command> <args> +""".strip(), + ), + ] + + +class CommandFormatter(argparse.HelpFormatter): + """Custom formatter to format just a subcommand.""" + + def add_usage(self, *args): + pass + + +class CommandAction(argparse.Action): + """An argparse action that handles mach commands. + + This class is essentially a reimplementation of argparse's sub-parsers + feature. We first tried to use sub-parsers. However, they were missing + features like grouping of commands (http://bugs.python.org/issue14037). + + The way this works involves light magic and a partial understanding of how + argparse works. + + Arguments registered with an argparse.ArgumentParser have an action + associated with them. An action is essentially a class that when called + does something with the encountered argument(s). This class is one of those + action classes. + + An instance of this class is created doing something like: + + parser.add_argument('command', action=CommandAction, registrar=r) + + Note that a mach.registrar.Registrar instance is passed in. The Registrar + holds information on all the mach commands that have been registered. + + When this argument is registered with the ArgumentParser, an instance of + this class is instantiated. One of the subtle but important things it does + is tell the argument parser that it's interested in *all* of the remaining + program arguments. So, when the ArgumentParser calls this action, we will + receive the command name plus all of its arguments. + + For more, read the docs in __call__. + """ + + def __init__( + self, + option_strings, + dest, + required=True, + default=None, + registrar=None, + context=None, + ): + # A proper API would have **kwargs here. However, since we are a little + # hacky, we intentionally omit it as a way of detecting potentially + # breaking changes with argparse's implementation. + # + # In a similar vein, default is passed in but is not needed, so we drop + # it. + argparse.Action.__init__( + self, + option_strings, + dest, + required=required, + help=argparse.SUPPRESS, + nargs=argparse.REMAINDER, + ) + + self._mach_registrar = registrar + self._context = context + + def __call__(self, parser, namespace, values, option_string=None): + """This is called when the ArgumentParser has reached our arguments. + + Since we always register ourselves with nargs=argparse.REMAINDER, + values should be a list of remaining arguments to parse. The first + argument should be the name of the command to invoke and all remaining + arguments are arguments for that command. + + The gist of the flow is that we look at the command being invoked. If + it's *help*, we handle that specially (because argparse's default help + handler isn't satisfactory). Else, we create a new, independent + ArgumentParser instance for just the invoked command (based on the + information contained in the command registrar) and feed the arguments + into that parser. We then merge the results with the main + ArgumentParser. + """ + if namespace.help: + # -h or --help is in the global arguments. + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif values: + command = values[0].lower() + args = values[1:] + if command == "help": + if args and args[0] not in ["-h", "--help"]: + # Make sure args[0] is indeed a command. + self._handle_command_help(parser, args[0], args) + else: + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif "-h" in args or "--help" in args: + # -h or --help is in the command arguments. + if "--" in args: + # -- is in command arguments + if ( + "-h" in args[: args.index("--")] + or "--help" in args[: args.index("--")] + ): + # Honor -h or --help only if it appears before -- + self._handle_command_help(parser, command, args) + sys.exit(0) + else: + self._handle_command_help(parser, command, args) + sys.exit(0) + else: + raise NoCommandError(namespace) + + # First see if the this is a user-defined alias + if command in self._context.settings.alias: + alias = self._context.settings.alias[command] + defaults = shlex.split(alias) + command = defaults.pop(0) + args = defaults + args + + if command not in self._mach_registrar.command_handlers: + # Try to find similar commands, may raise UnknownCommandError. + command = self._suggest_command(command) + + handler = self._mach_registrar.command_handlers.get(command) + + prog = command + usage = "%(prog)s [global arguments] " + command + " [command arguments]" + + subcommand = None + + # If there are sub-commands, parse the intent out immediately. + if handler.subcommand_handlers and args: + # mach <command> help <subcommand> + if set(args).intersection(("help", "--help")): + self._handle_subcommand_help(parser, handler, args) + sys.exit(0) + # mach <command> <subcommand> ... + elif args[0] in handler.subcommand_handlers: + subcommand = args[0] + handler = handler.subcommand_handlers[subcommand] + prog = prog + " " + subcommand + usage = ( + "%(prog)s [global arguments] " + + command + + " " + + subcommand + + " [command arguments]" + ) + args.pop(0) + + # We create a new parser, populate it with the command's arguments, + # then feed all remaining arguments to it, merging the results + # with ourselves. This is essentially what argparse subparsers + # do. + + parser_args = { + "add_help": False, + "usage": usage, + } + + remainder = None + + if handler.parser: + subparser = handler.parser + subparser.context = self._context + subparser.prog = subparser.prog + " " + prog + for arg in subparser._actions[:]: + if arg.nargs == argparse.REMAINDER: + subparser._actions.remove(arg) + remainder = ( + (arg.dest,), + {"default": arg.default, "nargs": arg.nargs, "help": arg.help}, + ) + else: + subparser = argparse.ArgumentParser(**parser_args) + + for arg in handler.arguments: + # Remove our group keyword; it's not needed here. + group_name = arg[1].get("group") + if group_name: + del arg[1]["group"] + + if arg[1].get("nargs") == argparse.REMAINDER: + # parse_known_args expects all argparse.REMAINDER ('...') + # arguments to be all stuck together. Instead, we want them to + # pick any extra argument, wherever they are. + # Assume a limited CommandArgument for those arguments. + assert len(arg[0]) == 1 + assert all(k in ("default", "nargs", "help", "metavar") for k in arg[1]) + remainder = arg + else: + subparser.add_argument(*arg[0], **arg[1]) + + # We define the command information on the main parser result so as to + # not interfere with arguments passed to the command. + setattr(namespace, "mach_handler", handler) + setattr(namespace, "command", command) + setattr(namespace, "subcommand", subcommand) + + command_namespace, extra = subparser.parse_known_args(args) + setattr(namespace, "command_args", command_namespace) + if remainder: + (name,), options = remainder + # parse_known_args usefully puts all arguments after '--' in + # extra, but also puts '--' there. We don't want to pass it down + # to the command handler. Note that if multiple '--' are on the + # command line, only the first one is removed, so that subsequent + # ones are passed down. + if "--" in extra: + extra.remove("--") + + # Commands with argparse.REMAINDER arguments used to force the + # other arguments to be '+' prefixed. If a user now passes such + # an argument, if will silently end up in extra. So, check if any + # of the allowed arguments appear in a '+' prefixed form, and error + # out if that's the case. + for args, _ in handler.arguments: + for arg in args: + arg = arg.replace("-", "+", 1) + if arg in extra: + raise UnrecognizedArgumentError(command, [arg]) + + if extra: + setattr(command_namespace, name, extra) + else: + setattr(command_namespace, name, options.get("default", [])) + elif extra and handler.cls.__name__ != "DeprecatedCommands": + raise UnrecognizedArgumentError(command, extra) + + def _handle_main_help(self, parser, verbose): + # Since we don't need full sub-parser support for the main help output, + # we create groups in the ArgumentParser and populate each group with + # arguments corresponding to command names. This has the side-effect + # that argparse renders it nicely. + r = self._mach_registrar + disabled_commands = [] + + cats = [(k, v[2]) for k, v in r.categories.items()] + sorted_cats = sorted(cats, key=itemgetter(1), reverse=True) + for category, priority in sorted_cats: + group = None + + for command in sorted(r.commands_by_category[category]): + handler = r.command_handlers[command] + + # Instantiate a handler class to see if it should be filtered + # out for the current context or not. Condition functions can be + # applied to the command's decorator. + if handler.conditions: + instance = handler.create_instance( + self._context, handler.virtualenv_name + ) + + is_filtered = False + for c in handler.conditions: + if not c(instance): + is_filtered = True + break + if is_filtered: + description = handler.description + disabled_command = { + "command": command, + "description": description, + } + disabled_commands.append(disabled_command) + continue + + if group is None: + title, description, _priority = r.categories[category] + group = parser.add_argument_group(title, description) + + description = handler.description + group.add_argument(command, help=description, action="store_true") + + if disabled_commands and "disabled" in r.categories: + title, description, _priority = r.categories["disabled"] + group = parser.add_argument_group(title, description) + if verbose: + for c in disabled_commands: + group.add_argument( + c["command"], help=c["description"], action="store_true" + ) + + parser.print_help() + + def _populate_command_group(self, parser, handler, group): + extra_groups = {} + for group_name in handler.argument_group_names: + group_full_name = "Command Arguments for " + group_name + extra_groups[group_name] = parser.add_argument_group(group_full_name) + + for arg in handler.arguments: + # Apply our group keyword. + group_name = arg[1].get("group") + if group_name: + del arg[1]["group"] + group = extra_groups[group_name] + group.add_argument(*arg[0], **arg[1]) + + def _get_command_arguments_help(self, handler): + # This code is worth explaining. Because we are doing funky things with + # argument registration to allow the same option in both global and + # command arguments, we can't simply put all arguments on the same + # parser instance because argparse would complain. We can't register an + # argparse subparser here because it won't properly show help for + # global arguments. So, we employ a strategy similar to command + # execution where we construct a 2nd, independent ArgumentParser for + # just the command data then supplement the main help's output with + # this 2nd parser's. We use a custom formatter class to ignore some of + # the help output. + parser_args = { + "formatter_class": CommandFormatter, + "add_help": False, + } + + if handler.parser: + c_parser = handler.parser + c_parser.context = self._context + c_parser.formatter_class = NoUsageFormatter + # Accessing _action_groups is a bit shady. We are highly dependent + # on the argparse implementation not changing. We fail fast to + # detect upstream changes so we can intelligently react to them. + group = c_parser._action_groups[1] + + # By default argparse adds two groups called "positional arguments" + # and "optional arguments". We want to rename these to reflect standard + # mach terminology. + c_parser._action_groups[0].title = "Command Parameters" + c_parser._action_groups[1].title = "Command Arguments" + + if not handler.description: + handler.description = c_parser.description + c_parser.description = None + else: + c_parser = argparse.ArgumentParser(**parser_args) + group = c_parser.add_argument_group("Command Arguments") + + self._populate_command_group(c_parser, handler, group) + + return c_parser + + def _handle_command_help(self, parser, command, args): + handler = self._mach_registrar.command_handlers.get(command) + + if not handler: + raise UnknownCommandError(command, "query") + + if handler.subcommand_handlers: + self._handle_subcommand_help(parser, handler, args) + return + + c_parser = self._get_command_arguments_help(handler) + + # Set the long help of the command to the docstring (if present) or + # the command decorator description argument (if present). + if handler.docstring: + parser.description = format_docstring(handler.docstring) + elif handler.description: + parser.description = handler.description + + parser.usage = "%(prog)s [global arguments] " + command + " [command arguments]" + + # This is needed to preserve line endings in the description field, + # which may be populated from a docstring. + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.print_help() + print("") + c_parser.print_help() + + def _handle_subcommand_main_help(self, parser, handler): + parser.usage = ( + "%(prog)s [global arguments] " + + handler.name + + " subcommand [subcommand arguments]" + ) + group = parser.add_argument_group("Sub Commands") + + def by_decl_order(item): + return item[1].decl_order + + def by_name(item): + return item[1].subcommand + + subhandlers = handler.subcommand_handlers.items() + for subcommand, subhandler in sorted( + subhandlers, + key=by_decl_order if handler.order == "declaration" else by_name, + ): + group.add_argument( + subcommand, help=subhandler.description, action="store_true" + ) + + if handler.docstring: + parser.description = format_docstring(handler.docstring) + + c_parser = self._get_command_arguments_help(handler) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + + parser.print_help() + print("") + c_parser.print_help() + + def _handle_subcommand_help(self, parser, handler, args): + subcommand = set(args).intersection(list(handler.subcommand_handlers.keys())) + if not subcommand: + return self._handle_subcommand_main_help(parser, handler) + + subcommand = subcommand.pop() + subhandler = handler.subcommand_handlers[subcommand] + + c_parser = subhandler.parser or argparse.ArgumentParser(add_help=False) + c_parser.formatter_class = CommandFormatter + + group = c_parser.add_argument_group("Sub Command Arguments") + self._populate_command_group(c_parser, subhandler, group) + + if subhandler.docstring: + parser.description = format_docstring(subhandler.docstring) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.usage = ( + "%(prog)s [global arguments] " + + handler.name + + " " + + subcommand + + " [command arguments]" + ) + + parser.print_help() + print("") + c_parser.print_help() + + def _suggest_command(self, command): + # Make sure we don't suggest any deprecated commands. + names = [ + h.name + for h in self._mach_registrar.command_handlers.values() + if h.cls.__name__ != "DeprecatedCommands" + ] + + # Bug 1577908 - We used to automatically re-execute the suggested + # command with the proper spelling. But because the `mach` driver now + # uses a whitelist to determine which command to run with Python 2, all + # misspellings are automatically run with Python 3 (and would fail if + # we were to correct a Python 2 command here). So we now suggest the + # command instead. Once the Python 3 migration has completed, we can + # turn autosuggestions back on. We could alternatively figure out a way + # to compare the suggested command against the mach whitelist. + suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5)) + suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} + raise UnknownCommandError(command, "run", suggested_commands) + + +class NoUsageFormatter(argparse.HelpFormatter): + def _format_usage(self, *args, **kwargs): + return "" + + +def format_docstring(docstring): + """Format a raw docstring into something suitable for presentation. + + This function is based on the example function in PEP-0257. + """ + if not docstring: + return "" + lines = docstring.expandtabs().splitlines() + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + return "\n".join(trimmed) diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py new file mode 100644 index 0000000000..332b44d793 --- /dev/null +++ b/python/mach/mach/logging.py @@ -0,0 +1,321 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains logging functionality for mach. It essentially provides +# support for a structured logging framework built on top of Python's built-in +# logging framework. + +from __future__ import absolute_import, unicode_literals + +try: + import blessings +except ImportError: + blessings = None + +import codecs +import json +import logging +import six +import sys +import time + + +# stdout and stderr may not necessarily be set up to write Unicode output, so +# reconfigure them if necessary. +def _wrap_stdstream(fh): + if fh in (sys.stderr, sys.stdout): + encoding = sys.getdefaultencoding() + encoding = "utf-8" if encoding in ("ascii", "charmap") else encoding + if six.PY2: + return codecs.getwriter(encoding)(fh, errors="replace") + else: + return codecs.getwriter(encoding)(fh.buffer, errors="replace") + else: + return fh + + +def format_seconds(total): + """Format number of seconds to MM:SS.DD form.""" + + minutes, seconds = divmod(total, 60) + + return "%2d:%05.2f" % (minutes, seconds) + + +class ConvertToStructuredFilter(logging.Filter): + """Filter that converts unstructured records into structured ones.""" + + def filter(self, record): + if hasattr(record, "action") and hasattr(record, "params"): + return True + + record.action = "unstructured" + record.params = {"msg": record.getMessage()} + record.msg = "{msg}" + + return True + + +class StructuredJSONFormatter(logging.Formatter): + """Log formatter that writes a structured JSON entry.""" + + def format(self, record): + action = getattr(record, "action", "UNKNOWN") + params = getattr(record, "params", {}) + + return json.dumps([record.created, action, params]) + + +class StructuredHumanFormatter(logging.Formatter): + """Log formatter that writes structured messages for humans. + + It is important that this formatter never be added to a logger that + produces unstructured/classic log messages. If it is, the call to format() + could fail because the string could contain things (like JSON) that look + like formatting character sequences. + + Because of this limitation, format() will fail with a KeyError if an + unstructured record is passed or if the structured message is malformed. + """ + + def __init__(self, start_time, write_interval=False, write_times=True): + self.start_time = start_time + self.write_interval = write_interval + self.write_times = write_times + self.last_time = None + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + elapsed = self._time(record) + + return "%s %s" % (format_seconds(elapsed), f) + + def _time(self, record): + t = record.created - self.start_time + + if self.write_interval and self.last_time is not None: + t = record.created - self.last_time + + self.last_time = record.created + + return t + + +class StructuredTerminalFormatter(StructuredHumanFormatter): + """Log formatter for structured messages writing to a terminal.""" + + def set_terminal(self, terminal): + self.terminal = terminal + self._sgr0 = terminal.normal if terminal and blessings else "" + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + t = self.terminal.blue(format_seconds(self._time(record))) + + # Some processes (notably Clang) don't reset terminal attributes after + # printing newlines. This can lead to terminal attributes getting in a + # wonky state. Work around this by sending the sgr0 sequence after every + # line to reset all attributes. For programs that rely on the next line + # inheriting the same attributes, this will prevent that from happening. + # But that's better than "corrupting" the terminal. + return "%s %s%s" % (t, self._colorize(f), self._sgr0) + + def _colorize(self, s): + if not self.terminal: + return s + + result = s + + reftest = s.startswith("REFTEST ") + if reftest: + s = s[8:] + + if s.startswith("TEST-PASS"): + result = self.terminal.green(s[0:9]) + s[9:] + elif s.startswith("TEST-UNEXPECTED"): + result = self.terminal.red(s[0:20]) + s[20:] + elif s.startswith("TEST-START"): + result = self.terminal.yellow(s[0:10]) + s[10:] + elif s.startswith("TEST-INFO"): + result = self.terminal.yellow(s[0:9]) + s[9:] + + if reftest: + result = "REFTEST " + result + + return result + + +class LoggingManager(object): + """Holds and controls global logging state. + + An application should instantiate one of these and configure it as needed. + + This class provides a mechanism to configure the output of logging data + both from mach and from the overall logging system (e.g. from other + modules). + """ + + def __init__(self): + self.start_time = time.time() + + self.json_handlers = [] + self.terminal_handler = None + self.terminal_formatter = None + + self.root_logger = logging.getLogger() + self.root_logger.setLevel(logging.DEBUG) + + # Installing NullHandler on the root logger ensures that *all* log + # messages have at least one handler. This prevents Python from + # complaining about "no handlers could be found for logger XXX." + self.root_logger.addHandler(logging.NullHandler()) + + mach_logger = logging.getLogger("mach") + mach_logger.setLevel(logging.DEBUG) + + self.structured_filter = ConvertToStructuredFilter() + + self.structured_loggers = [mach_logger] + + self._terminal = None + + @property + def terminal(self): + if not self._terminal and blessings: + # Sometimes blessings fails to set up the terminal. In that case, + # silently fail. + try: + terminal = blessings.Terminal(stream=_wrap_stdstream(sys.stdout)) + + if terminal.is_a_tty: + self._terminal = terminal + except Exception: + pass + + return self._terminal + + def add_json_handler(self, fh): + """Enable JSON logging on the specified file object.""" + + # Configure the consumer of structured messages. + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(StructuredJSONFormatter()) + handler.setLevel(logging.DEBUG) + + # And hook it up. + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.json_handlers.append(handler) + + def add_terminal_logging( + self, fh=sys.stdout, level=logging.INFO, write_interval=False, write_times=True + ): + """Enable logging to the terminal.""" + + fh = _wrap_stdstream(fh) + formatter = StructuredHumanFormatter( + self.start_time, write_interval=write_interval, write_times=write_times + ) + + if self.terminal: + formatter = StructuredTerminalFormatter( + self.start_time, write_interval=write_interval, write_times=write_times + ) + formatter.set_terminal(self.terminal) + + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(formatter) + handler.setLevel(level) + + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + self.terminal_formatter = formatter + + def replace_terminal_handler(self, handler): + """Replace the installed terminal handler. + + Returns the old handler or None if none was configured. + If the new handler is None, removes any existing handler and disables + logging to the terminal. + """ + old = self.terminal_handler + + if old: + for logger in self.structured_loggers: + logger.removeHandler(old) + + if handler: + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + + return old + + def enable_unstructured(self): + """Enable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.addFilter(self.structured_filter) + self.root_logger.addHandler(self.terminal_handler) + + def disable_unstructured(self): + """Disable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.removeFilter(self.structured_filter) + self.root_logger.removeHandler(self.terminal_handler) + + def register_structured_logger(self, logger, terminal=True, json=True): + """Register a structured logger. + + This needs to be called for all structured loggers that don't chain up + to the mach logger in order for their output to be captured. + """ + self.structured_loggers.append(logger) + + if terminal and self.terminal_handler: + logger.addHandler(self.terminal_handler) + + if json: + for handler in self.json_handlers: + logger.addHandler(handler) + + def enable_all_structured_loggers(self, terminal=True, json=True): + """Enable logging of all structured messages from all loggers. + + ``terminal`` and ``json`` determine which log handlers to operate + on. By default, all known handlers are operated on. + """ + + # Glean makes logs that we're not interested in, so we squelch them. + logging.getLogger("glean").setLevel(logging.CRITICAL) + + # Remove current handlers from all loggers so we don't double + # register handlers. + for logger in self.root_logger.manager.loggerDict.values(): + # Some entries might be logging.PlaceHolder. + if not isinstance(logger, logging.Logger): + continue + + if terminal: + logger.removeHandler(self.terminal_handler) + + if json: + for handler in self.json_handlers: + logger.removeHandler(handler) + + # Wipe out existing registered structured loggers since they + # all propagate to root logger. + self.structured_loggers = [] + self.register_structured_logger(self.root_logger, terminal=terminal, json=json) diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py new file mode 100644 index 0000000000..a25f7f4138 --- /dev/null +++ b/python/mach/mach/main.py @@ -0,0 +1,685 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides functionality for the command-line build tool +# (mach). It is packaged as a module because everything is a library. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import codecs +import errno +import imp +import logging +import os +import sys +import traceback +import uuid +from collections import Iterable + +from six import string_types + +from .base import ( + CommandContext, + MachError, + MissingFileError, + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, + FailedCommandError, +) +from .config import ConfigSettings +from .dispatcher import CommandAction +from .logging import LoggingManager +from .registrar import Registrar +from .sentry import register_sentry, NoopErrorReporter +from .telemetry import report_invocation_metrics, create_telemetry_from_environment +from .util import setenv, UserError + +SUGGEST_MACH_BUSTED_TEMPLATE = r""" +You can invoke |./mach busted| to check if this issue is already on file. If it +isn't, please use |./mach busted file %s| to report it. If |./mach busted| is +misbehaving, you can also inspect the dependencies of bug 1543241. +""".lstrip() + +MACH_ERROR_TEMPLATE = ( + r""" +The error occurred in mach itself. This is likely a bug in mach itself or a +fundamental problem with a loaded module. + +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +ERROR_FOOTER = r""" +If filing a bug, please include the full output of mach, including this error +message. + +The details of the failure are as follows: +""".lstrip() + +USER_ERROR = r""" +This is a user error and does not appear to be a bug in mach. +""".lstrip() + +COMMAND_ERROR_TEMPLATE = ( + r""" +The error occurred in the implementation of the invoked mach command. + +This should never occur and is likely a bug in the implementation of that +command. +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +MODULE_ERROR_TEMPLATE = ( + r""" +The error occurred in code that was called by the mach command. This is either +a bug in the called code itself or in the way that mach is calling it. +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +NO_COMMAND_ERROR = r""" +It looks like you tried to run mach without a command. + +Run |mach help| to show a list of commands. +""".lstrip() + +UNKNOWN_COMMAND_ERROR = r""" +It looks like you are trying to %s an unknown mach command: %s +%s +Run |mach help| to show a list of commands. +""".lstrip() + +SUGGESTED_COMMANDS_MESSAGE = r""" +Did you want to %s any of these commands instead: %s? +""" + +UNRECOGNIZED_ARGUMENT_ERROR = r""" +It looks like you passed an unrecognized argument into mach. + +The %s command does not accept the arguments: %s +""".lstrip() + +INVALID_ENTRY_POINT = r""" +Entry points should return a list of command providers or directories +containing command providers. The following entry point is invalid: + + %s + +You are seeing this because there is an error in an external module attempting +to implement a mach command. Please fix the error, or uninstall the module from +your system. +""".lstrip() + + +class ArgumentParser(argparse.ArgumentParser): + """Custom implementation argument parser to make things look pretty.""" + + def error(self, message): + """Custom error reporter to give more helpful text on bad commands.""" + if not message.startswith("argument command: invalid choice"): + argparse.ArgumentParser.error(self, message) + assert False + + print("Invalid command specified. The list of commands is below.\n") + self.print_help() + sys.exit(1) + + def format_help(self): + text = argparse.ArgumentParser.format_help(self) + + # Strip out the silly command list that would preceed the pretty list. + # + # Commands: + # {foo,bar} + # foo Do foo. + # bar Do bar. + search = "Commands:\n {" + start = text.find(search) + + if start != -1: + end = text.find("}\n", start) + assert end != -1 + + real_start = start + len("Commands:\n") + real_end = end + len("}\n") + + text = text[0:real_start] + text[real_end:] + + return text + + +class ContextWrapper(object): + def __init__(self, context, handler): + object.__setattr__(self, "_context", context) + object.__setattr__(self, "_handler", handler) + + def __getattribute__(self, key): + try: + return getattr(object.__getattribute__(self, "_context"), key) + except AttributeError as e: + try: + ret = object.__getattribute__(self, "_handler")(key) + except (AttributeError, TypeError): + # TypeError is in case the handler comes from old code not + # taking a key argument. + raise e + setattr(self, key, ret) + return ret + + def __setattr__(self, key, value): + setattr(object.__getattribute__(self, "_context"), key, value) + + +class Mach(object): + """Main mach driver type. + + This type is responsible for holding global mach state and dispatching + a command from arguments. + + The following attributes may be assigned to the instance to influence + behavior: + + populate_context_handler -- If defined, it must be a callable. The + callable signature is the following: + populate_context_handler(key=None) + It acts as a fallback getter for the mach.base.CommandContext + instance. + This allows to augment the context instance with arbitrary data + for use in command handlers. + + require_conditions -- If True, commands that do not have any condition + functions applied will be skipped. Defaults to False. + + settings_paths -- A list of files or directories in which to search + for settings files to load. + + """ + + USAGE = """%(prog)s [global arguments] command [command arguments] + +mach (German for "do") is the main interface to the Mozilla build system and +common developer tasks. + +You tell mach the command you want to perform and it does it for you. + +Some common commands are: + + %(prog)s build Build/compile the source tree. + %(prog)s help Show full help, including the list of all commands. + +To see more help for a specific command, run: + + %(prog)s help <command> +""" + + def __init__(self, cwd): + assert os.path.isdir(cwd) + + self.cwd = cwd + self.log_manager = LoggingManager() + self.logger = logging.getLogger(__name__) + self.settings = ConfigSettings() + self.settings_paths = [] + + if "MACHRC" in os.environ: + self.settings_paths.append(os.environ["MACHRC"]) + + self.log_manager.register_structured_logger(self.logger) + self.populate_context_handler = None + + def load_commands_from_directory(self, path): + """Scan for mach commands from modules in a directory. + + This takes a path to a directory, loads the .py files in it, and + registers and found mach command providers with this mach instance. + """ + for f in sorted(os.listdir(path)): + if not f.endswith(".py") or f == "__init__.py": + continue + + full_path = os.path.join(path, f) + module_name = "mach.commands.%s" % f[0:-3] + + self.load_commands_from_file(full_path, module_name=module_name) + + def load_commands_from_file(self, path, module_name=None): + """Scan for mach commands from a file. + + This takes a path to a file and loads it as a Python module under the + module name specified. If no name is specified, a random one will be + chosen. + """ + if module_name is None: + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if "mach.commands" not in sys.modules: + mod = imp.new_module("mach.commands") + sys.modules["mach.commands"] = mod + + module_name = "mach.commands.%s" % uuid.uuid4().hex + + try: + imp.load_source(module_name, path) + except IOError as e: + if e.errno != errno.ENOENT: + raise + + raise MissingFileError("%s does not exist" % path) + + def load_commands_from_entry_point(self, group="mach.providers"): + """Scan installed packages for mach command provider entry points. An + entry point is a function that returns a list of paths to files or + directories containing command providers. + + This takes an optional group argument which specifies the entry point + group to use. If not specified, it defaults to 'mach.providers'. + """ + try: + import pkg_resources + except ImportError: + print( + "Could not find setuptools, ignoring command entry points", + file=sys.stderr, + ) + return + + for entry in pkg_resources.iter_entry_points(group=group, name=None): + paths = entry.load()() + if not isinstance(paths, Iterable): + print(INVALID_ENTRY_POINT % entry) + sys.exit(1) + + for path in paths: + if os.path.isfile(path): + self.load_commands_from_file(path) + elif os.path.isdir(path): + self.load_commands_from_directory(path) + else: + print("command provider '%s' does not exist" % path) + + def define_category(self, name, title, description, priority=50): + """Provide a description for a named command category.""" + + Registrar.register_category(name, title, description, priority) + + @property + def require_conditions(self): + return Registrar.require_conditions + + @require_conditions.setter + def require_conditions(self, value): + Registrar.require_conditions = value + + def run(self, argv, stdin=None, stdout=None, stderr=None): + """Runs mach with arguments provided from the command line. + + Returns the integer exit code that should be used. 0 means success. All + other values indicate failure. + """ + sentry = NoopErrorReporter() + + # If no encoding is defined, we default to UTF-8 because without this + # Python 2.7 will assume the default encoding of ASCII. This will blow + # up with UnicodeEncodeError as soon as it encounters a non-ASCII + # character in a unicode instance. We simply install a wrapper around + # the streams and restore once we have finished. + stdin = sys.stdin if stdin is None else stdin + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + + orig_stdin = sys.stdin + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + sys.stdin = stdin + sys.stdout = stdout + sys.stderr = stderr + + orig_env = dict(os.environ) + + try: + # Load settings as early as possible so things in dispatcher.py + # can use them. + for provider in Registrar.settings_providers: + self.settings.register_provider(provider) + self.load_settings(self.settings_paths) + + if self.populate_context_handler: + topsrcdir = self.populate_context_handler("topdir") + sentry = register_sentry(argv, self.settings, topsrcdir) + else: + sentry = register_sentry(argv, self.settings) + + if sys.version_info < (3, 0): + if stdin.encoding is None: + sys.stdin = codecs.getreader("utf-8")(stdin) + + if stdout.encoding is None: + sys.stdout = codecs.getwriter("utf-8")(stdout) + + if stderr.encoding is None: + sys.stderr = codecs.getwriter("utf-8")(stderr) + + # Allow invoked processes (which may not have a handle on the + # original stdout file descriptor) to know if the original stdout + # is a TTY. This provides a mechanism to allow said processes to + # enable emitting code codes, for example. + if os.isatty(orig_stdout.fileno()): + setenv("MACH_STDOUT_ISATTY", "1") + + return self._run(argv, sentry) + except KeyboardInterrupt: + print("mach interrupted by signal or user action. Stopping.") + return 1 + + except Exception: + # _run swallows exceptions in invoked handlers and converts them to + # a proper exit code. So, the only scenario where we should get an + # exception here is if _run itself raises. If _run raises, that's a + # bug in mach (or a loaded command module being silly) and thus + # should be reported differently. + self._print_error_header(argv, sys.stdout) + print(MACH_ERROR_TEMPLATE % "general") + + exc_type, exc_value, exc_tb = sys.exc_info() + stack = traceback.extract_tb(exc_tb) + + sentry_event_id = sentry.report_exception(exc_value) + self._print_exception( + sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id + ) + + return 1 + + finally: + os.environ.clear() + os.environ.update(orig_env) + + sys.stdin = orig_stdin + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def _run(self, argv, sentry): + telemetry = create_telemetry_from_environment(self.settings) + context = CommandContext( + cwd=self.cwd, + settings=self.settings, + log_manager=self.log_manager, + commands=Registrar, + telemetry=telemetry, + ) + + if self.populate_context_handler: + context = ContextWrapper(context, self.populate_context_handler) + + parser = self.get_argument_parser(context) + context.global_parser = parser + + if not len(argv): + # We don't register the usage until here because if it is globally + # registered, argparse always prints it. This is not desired when + # running with --help. + parser.usage = Mach.USAGE + parser.print_usage() + return 0 + + try: + args = parser.parse_args(argv) + except NoCommandError: + print(NO_COMMAND_ERROR) + return 1 + except UnknownCommandError as e: + suggestion_message = ( + SUGGESTED_COMMANDS_MESSAGE % (e.verb, ", ".join(e.suggested_commands)) + if e.suggested_commands + else "" + ) + print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) + return 1 + except UnrecognizedArgumentError as e: + print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, " ".join(e.arguments))) + return 1 + + if not hasattr(args, "mach_handler"): + raise MachError("ArgumentParser result missing mach handler info.") + + handler = getattr(args, "mach_handler") + report_invocation_metrics(context.telemetry, handler.name) + + # Add JSON logging to a file if requested. + if args.logfile: + self.log_manager.add_json_handler(args.logfile) + + # Up the logging level if requested. + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + + self.log_manager.register_structured_logger(logging.getLogger("mach")) + + write_times = True + if args.log_no_times or "MACH_NO_WRITE_TIMES" in os.environ: + write_times = False + + # Always enable terminal logging. The log manager figures out if we are + # actually in a TTY or are a pipe and does the right thing. + self.log_manager.add_terminal_logging( + level=log_level, write_interval=args.log_interval, write_times=write_times + ) + + if args.settings_file: + # Argument parsing has already happened, so settings that apply + # to command line handling (e.g alias, defaults) will be ignored. + self.load_settings(args.settings_file) + + try: + return Registrar._run_command_handler( + handler, + context, + debug_command=args.debug_command, + **vars(args.command_args) + ) + except KeyboardInterrupt as ki: + raise ki + except FailedCommandError as e: + print(e) + return e.exit_code + except UserError: + # We explicitly don't report UserErrors to Sentry. + exc_type, exc_value, exc_tb = sys.exc_info() + # The first two frames are us and are never used. + stack = traceback.extract_tb(exc_tb)[2:] + self._print_error_header(argv, sys.stdout) + print(USER_ERROR) + self._print_exception(sys.stdout, exc_type, exc_value, stack) + return 1 + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + sentry_event_id = sentry.report_exception(exc_value) + + # The first two frames are us and are never used. + stack = traceback.extract_tb(exc_tb)[2:] + + # If we have nothing on the stack, the exception was raised as part + # of calling the @Command method itself. This likely means a + # mismatch between @CommandArgument and arguments to the method. + # e.g. there exists a @CommandArgument without the corresponding + # argument on the method. We handle that here until the module + # loader grows the ability to validate better. + if not len(stack): + print(COMMAND_ERROR_TEMPLATE % handler.name) + self._print_exception( + sys.stdout, + exc_type, + exc_value, + traceback.extract_tb(exc_tb), + sentry_event_id=sentry_event_id, + ) + return 1 + + # Split the frames into those from the module containing the + # command and everything else. + command_frames = [] + other_frames = [] + + initial_file = stack[0][0] + + for frame in stack: + if frame[0] == initial_file: + command_frames.append(frame) + else: + other_frames.append(frame) + + # If the exception was in the module providing the command, it's + # likely the bug is in the mach command module, not something else. + # If there are other frames, the bug is likely not the mach + # command's fault. + self._print_error_header(argv, sys.stdout) + + if len(other_frames): + print(MODULE_ERROR_TEMPLATE % handler.name) + else: + print(COMMAND_ERROR_TEMPLATE % handler.name) + + self._print_exception( + sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id + ) + + return 1 + + def log(self, level, action, params, format_str): + """Helper method to record a structured log event.""" + self.logger.log(level, format_str, extra={"action": action, "params": params}) + + def _print_error_header(self, argv, fh): + fh.write("Error running mach:\n\n") + fh.write(" ") + fh.write(repr(argv)) + fh.write("\n\n") + + def _print_exception(self, fh, exc_type, exc_value, stack, sentry_event_id=None): + fh.write(ERROR_FOOTER) + fh.write("\n") + + for l in traceback.format_exception_only(exc_type, exc_value): + fh.write(l) + + fh.write("\n") + for l in traceback.format_list(stack): + fh.write(l) + + if not sentry_event_id: + return + + fh.write("\nSentry event ID: {}\n".format(sentry_event_id)) + + def load_settings(self, paths): + """Load the specified settings files. + + If a directory is specified, the following basenames will be + searched for in this order: + + machrc, .machrc + """ + if isinstance(paths, string_types): + paths = [paths] + + valid_names = ("machrc", ".machrc") + + def find_in_dir(base): + if os.path.isfile(base): + return base + + for name in valid_names: + path = os.path.join(base, name) + if os.path.isfile(path): + return path + + files = map(find_in_dir, self.settings_paths) + files = filter(bool, files) + + self.settings.load_files(files) + + def get_argument_parser(self, context): + """Returns an argument parser for the command-line interface.""" + + parser = ArgumentParser( + add_help=False, + usage="%(prog)s [global arguments] " "command [command arguments]", + ) + + # WARNING!!! If you add a global argument here, also add it to the + # global argument handling in the top-level `mach` script. + # Order is important here as it dictates the order the auto-generated + # help messages are printed. + global_group = parser.add_argument_group("Global Arguments") + + global_group.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="Print verbose output.", + ) + global_group.add_argument( + "-l", + "--log-file", + dest="logfile", + metavar="FILENAME", + type=argparse.FileType("a"), + help="Filename to write log data to.", + ) + global_group.add_argument( + "--log-interval", + dest="log_interval", + action="store_true", + default=False, + help="Prefix log line with interval from last message rather " + "than relative time. Note that this is NOT execution time " + "if there are parallel operations.", + ) + suppress_log_by_default = False + if "INSIDE_EMACS" in os.environ: + suppress_log_by_default = True + global_group.add_argument( + "--log-no-times", + dest="log_no_times", + action="store_true", + default=suppress_log_by_default, + help="Do not prefix log lines with times. By default, " + "mach will prefix each output line with the time since " + "command start.", + ) + global_group.add_argument( + "-h", + "--help", + dest="help", + action="store_true", + default=False, + help="Show this help message.", + ) + global_group.add_argument( + "--debug-command", + action="store_true", + help="Start a Python debugger when command is dispatched.", + ) + global_group.add_argument( + "--settings", + dest="settings_file", + metavar="FILENAME", + default=None, + help="Path to settings file.", + ) + + # We need to be last because CommandAction swallows all remaining + # arguments and argparse parses arguments in the order they were added. + parser.add_argument( + "command", action=CommandAction, registrar=Registrar, context=context + ) + + return parser diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/mixin/__init__.py diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py new file mode 100644 index 0000000000..a7d36754a4 --- /dev/null +++ b/python/mach/mach/mixin/logging.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging + + +class LoggingMixin(object): + """Provides functionality to control logging.""" + + def populate_logger(self, name=None): + """Ensure this class instance has a logger associated with it. + + Users of this mixin that call log() will need to ensure self._logger is + a logging.Logger instance before they call log(). This function ensures + self._logger is defined by populating it if it isn't. + """ + if hasattr(self, "_logger"): + return + + if name is None: + name = ".".join([self.__module__, self.__class__.__name__]) + + self._logger = logging.getLogger(name) + + def log(self, level, action, params, format_str): + """Log a structured log event. + + A structured log event consists of a logging level, a string action, a + dictionary of attributes, and a formatting string. + + The logging level is one of the logging.* constants, such as + logging.INFO. + + The action string is essentially the enumeration of the event. Each + different type of logged event should have a different action. + + The params dict is the metadata constituting the logged event. + + The formatting string is used to convert the structured message back to + human-readable format. Conversion back to human-readable form is + performed by calling format() on this string, feeding into it the dict + of attributes constituting the event. + + Example Usage + ------------- + + self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, + 'User login: {username}') + """ + self._logger.log(level, format_str, extra={"action": action, "params": params}) diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py new file mode 100644 index 0000000000..9b09c4f387 --- /dev/null +++ b/python/mach/mach/mixin/process.py @@ -0,0 +1,206 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides mixins to perform process execution. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import signal +import subprocess +import sys + +from mozbuild.util import ensure_subprocess_env +from mozprocess.processhandler import ProcessHandlerMixin + +from .logging import LoggingMixin + + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if "SHELL" in os.environ: + _current_shell = os.environ["SHELL"] +elif "MOZILLABUILD" in os.environ: + _current_shell = os.environ["MOZILLABUILD"] + "/msys/bin/sh.exe" +elif "COMSPEC" in os.environ: + _current_shell = os.environ["COMSPEC"] +elif sys.platform != "win32": + # Fall back to a standard shell. + _current_shell = "/bin/sh" +else: + raise Exception("Could not detect environment shell!") + +_in_msys = False + +if os.environ.get("MSYSTEM", None) in ("MINGW32", "MINGW64"): + _in_msys = True + + if not _current_shell.lower().endswith(".exe"): + _current_shell += ".exe" + + +class ProcessExecutionMixin(LoggingMixin): + """Mix-in that provides process execution functionality.""" + + def run_process( + self, + args=None, + cwd=None, + append_env=None, + explicit_env=None, + log_name=None, + log_level=logging.INFO, + line_handler=None, + require_unix_environment=False, + ensure_exit_code=0, + ignore_children=False, + pass_thru=False, + python_unbuffered=True, + ): + """Runs a single process to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + + ignore_children is proxied to mozprocess's ignore_children. + + ensure_exit_code is used to ensure the exit code of a process matches + what is expected. If it is an integer, we raise an Exception if the + exit code does not match this value. If it is True, we ensure the exit + code is 0. If it is False, we don't perform any exit code validation. + + pass_thru is a special execution mode where the child process inherits + this process's standard file handles (stdin, stdout, stderr) as well as + additional file descriptors. It should be used for interactive processes + where buffering from mozprocess could be an issue. pass_thru does not + use mozprocess. Therefore, arguments like log_name, line_handler, + and ignore_children have no effect. + + When python_unbuffered is set, the PYTHONUNBUFFERED environment variable + will be set in the child process. This is normally advantageous (see bug + 1627873) but is detrimental in certain circumstances (specifically, we + have seen issues when using pass_thru mode to open a Python subshell, as + in bug 1628838). This variable should be set to False to avoid bustage + in those circumstances. + """ + args = self._normalize_command(args, require_unix_environment) + + self.log(logging.INFO, "new_process", {"args": " ".join(args)}, "{args}") + + def handleLine(line): + # Converts str to unicode on Python 2 and bytes to str on Python 3. + if isinstance(line, bytes): + line = line.decode(sys.stdout.encoding or "utf-8", "replace") + + if line_handler: + line_handler(line) + + if line.startswith("BUILDTASK") or not log_name: + return + + self.log(log_level, log_name, {"line": line.rstrip()}, "{line}") + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(append_env) + + if python_unbuffered: + use_env["PYTHONUNBUFFERED"] = "1" + + self.log(logging.DEBUG, "process", {"env": str(use_env)}, "Environment: {env}") + + use_env = ensure_subprocess_env(use_env) + if pass_thru: + proc = subprocess.Popen(args, cwd=cwd, env=use_env, close_fds=False) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as + # a result of Ctrl+C, proc.wait() will return a status code, and, + # we get out of the loop. If it doesn't, like e.g. gdb, we continue + # waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + else: + p = ProcessHandlerMixin( + args, + cwd=cwd, + env=use_env, + processOutputLine=[handleLine], + universal_newlines=True, + ignore_children=ignore_children, + ) + p.run() + p.processOutput() + status = None + sig = None + while status is None: + try: + if sig is None: + status = p.wait() + else: + status = p.kill(sig=sig) + except KeyboardInterrupt: + if sig is None: + sig = signal.SIGINT + elif sig == signal.SIGINT: + # If we've already tried SIGINT, escalate. + sig = signal.SIGKILL + + if ensure_exit_code is False: + return status + + if ensure_exit_code is True: + ensure_exit_code = 0 + + if status != ensure_exit_code: + raise Exception( + "Process executed with non-0 exit code %d: %s" % (status, args) + ) + + return status + + def _normalize_command(self, args, require_unix_environment): + """Adjust command arguments to run in the necessary environment. + + This exists mainly to facilitate execution of programs requiring a *NIX + shell when running on Windows. The caller specifies whether a shell + environment is required. If it is and we are running on Windows but + aren't running in the UNIX-like msys environment, then we rewrite the + command to execute via a shell. + """ + assert isinstance(args, list) and len(args) + + if not require_unix_environment or not _in_msys: + return args + + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace("\\", "/") + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:]) + return [_current_shell, "-c", cline] diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py new file mode 100644 index 0000000000..ef18869921 --- /dev/null +++ b/python/mach/mach/registrar.py @@ -0,0 +1,164 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import time + +import six + +from .base import MachError + +INVALID_COMMAND_CONTEXT = r""" +It looks like you tried to run a mach command from an invalid context. The %s +command failed to meet the following conditions: %s + +Run |mach help| to show a list of all commands available to the current context. +""".lstrip() + + +class MachRegistrar(object): + """Container for mach command and config providers.""" + + def __init__(self): + self.command_handlers = {} + self.commands_by_category = {} + self.settings_providers = set() + self.categories = {} + self.require_conditions = False + self.command_depth = 0 + + def register_command_handler(self, handler): + name = handler.name + + if not handler.category: + raise MachError( + "Cannot register a mach command without a " "category: %s" % name + ) + + if handler.category not in self.categories: + raise MachError( + "Cannot register a command to an undefined " + "category: %s -> %s" % (name, handler.category) + ) + + self.command_handlers[name] = handler + self.commands_by_category[handler.category].add(name) + + def register_settings_provider(self, cls): + self.settings_providers.add(cls) + + def register_category(self, name, title, description, priority=50): + self.categories[name] = (title, description, priority) + self.commands_by_category[name] = set() + + @classmethod + def _condition_failed_message(cls, name, conditions): + msg = ["\n"] + for c in conditions: + part = [" %s" % getattr(c, "__name__", c)] + if c.__doc__ is not None: + part.append(c.__doc__) + msg.append(" - ".join(part)) + return INVALID_COMMAND_CONTEXT % (name, "\n".join(msg)) + + @classmethod + def _instance(_, handler, context, **kwargs): + if context is None: + raise ValueError("Expected a non-None context.") + + prerun = getattr(context, "pre_dispatch_handler", None) + if prerun: + prerun(context, handler, args=kwargs) + + context.handler = handler + return handler.create_instance(context, handler.virtualenv_name) + + @classmethod + def _fail_conditions(_, handler, instance): + fail_conditions = [] + if handler.conditions: + for c in handler.conditions: + if not c(instance): + fail_conditions.append(c) + + return fail_conditions + + def _run_command_handler(self, handler, context, debug_command=False, **kwargs): + instance = MachRegistrar._instance(handler, context, **kwargs) + fail_conditions = MachRegistrar._fail_conditions(handler, instance) + if fail_conditions: + print( + MachRegistrar._condition_failed_message(handler.name, fail_conditions) + ) + return 1 + + self.command_depth += 1 + fn = getattr(instance, handler.method) + + start_time = time.time() + + if debug_command: + import pdb + + result = pdb.runcall(fn, **kwargs) + else: + result = fn(**kwargs) + + end_time = time.time() + + result = result or 0 + assert isinstance(result, six.integer_types) + + if not debug_command: + postrun = getattr(context, "post_dispatch_handler", None) + if postrun: + postrun( + context, + handler, + instance, + not result, + start_time, + end_time, + self.command_depth, + args=kwargs, + ) + self.command_depth -= 1 + + return result + + def dispatch(self, name, context, argv=None, subcommand=None, **kwargs): + """Dispatch/run a command. + + Commands can use this to call other commands. + """ + handler = self.command_handlers[name] + + if subcommand: + handler = handler.subcommand_handlers[subcommand] + + if handler.parser: + parser = handler.parser + + # save and restore existing defaults so **kwargs don't persist across + # subsequent invocations of Registrar.dispatch() + old_defaults = parser._defaults.copy() + parser.set_defaults(**kwargs) + kwargs, unknown = parser.parse_known_args(argv or []) + kwargs = vars(kwargs) + parser._defaults = old_defaults + + if unknown: + if subcommand: + name = "{} {}".format(name, subcommand) + parser.error( + "unrecognized arguments for {}: {}".format( + name, ", ".join(["'{}'".format(arg) for arg in unknown]) + ) + ) + + return self._run_command_handler(handler, context, **kwargs) + + +Registrar = MachRegistrar() diff --git a/python/mach/mach/sentry.py b/python/mach/mach/sentry.py new file mode 100644 index 0000000000..94cbb1dcf6 --- /dev/null +++ b/python/mach/mach/sentry.py @@ -0,0 +1,182 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import abc +import re +from os.path import ( + abspath, + expanduser, + join, +) + +import sentry_sdk +from mozboot.util import get_state_dir +from mach.telemetry import is_telemetry_enabled +from mozversioncontrol import ( + get_repository_object, + InvalidRepoPath, + MissingUpstreamRepo, + MissingVCSTool, +) +from six import string_types + +# The following developers frequently modify mach code, and testing will commonly cause +# exceptions to be thrown. We don't want these exceptions reported to Sentry. +_DEVELOPER_BLOCKLIST = [ + "ahalberstadt@mozilla.com", + "mhentges@mozilla.com", + "rstewart@mozilla.com", + "sledru@mozilla.com", +] +# https://sentry.prod.mozaws.net/operations/mach/ +_SENTRY_DSN = "https://8228c9aff64949c2ba4a2154dc515f55@sentry.prod.mozaws.net/525" + + +class ErrorReporter(object): + @abc.abstractmethod + def report_exception(self, exception): + """Report the exception to remote error-tracking software.""" + + +class SentryErrorReporter(ErrorReporter): + """Reports errors using Sentry.""" + + def report_exception(self, exception): + return sentry_sdk.capture_exception(exception) + + +class NoopErrorReporter(ErrorReporter): + """Drops errors instead of reporting them. + + This is useful in cases where error-reporting is specifically disabled, such as + when telemetry hasn't been allowed. + """ + + def report_exception(self, exception): + return None + + +def register_sentry(argv, settings, topsrcdir=None): + if not is_telemetry_enabled(settings): + return NoopErrorReporter() + + if topsrcdir: + repo = _get_repository_object(topsrcdir) + if repo is not None: + email = repo.get_user_email() + if email in _DEVELOPER_BLOCKLIST: + return NoopErrorReporter() + + sentry_sdk.init( + _SENTRY_DSN, before_send=lambda event, _: _process_event(event, topsrcdir) + ) + sentry_sdk.add_breadcrumb(message="./mach {}".format(" ".join(argv))) + return SentryErrorReporter() + + +def _process_event(sentry_event, topsrcdir): + if _any_modified_files_matching_event(sentry_event, topsrcdir): + # Returning None causes the event to be dropped: + # https://docs.sentry.io/platforms/python/configuration/filtering/#using-beforesend + return None + for map_fn in (_settle_mach_module_id, _patch_absolute_paths, _delete_server_name): + sentry_event = map_fn(sentry_event, topsrcdir) + return sentry_event + + +def _settle_mach_module_id(sentry_event, _): + # Sentry groups issues according to the stack frames and their associated + # "module" properties. However, one of the modules is being reported + # like "mach.commands.26a828ef5164403eaff4305ab4cb0fab" (with a generated id). + # This function replaces that generated id with the static string "<generated>" + # so that grouping behaves as expected + + stacktrace_frames = sentry_event["exception"]["values"][0]["stacktrace"]["frames"] + for frame in stacktrace_frames: + module = frame.get("module") + if not module: + continue + + module = re.sub( + "mach\\.commands\\.[a-f0-9]{32}", "mach.commands.<generated>", module + ) + frame["module"] = module + return sentry_event + + +def _patch_absolute_paths(sentry_event, topsrcdir): + # As discussed here (https://bugzilla.mozilla.org/show_bug.cgi?id=1636251#c28), + # we remove usernames from file names with a best-effort basis. The most likely + # place for usernames to manifest in Sentry information is within absolute paths, + # such as: "/home/mitch/dev/firefox/mach" + # We replace the state_dir, obj_dir, src_dir with "<...>" placeholders. + # Note that we also do a blanket find-and-replace of the user's name with "<user>", + # which may have ill effects if the user's name is, by happenstance, a substring + # of some other value within the Sentry event. + def recursive_patch(value, needle, replacement): + if isinstance(value, list): + return [recursive_patch(v, needle, replacement) for v in value] + elif isinstance(value, dict): + for key in list(value.keys()): + next_value = value.pop(key) + key = needle.sub(replacement, key) + value[key] = recursive_patch(next_value, needle, replacement) + return value + elif isinstance(value, string_types): + return needle.sub(replacement, value) + else: + return value + + for (needle, replacement) in ( + (get_state_dir(), "<statedir>"), + (topsrcdir, "<topsrcdir>"), + (expanduser("~"), "~"), + # Sentry converts "vars" to their "representations". When paths are in local + # variables on Windows, "C:\Users\MozillaUser\Desktop" becomes + # "'C:\\Users\\MozillaUser\\Desktop'". To still catch this case, we "repr" + # the home directory and scrub the beginning and end quotes, then + # find-and-replace on that. + (repr(expanduser("~"))[1:-1], "~"), + ): + if needle is None: + continue # topsrcdir isn't always defined + needle_regex = re.compile(re.escape(needle), re.IGNORECASE) + sentry_event = recursive_patch(sentry_event, needle_regex, replacement) + return sentry_event + + +def _delete_server_name(sentry_event, _): + sentry_event.pop("server_name") + return sentry_event + + +def _get_repository_object(topsrcdir): + try: + return get_repository_object(topsrcdir) + except (InvalidRepoPath, MissingVCSTool): + return None + + +def _any_modified_files_matching_event(sentry_event, topsrcdir): + repo = _get_repository_object(topsrcdir) + if repo is None: + return False # Conservatively assume the tree is clean. + + try: + files = set(repo.get_outgoing_files()) | set(repo.get_changed_files()) + except MissingUpstreamRepo: + return False + + files = set(abspath(join(topsrcdir, s)) for s in files) + + # Return True iff the abs_path in any of the stack traces match the set of + # changed files locally. Be careful not to crash if something's missing from + # the dictionary. + for exception in sentry_event.get("exception", {}).get("values", []): + for frame in exception.get("stacktrace", {}).get("frames", []): + if frame.get("abs_path", None) in files: + return True + return False diff --git a/python/mach/mach/telemetry.py b/python/mach/mach/telemetry.py new file mode 100644 index 0000000000..4405d1ce2d --- /dev/null +++ b/python/mach/mach/telemetry.py @@ -0,0 +1,147 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, absolute_import + +import os +import sys + +import six +from mock import Mock + +from mozboot.util import get_state_dir, get_mach_virtualenv_binary +from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException +from mozbuild.telemetry import filter_args +import mozpack.path + +MACH_METRICS_PATH = os.path.abspath(os.path.join(__file__, "..", "..", "metrics.yaml")) + + +class NoopTelemetry(object): + def __init__(self, failed_glean_import): + self._failed_glean_import = failed_glean_import + + def metrics(self, metrics_path): + return Mock() + + def submit(self, is_bootstrap): + if self._failed_glean_import and not is_bootstrap: + print( + "Glean could not be found, so telemetry will not be reported. " + "You may need to run |mach bootstrap|.", + file=sys.stderr, + ) + + +class GleanTelemetry(object): + """Records and sends Telemetry using Glean. + + Metrics are defined in python/mozbuild/metrics.yaml. + Pings are defined in python/mozbuild/pings.yaml. + + The "metrics" and "pings" properties may be replaced with no-op implementations if + Glean isn't available. This allows consumers to report telemetry without having + to guard against incompatible environments. + """ + + def __init__( + self, + ): + self._metrics_cache = {} + + def metrics(self, metrics_path): + if metrics_path not in self._metrics_cache: + from glean import load_metrics + + metrics = load_metrics(metrics_path) + self._metrics_cache[metrics_path] = metrics + + return self._metrics_cache[metrics_path] + + def submit(self, _): + from pathlib import Path + from glean import load_pings + + pings = load_pings(Path(__file__).parent.parent / "pings.yaml") + pings.usage.submit() + + +def create_telemetry_from_environment(settings): + """Creates and a Telemetry instance based on system details. + + If telemetry isn't enabled, the current interpreter isn't Python 3, or Glean + can't be imported, then a "mock" telemetry instance is returned that doesn't + set or record any data. This allows consumers to optimistically set telemetry + data without needing to specifically handle the case where the current system + doesn't support it. + """ + + is_mach_virtualenv = mozpack.path.normpath(sys.executable) == mozpack.path.normpath( + get_mach_virtualenv_binary(py2=six.PY2) + ) + + if not ( + is_applicable_telemetry_environment() + # Glean is not compatible with Python 2 + and sys.version_info >= (3, 0) + # If not using the mach virtualenv (e.g.: bootstrap uses native python) + # then we can't guarantee that the glean package that we import is a + # compatible version. Therefore, don't use glean. + and is_mach_virtualenv + ): + return NoopTelemetry(False) + + try: + from glean import Glean + except ImportError: + return NoopTelemetry(True) + + from pathlib import Path + + Glean.initialize( + "mozilla.mach", + "Unknown", + is_telemetry_enabled(settings), + data_dir=Path(get_state_dir()) / "glean", + ) + return GleanTelemetry() + + +def report_invocation_metrics(telemetry, command): + metrics = telemetry.metrics(MACH_METRICS_PATH) + metrics.mach.command.set(command) + metrics.mach.duration.start() + + try: + instance = MozbuildObject.from_environment() + except BuildEnvironmentNotFoundException: + # Mach may be invoked with the state dir as the current working + # directory, in which case we're not able to find the topsrcdir (so + # we can't create a MozbuildObject instance). + # Without this information, we're unable to filter argv paths, so + # we skip submitting them to telemetry. + return + metrics.mach.argv.set(filter_args(command, sys.argv, instance)) + + +def is_applicable_telemetry_environment(): + if os.environ.get("MACH_MAIN_PID") != str(os.getpid()): + # This is a child mach process. Since we're collecting telemetry for the parent, + # we don't want to collect telemetry again down here. + return False + + if any(e in os.environ for e in ("MOZ_AUTOMATION", "TASK_ID")): + return False + + return True + + +def is_telemetry_enabled(settings): + if os.environ.get("DISABLE_TELEMETRY") == "1": + return False + + try: + return settings.build.telemetry + except (AttributeError, KeyError): + return False diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py new file mode 100644 index 0000000000..01f9f43e8a --- /dev/null +++ b/python/mach/mach/terminal.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This file contains code for interacting with terminals. + +All the terminal interaction code is consolidated so the complexity can be in +one place, away from code that is commonly looked at. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import sys + +from six.moves import range + + +class LoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This is alternative terminal logging handler which contains smarts for + emitting terminal control characters properly. Currently, it has generic + support for "footer" elements at the bottom of the screen. Functionality + can be added when needed. + """ + + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write("\n") + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.flush() + + +class TerminalFooter(object): + """Represents something drawn on the bottom of a terminal.""" + + def __init__(self, terminal): + self.t = terminal + self.fh = sys.stdout + + def _clear_lines(self, n): + for i in range(n): + self.fh.write(self.t.move_x(0)) + self.fh.write(self.t.clear_eol()) + self.fh.write(self.t.move_up()) + + self.fh.write(self.t.move_down()) + self.fh.write(self.t.move_x(0)) + + def clear(self): + raise Exception("clear() must be implemented.") + + def draw(self): + raise Exception("draw() must be implemented.") diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/__init__.py diff --git a/python/mach/mach/test/conftest.py b/python/mach/mach/test/conftest.py new file mode 100644 index 0000000000..b68f5f232f --- /dev/null +++ b/python/mach/mach/test/conftest.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import os +import sys +import unittest + +import pytest +import six +from buildconfig import topsrcdir + +try: + from StringIO import StringIO +except ImportError: + # TODO io.StringIO causes failures with Python 2 (needs to be sorted out) + from io import StringIO + +from mach.main import Mach + +here = os.path.abspath(os.path.dirname(__file__)) +PROVIDER_DIR = os.path.join(here, "providers") + + +@pytest.fixture(scope="class") +def get_mach(request): + def _populate_context(key): + if key == "topdir": + return topsrcdir + + def inner(provider_files=None, entry_point=None, context_handler=None): + m = Mach(os.getcwd()) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + m.define_category("misc", "Mach misc", "Testing for mach core", 20) + m.populate_context_handler = context_handler or _populate_context + + if provider_files: + if isinstance(provider_files, six.string_types): + provider_files = [provider_files] + + for path in provider_files: + m.load_commands_from_file(os.path.join(PROVIDER_DIR, path)) + + if entry_point: + m.load_commands_from_entry_point(entry_point) + + return m + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls.get_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.fixture(scope="class") +def run_mach(request, get_mach): + def inner(argv, *args, **kwargs): + m = get_mach(*args, **kwargs) + + stdout = StringIO() + stderr = StringIO() + + if sys.version_info < (3, 0): + stdout.encoding = "UTF-8" + stderr.encoding = "UTF-8" + + try: + result = m.run(argv, stdout=stdout, stderr=stderr) + except SystemExit: + result = None + + return (result, stdout.getvalue(), stderr.getvalue()) + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls._run_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestBase(unittest.TestCase): + pass diff --git a/python/mach/mach/test/invoke_mach_command.py b/python/mach/mach/test/invoke_mach_command.py new file mode 100644 index 0000000000..014010fb33 --- /dev/null +++ b/python/mach/mach/test/invoke_mach_command.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import +import subprocess +import sys + +subprocess.check_call([sys.executable] + sys.argv[1:]) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/providers/__init__.py diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py new file mode 100644 index 0000000000..3eeed5a9ec --- /dev/null +++ b/python/mach/mach/test/providers/basic.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +from __future__ import unicode_literals + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase + + +@CommandProvider +class ConditionsProvider(MachCommandBase): + @Command("cmd_foo", category="testing") + def run_foo(self): + pass + + @Command("cmd_bar", category="testing") + @CommandArgument("--baz", action="store_true", help="Run with baz") + def run_bar(self, baz=None): + pass diff --git a/python/mach/mach/test/providers/commands.py b/python/mach/mach/test/providers/commands.py new file mode 100644 index 0000000000..9463c6b403 --- /dev/null +++ b/python/mach/mach/test/providers/commands.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from functools import partial + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase + + +def is_foo(cls): + """Foo must be true""" + return cls.foo + + +def is_bar(val, cls): + """Bar must equal val""" + return cls.bar == val + + +@CommandProvider +class MachCommands(MachCommandBase): + foo = True + bar = False + + @Command("cmd_foo", category="testing") + @CommandArgument("--arg", default=None, help="Argument help.") + def run_foo(self): + pass + + @Command("cmd_bar", category="testing", conditions=[partial(is_bar, False)]) + def run_bar(self): + pass + + @Command( + "cmd_foobar", category="testing", conditions=[is_foo, partial(is_bar, True)] + ) + def run_foobar(self): + pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py new file mode 100644 index 0000000000..2ca619fa4d --- /dev/null +++ b/python/mach/mach/test/providers/conditions.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase + + +def is_foo(cls): + """Foo must be true""" + return cls.foo + + +def is_bar(cls): + """Bar must be true""" + return cls.bar + + +@CommandProvider +class ConditionsProvider(MachCommandBase): + foo = True + bar = False + + @Command("cmd_foo", category="testing", conditions=[is_foo]) + def run_foo(self): + pass + + @Command("cmd_bar", category="testing", conditions=[is_bar]) + def run_bar(self): + pass + + @Command("cmd_foobar", category="testing", conditions=[is_foo, is_bar]) + def run_foobar(self): + pass + + +@CommandProvider +class ConditionsContextProvider(MachCommandBase): + def __init__(self, *args, **kwargs): + super(ConditionsContextProvider, self).__init__(*args, **kwargs) + self.foo = self._mach_context.foo + self.bar = self._mach_context.bar + + @Command("cmd_foo_ctx", category="testing", conditions=[is_foo]) + def run_foo(self): + pass + + @Command("cmd_bar_ctx", category="testing", conditions=[is_bar]) + def run_bar(self): + pass + + @Command("cmd_foobar_ctx", category="testing", conditions=[is_foo, is_bar]) + def run_foobar(self): + pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py new file mode 100644 index 0000000000..5685fafb8d --- /dev/null +++ b/python/mach/mach/test/providers/conditions_invalid.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase + + +@CommandProvider +class ConditionsProvider(MachCommandBase): + @Command("cmd_foo", category="testing", conditions=["invalid"]) + def run_foo(self): + pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py new file mode 100644 index 0000000000..b562ee1566 --- /dev/null +++ b/python/mach/mach/test/providers/throw.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from __future__ import unicode_literals + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) +from mach.test.providers import throw2 +from mozbuild.base import MachCommandBase + + +@CommandProvider +class TestCommandProvider(MachCommandBase): + @Command("throw", category="testing") + @CommandArgument("--message", "-m", default="General Error") + def throw(self, message): + raise Exception(message) + + @Command("throw_deep", category="testing") + @CommandArgument("--message", "-m", default="General Error") + def throw_deep(self, message): + throw2.throw_deep(message) diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py new file mode 100644 index 0000000000..766b6bbc65 --- /dev/null +++ b/python/mach/mach/test/providers/throw2.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file exists to trigger the differences in mach error reporting between +# exceptions that occur in mach command modules themselves and in the things +# they call. + +from __future__ import absolute_import + + +def throw_deep(message): + return throw_real(message) + + +def throw_real(message): + raise Exception(message) diff --git a/python/mach/mach/test/python.ini b/python/mach/mach/test/python.ini new file mode 100644 index 0000000000..eb26eccfe2 --- /dev/null +++ b/python/mach/mach/test/python.ini @@ -0,0 +1,15 @@ +[DEFAULT] +subsuite = mach + +[test_commands.py] +[test_conditions.py] +skip-if = python == 3 +[test_config.py] +[test_dispatcher.py] +[test_entry_point.py] +[test_error_output.py] +skip-if = python == 3 +[test_logger.py] +[test_mach.py] +[test_telemetry.py] +skip-if = python == 2 && os == "mac" diff --git a/python/mach/mach/test/test_commands.py b/python/mach/mach/test/test_commands.py new file mode 100644 index 0000000000..ffa3c78a67 --- /dev/null +++ b/python/mach/mach/test/test_commands.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import sys + +import pytest +from mozunit import main + +from buildconfig import topsrcdir +import mach + +ALL_COMMANDS = [ + "cmd_bar", + "cmd_foo", + "cmd_foobar", + "mach-commands", + "mach-completion", + "mach-debug-commands", +] + + +@pytest.fixture +def run_completion(run_mach): + def inner(args=[]): + mach_dir = os.path.dirname(mach.__file__) + providers = [ + "commands.py", + os.path.join(mach_dir, "commands", "commandinfo.py"), + ] + + def context_handler(key): + if key == "topdir": + return topsrcdir + + args = ["mach-completion"] + args + return run_mach(args, providers, context_handler=context_handler) + + return inner + + +def format(targets): + return "\n".join(targets) + "\n" + + +def test_mach_completion(run_completion): + result, stdout, stderr = run_completion() + assert result == 0 + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_f"]) + assert result == 0 + # While it seems like this should return only commands that have + # 'cmd_f' as a prefix, the completion script will handle this case + # properly. + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_foo"]) + assert result == 0 + assert stdout == format(["help", "--arg"]) + + +@pytest.mark.parametrize("shell", ("bash", "fish", "zsh")) +def test_generate_mach_completion_script(run_completion, shell): + rv, out, err = run_completion([shell]) + print(out) + print(err, file=sys.stderr) + assert rv == 0 + assert err == "" + + assert "cmd_foo" in out + assert "arg" in out + assert "cmd_foobar" in out + assert "cmd_bar" in out + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py new file mode 100644 index 0000000000..78c2f8d25d --- /dev/null +++ b/python/mach/mach/test/test_conditions.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + +from buildconfig import topsrcdir +from mach.base import MachError +from mach.main import Mach +from mach.registrar import Registrar +from mach.test.conftest import TestBase, PROVIDER_DIR + +from mozunit import main + + +def _make_populate_context(include_extra_attributes): + def _populate_context(key=None): + if key is None: + return + + attributes = { + "topdir": topsrcdir, + } + if include_extra_attributes: + attributes["foo"] = True + attributes["bar"] = False + + try: + return attributes[key] + except KeyError: + raise AttributeError(key) + + return _populate_context + + +_populate_bare_context = _make_populate_context(False) +_populate_context = _make_populate_context(True) + + +class TestConditions(TestBase): + """Tests for conditionally filtering commands.""" + + def _run(self, args, context_handler=_populate_bare_context): + return self._run_mach(args, "conditions.py", context_handler=context_handler) + + def test_conditions_pass(self): + """Test that a command which passes its conditions is runnable.""" + + self.assertEquals((0, "", ""), self._run(["cmd_foo"])) + self.assertEquals((0, "", ""), self._run(["cmd_foo_ctx"], _populate_context)) + + def test_invalid_context_message(self): + """Test that commands which do not pass all their conditions + print the proper failure message.""" + + def is_bar(): + """Bar must be true""" + + fail_conditions = [is_bar] + + for name in ("cmd_bar", "cmd_foobar"): + result, stdout, stderr = self._run([name]) + self.assertEquals(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + for name in ("cmd_bar_ctx", "cmd_foobar_ctx"): + result, stdout, stderr = self._run([name], _populate_context) + self.assertEquals(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + def test_invalid_type(self): + """Test that a condition which is not callable raises an exception.""" + + m = Mach(os.getcwd()) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + self.assertRaises( + MachError, + m.load_commands_from_file, + os.path.join(PROVIDER_DIR, "conditions_invalid.py"), + ) + + def test_help_message(self): + """Test that commands that are not runnable do not show up in help.""" + + result, stdout, stderr = self._run(["help"], _populate_context) + self.assertIn("cmd_foo", stdout) + self.assertNotIn("cmd_bar", stdout) + self.assertNotIn("cmd_foobar", stdout) + self.assertIn("cmd_foo_ctx", stdout) + self.assertNotIn("cmd_bar_ctx", stdout) + self.assertNotIn("cmd_foobar_ctx", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py new file mode 100644 index 0000000000..1d8940c725 --- /dev/null +++ b/python/mach/mach/test/test_config.py @@ -0,0 +1,295 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +import unittest + +from mozfile.mozfile import NamedTemporaryFile + +from mach.config import ( + BooleanType, + ConfigException, + ConfigSettings, + IntegerType, + PathType, + PositiveIntegerType, + StringType, +) +from mach.decorators import SettingsProvider +from mozunit import main +from six import string_types + + +CONFIG1 = r""" +[foo] + +bar = bar_value +baz = /baz/foo.c +""" + +CONFIG2 = r""" +[foo] + +bar = value2 +""" + + +@SettingsProvider +class Provider1(object): + config_settings = [ + ("foo.bar", StringType, "desc"), + ("foo.baz", PathType, "desc"), + ] + + +@SettingsProvider +class ProviderDuplicate(object): + config_settings = [ + ("dupesect.foo", StringType, "desc"), + ("dupesect.foo", StringType, "desc"), + ] + + +@SettingsProvider +class Provider2(object): + config_settings = [ + ("a.string", StringType, "desc"), + ("a.boolean", BooleanType, "desc"), + ("a.pos_int", PositiveIntegerType, "desc"), + ("a.int", IntegerType, "desc"), + ("a.path", PathType, "desc"), + ] + + +@SettingsProvider +class Provider3(object): + @classmethod + def config_settings(cls): + return [ + ("a.string", "string", "desc"), + ("a.boolean", "boolean", "desc"), + ("a.pos_int", "pos_int", "desc"), + ("a.int", "int", "desc"), + ("a.path", "path", "desc"), + ] + + +@SettingsProvider +class Provider4(object): + config_settings = [ + ("foo.abc", StringType, "desc", "a", {"choices": set("abc")}), + ("foo.xyz", StringType, "desc", "w", {"choices": set("xyz")}), + ] + + +@SettingsProvider +class Provider5(object): + config_settings = [ + ("foo.*", "string", "desc"), + ("foo.bar", "string", "desc"), + ] + + +class TestConfigSettings(unittest.TestCase): + def test_empty(self): + s = ConfigSettings() + + self.assertEqual(len(s), 0) + self.assertNotIn("foo", s) + + def test_duplicate_option(self): + s = ConfigSettings() + + with self.assertRaises(ConfigException): + s.register_provider(ProviderDuplicate) + + def test_simple(self): + s = ConfigSettings() + s.register_provider(Provider1) + + self.assertEqual(len(s), 1) + self.assertIn("foo", s) + + foo = s["foo"] + foo = s.foo + + self.assertEqual(len(foo), 0) + self.assertEqual(len(foo._settings), 2) + + self.assertIn("bar", foo._settings) + self.assertIn("baz", foo._settings) + + self.assertNotIn("bar", foo) + foo["bar"] = "value1" + self.assertIn("bar", foo) + + self.assertEqual(foo["bar"], "value1") + self.assertEqual(foo.bar, "value1") + + def test_assignment_validation(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + # Assigning an undeclared setting raises. + exc_type = AttributeError if sys.version_info < (3, 0) else KeyError + with self.assertRaises(exc_type): + a.undefined = True + + with self.assertRaises(KeyError): + a["undefined"] = True + + # Basic type validation. + a.string = "foo" + a.string = "foo" + + with self.assertRaises(TypeError): + a.string = False + + a.boolean = True + a.boolean = False + + with self.assertRaises(TypeError): + a.boolean = "foo" + + a.pos_int = 5 + a.pos_int = 0 + + with self.assertRaises(ValueError): + a.pos_int = -1 + + with self.assertRaises(TypeError): + a.pos_int = "foo" + + a.int = 5 + a.int = 0 + a.int = -5 + + with self.assertRaises(TypeError): + a.int = 1.24 + + with self.assertRaises(TypeError): + a.int = "foo" + + a.path = "/home/gps" + a.path = "foo.c" + a.path = "foo/bar" + a.path = "./foo" + + def retrieval_type_helper(self, provider): + s = ConfigSettings() + s.register_provider(provider) + + a = s.a + + a.string = "foo" + a.boolean = True + a.pos_int = 12 + a.int = -4 + a.path = "./foo/bar" + + self.assertIsInstance(a.string, string_types) + self.assertIsInstance(a.boolean, bool) + self.assertIsInstance(a.pos_int, int) + self.assertIsInstance(a.int, int) + self.assertIsInstance(a.path, string_types) + + def test_retrieval_type(self): + self.retrieval_type_helper(Provider2) + self.retrieval_type_helper(Provider3) + + def test_choices_validation(self): + s = ConfigSettings() + s.register_provider(Provider4) + + foo = s.foo + foo.abc + with self.assertRaises(ValueError): + foo.xyz + + with self.assertRaises(ValueError): + foo.abc = "e" + + foo.abc = "b" + foo.xyz = "y" + + def test_wildcard_options(self): + s = ConfigSettings() + s.register_provider(Provider5) + + foo = s.foo + + self.assertIn("*", foo._settings) + self.assertNotIn("*", foo) + + foo.baz = "value1" + foo.bar = "value2" + + self.assertIn("baz", foo) + self.assertEqual(foo.baz, "value1") + + self.assertIn("bar", foo) + self.assertEqual(foo.bar, "value2") + + def test_file_reading_single(self): + temp = NamedTemporaryFile(mode="wt") + temp.write(CONFIG1) + temp.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_file(temp.name) + + self.assertEqual(s.foo.bar, "bar_value") + + def test_file_reading_multiple(self): + """Loading multiple files has proper overwrite behavior.""" + temp1 = NamedTemporaryFile(mode="wt") + temp1.write(CONFIG1) + temp1.flush() + + temp2 = NamedTemporaryFile(mode="wt") + temp2.write(CONFIG2) + temp2.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_files([temp1.name, temp2.name]) + + self.assertEqual(s.foo.bar, "value2") + + def test_file_reading_missing(self): + """Missing files should silently be ignored.""" + + s = ConfigSettings() + + s.load_file("/tmp/foo.ini") + + def test_file_writing(self): + s = ConfigSettings() + s.register_provider(Provider2) + + s.a.string = "foo" + s.a.boolean = False + + temp = NamedTemporaryFile("wt") + s.write(temp) + temp.flush() + + s2 = ConfigSettings() + s2.register_provider(Provider2) + + s2.load_file(temp.name) + + self.assertEqual(s.a.string, s2.a.string) + self.assertEqual(s.a.boolean, s2.a.boolean) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_dispatcher.py b/python/mach/mach/test/test_dispatcher.py new file mode 100644 index 0000000000..faf165863d --- /dev/null +++ b/python/mach/mach/test/test_dispatcher.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import unittest +from io import StringIO + +import pytest +from mozunit import main +from six import string_types + +from mach.base import CommandContext +from mach.registrar import Registrar + +here = os.path.abspath(os.path.dirname(__file__)) + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestDispatcher(unittest.TestCase): + """Tests dispatch related code""" + + def get_parser(self, config=None): + mach = self.get_mach("basic.py") + + for provider in Registrar.settings_providers: + mach.settings.register_provider(provider) + + if config: + if isinstance(config, string_types): + config = StringIO(config) + mach.settings.load_fps([config]) + + context = CommandContext(settings=mach.settings) + return mach.get_argument_parser(context) + + def test_command_aliases(self): + config = """ +[alias] +foo = cmd_foo +bar = cmd_bar +baz = cmd_bar --baz +cmd_bar = cmd_bar --baz +""" + parser = self.get_parser(config=config) + + args = parser.parse_args(["foo"]) + self.assertEquals(args.command, "cmd_foo") + + def assert_bar_baz(argv): + args = parser.parse_args(argv) + self.assertEquals(args.command, "cmd_bar") + self.assertTrue(args.command_args.baz) + + # The following should all result in |cmd_bar --baz| + assert_bar_baz(["bar", "--baz"]) + assert_bar_baz(["baz"]) + assert_bar_baz(["cmd_bar"]) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py new file mode 100644 index 0000000000..0a6b313417 --- /dev/null +++ b/python/mach/mach/test/test_entry_point.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +from __future__ import unicode_literals + +import imp +import os +import sys + +from mach.base import MachError +from mach.test.conftest import TestBase +from mock import patch + +from mozunit import main + + +here = os.path.abspath(os.path.dirname(__file__)) + + +class Entry: + """Stub replacement for pkg_resources.EntryPoint""" + + def __init__(self, providers): + self.providers = providers + + def load(self): + def _providers(): + return self.providers + + return _providers + + +class TestEntryPoints(TestBase): + """Test integrating with setuptools entry points""" + + provider_dir = os.path.join(here, "providers") + + def _run_help(self): + return self._run_mach(["help"], entry_point="mach.providers") + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_directory(self, mock): + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if "mach.commands" not in sys.modules: + mod = imp.new_module("mach.commands") + sys.modules["mach.commands"] = mod + + mock.return_value = [Entry([self.provider_dir])] + # Mach error raised due to conditions_invalid.py + with self.assertRaises(MachError): + self._run_help() + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_file(self, mock): + mock.return_value = [Entry([os.path.join(self.provider_dir, "basic.py")])] + + result, stdout, stderr = self._run_help() + self.assertIsNone(result) + self.assertIn("cmd_foo", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py new file mode 100644 index 0000000000..73abd61e41 --- /dev/null +++ b/python/mach/mach/test/test_error_output.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from mach.main import COMMAND_ERROR_TEMPLATE, MODULE_ERROR_TEMPLATE +from mozunit import main + + +def test_command_error(run_mach): + result, stdout, stderr = run_mach( + ["throw", "--message", "Command Error"], provider_files="throw.py" + ) + assert result == 1 + assert COMMAND_ERROR_TEMPLATE % "throw" in stdout + + +def test_invoked_error(run_mach): + result, stdout, stderr = run_mach( + ["throw_deep", "--message", "Deep stack"], provider_files="throw.py" + ) + assert result == 1 + assert MODULE_ERROR_TEMPLATE % "throw_deep" in stdout + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py new file mode 100644 index 0000000000..1561546fa8 --- /dev/null +++ b/python/mach/mach/test/test_logger.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging +import time +import unittest + +from mach.logging import StructuredHumanFormatter + +from mozunit import main + + +class DummyLogger(logging.Logger): + def __init__(self, cb): + logging.Logger.__init__(self, "test") + + self._cb = cb + + def handle(self, record): + self._cb(record) + + +class TestStructuredHumanFormatter(unittest.TestCase): + def test_non_ascii_logging(self): + # Ensures the formatter doesn't choke when non-ASCII characters are + # present in printed parameters. + formatter = StructuredHumanFormatter(time.time()) + + def on_record(record): + result = formatter.format(record) + relevant = result[9:] + + self.assertEqual(relevant, "Test: s\xe9curit\xe9") + + logger = DummyLogger(on_record) + + value = "s\xe9curit\xe9" + + logger.log( + logging.INFO, + "Test: {utf}", + extra={"action": "action", "params": {"utf": value}}, + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_mach.py b/python/mach/mach/test/test_mach.py new file mode 100644 index 0000000000..852f90838c --- /dev/null +++ b/python/mach/mach/test/test_mach.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import os + +from mozunit import main + + +def test_set_isatty_environ(monkeypatch, get_mach): + # Make sure the 'MACH_STDOUT_ISATTY' variable gets set. + monkeypatch.delenv("MACH_STDOUT_ISATTY", raising=False) + monkeypatch.setattr(os, "isatty", lambda fd: True) + + m = get_mach() + orig_run = m._run + env_is_set = [] + + def wrap_run(*args, **kwargs): + env_is_set.append("MACH_STDOUT_ISATTY" in os.environ) + return orig_run(*args, **kwargs) + + monkeypatch.setattr(m, "_run", wrap_run) + + ret = m.run([]) + assert ret == 0 + assert env_is_set[0] + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_telemetry.py b/python/mach/mach/test/test_telemetry.py new file mode 100644 index 0000000000..ded23420b8 --- /dev/null +++ b/python/mach/mach/test/test_telemetry.py @@ -0,0 +1,198 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from __future__ import absolute_import, print_function + +import json +import os +import platform +import subprocess +import sys + +import buildconfig +import mozunit +import pytest +from six import text_type, PY3 + +from mozboot.bootstrap import update_or_create_build_telemetry_config + +TELEMETRY_LOAD_ERROR = """ +Error loading telemetry. mach output: +========================================================= +%s +========================================================= +""" + + +@pytest.fixture +def run_mach(tmpdir): + """Return a function that runs mach with the provided arguments and then returns + a list of the data contained within any telemetry entries generated during the command. + """ + # Use tmpdir as the mozbuild state path, and enable telemetry in + # a machrc there. + if PY3: + update_or_create_build_telemetry_config(str(tmpdir.join("machrc"))) + else: + update_or_create_build_telemetry_config(text_type(tmpdir.join("machrc"))) + env = dict(os.environ) + env["MOZBUILD_STATE_PATH"] = str(tmpdir) + env["TEST_MACH_TELEMETRY_NO_SUBMIT"] = "1" + mach = os.path.join(buildconfig.topsrcdir, "mach") + + def run(*args, **kwargs): + # Let whatever mach command we invoke from tests believe it's the main command. + mach_main_pid = env.pop("MACH_MAIN_PID") + moz_automation = env.pop("MOZ_AUTOMATION", None) + task_id = env.pop("TASK_ID", None) + + # Run mach with the provided arguments + out = subprocess.check_output( + [sys.executable, mach] + list(args), + stderr=subprocess.STDOUT, + env=env, + **kwargs + ) + + env["MACH_MAIN_PID"] = mach_main_pid + env["MOZ_AUTOMATION"] = moz_automation + env["TASK_ID"] = task_id + # Load any telemetry data that was written + path = tmpdir.join("telemetry", "outgoing") + try: + if PY3: + read_mode = "r" + else: + read_mode = "rb" + return [json.load(f.open(read_mode)) for f in path.listdir()] + except EnvironmentError: + print(TELEMETRY_LOAD_ERROR % out, file=sys.stderr) + for p in path.parts(reverse=True): + if not p.check(dir=1): + print('Path does not exist: "%s"' % p, file=sys.stderr) + raise + + return run + + +def test_simple(run_mach, tmpdir): + data = run_mach("python", "-c", "pass") + assert len(data) == 1 + d = data[0] + assert d["command"] == "python" + assert d["argv"] == ["-c", "pass"] + if PY3: + read_mode = "r" + else: + read_mode = "rb" + client_id_data = json.load(tmpdir.join("telemetry_client_id.json").open(read_mode)) + assert "client_id" in client_id_data + assert client_id_data["client_id"] == d["client_id"] + + +@pytest.mark.xfail( + platform.system() == "Windows" and PY3, + reason="Windows and Python3 mozpath filtering issues", +) +def test_path_filtering(run_mach, tmpdir): + srcdir_path = os.path.join(buildconfig.topsrcdir, "a") + srcdir_path_2 = os.path.join(buildconfig.topsrcdir, "a/b/c") + objdir_path = os.path.join(buildconfig.topobjdir, "x") + objdir_path_2 = os.path.join(buildconfig.topobjdir, "x/y/z") + home_path = os.path.join(os.path.expanduser("~"), "something_in_home") + other_path = str(tmpdir.join("other")) + data = run_mach( + "python", + "-c", + "pass", + srcdir_path, + srcdir_path_2, + objdir_path, + objdir_path_2, + home_path, + other_path, + cwd=buildconfig.topsrcdir, + ) + assert len(data) == 1 + d = data[0] + expected = [ + "-c", + "pass", + "a", + "a/b/c", + "$topobjdir/x", + "$topobjdir/x/y/z", + "$HOME/something_in_home", + "<path omitted>", + ] + assert d["argv"] == expected + + +@pytest.mark.xfail( + platform.system() == "Windows" and PY3, + reason="Windows and Python3 mozpath filtering issues", +) +def test_path_filtering_in_objdir(run_mach, tmpdir): + srcdir_path = os.path.join(buildconfig.topsrcdir, "a") + srcdir_path_2 = os.path.join(buildconfig.topsrcdir, "a/b/c") + objdir_path = os.path.join(buildconfig.topobjdir, "x") + objdir_path_2 = os.path.join(buildconfig.topobjdir, "x/y/z") + other_path = str(tmpdir.join("other")) + data = run_mach( + "python", + "-c", + "pass", + srcdir_path, + srcdir_path_2, + objdir_path, + objdir_path_2, + other_path, + cwd=buildconfig.topobjdir, + ) + assert len(data) == 1 + d = data[0] + expected = [ + "-c", + "pass", + "$topsrcdir/a", + "$topsrcdir/a/b/c", + "x", + "x/y/z", + "<path omitted>", + ] + assert d["argv"] == expected + + +def test_path_filtering_other_cwd(run_mach, tmpdir): + srcdir_path = os.path.join(buildconfig.topsrcdir, "a") + srcdir_path_2 = os.path.join(buildconfig.topsrcdir, "a/b/c") + other_path = str(tmpdir.join("other")) + data = run_mach( + "python", "-c", "pass", srcdir_path, srcdir_path_2, other_path, cwd=str(tmpdir) + ) + assert len(data) == 1 + d = data[0] + expected = [ + # non-path arguments should escape unscathed + "-c", + "pass", + "$topsrcdir/a", + "$topsrcdir/a/b/c", + # cwd-relative paths should be relativized + "other", + ] + assert d["argv"] == expected + + +def test_zero_microseconds(run_mach): + data = run_mach( + "python", + "--exec-file", + os.path.join(os.path.dirname(__file__), "zero_microseconds.py"), + ) + d = data[0] + assert d["command"] == "python" + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mach/mach/test/zero_microseconds.py b/python/mach/mach/test/zero_microseconds.py new file mode 100644 index 0000000000..1854747bb6 --- /dev/null +++ b/python/mach/mach/test/zero_microseconds.py @@ -0,0 +1,14 @@ +# This code is loaded via `mach python --exec-file`, so it runs in the scope of +# the `mach python` command. +from __future__ import absolute_import + +old = self._mach_context.post_dispatch_handler # noqa: F821 + + +def handler(context, handler, instance, result, start_time, end_time, depth, args): + global old + # Round off sub-second precision. + old(context, handler, instance, result, int(start_time), end_time, depth, args) + + +self._mach_context.post_dispatch_handler = handler # noqa: F821 diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py new file mode 100644 index 0000000000..734f8d81e9 --- /dev/null +++ b/python/mach/mach/util.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import os +import sys + +from six import text_type + + +class UserError(Exception): + """Represents an error caused by something the user did wrong rather than + an internal `mach` failure. Exceptions that are subclasses of this class + will not be reported as failures to Sentry. + """ + + +def setenv(key, value): + """Compatibility shim to ensure the proper string type is used with + os.environ for the version of Python being used. + """ + encoding = "mbcs" if sys.platform == "win32" else "utf-8" + + if sys.version_info[0] == 2: + if isinstance(key, text_type): + key = key.encode(encoding) + if isinstance(value, text_type): + value = value.encode(encoding) + else: + if isinstance(key, bytes): + key = key.decode(encoding) + if isinstance(value, bytes): + value = value.decode(encoding) + + os.environ[key] = value diff --git a/python/mach/metrics.yaml b/python/mach/metrics.yaml new file mode 100644 index 0000000000..0d60a1c920 --- /dev/null +++ b/python/mach/metrics.yaml @@ -0,0 +1,161 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# If this file is changed, update the generated docs: +# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs +--- +$schema: moz://mozilla.org/schemas/glean/metrics/1-0-0 + +mach: + command: + type: string + description: > + The name of the mach command that was invoked, such as "build", + "doc", or "try". + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + argv: + type: string_list + description: > + Parameters provided to mach. Absolute paths are sanitized to be relative + to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", + or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" + would be replaced with "$topsrcdir/python/mozbuild". + If a valid replacement base path cannot be found, the path is replaced + with "<path omitted>". + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + success: + type: boolean + description: True if the mach invocation succeeded. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + duration: + type: timespan + description: How long it took for the command to complete. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + +mach.system: + cpu_brand: + type: string + description: CPU brand string from CPUID. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + distro: + type: string + description: The name of the operating system distribution. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + distro_version: + type: string + description: The high-level OS version. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + logical_cores: + type: counter + description: Number of logical CPU cores present. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + physical_cores: + type: counter + description: Number of physical CPU cores present. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + memory: + type: memory_distribution + memory_unit: gigabyte + description: Amount of system memory. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage diff --git a/python/mach/pings.yaml b/python/mach/pings.yaml new file mode 100644 index 0000000000..9c12697f38 --- /dev/null +++ b/python/mach/pings.yaml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# If this file is changed, update the generated docs: +# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs +--- +$schema: moz://mozilla.org/schemas/glean/pings/1-0-0 + +usage: + description: > + Sent when the mach invocation is completed (regardless of result). + Contains information about the mach invocation that was made, its result, + and some details about the current environment and hardware. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + include_client_id: true + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com diff --git a/python/mach/setup.cfg b/python/mach/setup.cfg new file mode 100644 index 0000000000..3c6e79cf31 --- /dev/null +++ b/python/mach/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/python/mach/setup.py b/python/mach/setup.py new file mode 100644 index 0000000000..ac65398c07 --- /dev/null +++ b/python/mach/setup.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + + +VERSION = "1.0.0" + +README = open("README.rst").read() + +setup( + name="mach", + description="Generic command line command dispatching framework.", + long_description=README, + license="MPL 2.0", + author="Gregory Szorc", + author_email="gregory.szorc@gmail.com", + url="https://developer.mozilla.org/en-US/docs/Developer_Guide/mach", + packages=["mach", "mach.mixin"], + version=VERSION, + classifiers=[ + "Environment :: Console", + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Natural Language :: English", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + ], + install_requires=[ + "blessings", + "mozfile", + "mozprocess", + "six", + ], + tests_require=["mock"], +) |