summaryrefslogtreecommitdiffstats
path: root/python/mach
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /python/mach
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python/mach')
-rw-r--r--python/mach/.ruff.toml4
-rw-r--r--python/mach/README.rst13
-rw-r--r--python/mach/bash-completion.sh18
-rw-r--r--python/mach/docs/commands.rst129
-rw-r--r--python/mach/docs/driver.rst32
-rw-r--r--python/mach/docs/faq.rst152
-rw-r--r--python/mach/docs/index.rst89
-rw-r--r--python/mach/docs/logging.rst100
-rw-r--r--python/mach/docs/metrics.md55
-rw-r--r--python/mach/docs/settings.rst138
-rw-r--r--python/mach/docs/telemetry.rst37
-rw-r--r--python/mach/docs/usage.rst150
-rw-r--r--python/mach/docs/windows-usage-outside-mozillabuild.rst124
-rw-r--r--python/mach/mach/__init__.py0
-rw-r--r--python/mach/mach/base.py73
-rw-r--r--python/mach/mach/commands/__init__.py0
-rw-r--r--python/mach/mach/commands/commandinfo.py487
-rw-r--r--python/mach/mach/commands/completion_templates/bash.template62
-rw-r--r--python/mach/mach/commands/completion_templates/fish.template64
-rw-r--r--python/mach/mach/commands/completion_templates/zsh.template62
-rw-r--r--python/mach/mach/commands/settings.py51
-rw-r--r--python/mach/mach/config.py415
-rw-r--r--python/mach/mach/decorators.py340
-rw-r--r--python/mach/mach/dispatcher.py516
-rw-r--r--python/mach/mach/logging.py398
-rw-r--r--python/mach/mach/main.py735
-rw-r--r--python/mach/mach/mixin/__init__.py0
-rw-r--r--python/mach/mach/mixin/logging.py52
-rw-r--r--python/mach/mach/mixin/process.py217
-rw-r--r--python/mach/mach/python_lockfile.py79
-rw-r--r--python/mach/mach/registrar.py186
-rw-r--r--python/mach/mach/requirements.py183
-rw-r--r--python/mach/mach/sentry.py222
-rw-r--r--python/mach/mach/site.py1405
-rw-r--r--python/mach/mach/telemetry.py305
-rw-r--r--python/mach/mach/telemetry_interface.py77
-rw-r--r--python/mach/mach/terminal.py76
-rw-r--r--python/mach/mach/test/__init__.py0
-rw-r--r--python/mach/mach/test/conftest.py84
-rw-r--r--python/mach/mach/test/invoke_mach_command.py4
-rw-r--r--python/mach/mach/test/providers/__init__.py0
-rw-r--r--python/mach/mach/test/providers/basic.py15
-rw-r--r--python/mach/mach/test/providers/commands.py33
-rw-r--r--python/mach/mach/test/providers/conditions.py55
-rw-r--r--python/mach/mach/test/providers/conditions_invalid.py10
-rw-r--r--python/mach/mach/test/providers/throw.py18
-rw-r--r--python/mach/mach/test/providers/throw2.py15
-rw-r--r--python/mach/mach/test/python.ini22
-rw-r--r--python/mach/mach/test/script_site_activation.py67
-rw-r--r--python/mach/mach/test/test_commands.py79
-rw-r--r--python/mach/mach/test/test_conditions.py101
-rw-r--r--python/mach/mach/test/test_config.py292
-rw-r--r--python/mach/mach/test/test_decorators.py133
-rw-r--r--python/mach/mach/test/test_dispatcher.py60
-rw-r--r--python/mach/mach/test/test_entry_point.py59
-rw-r--r--python/mach/mach/test/test_error_output.py29
-rw-r--r--python/mach/mach/test/test_logger.py48
-rw-r--r--python/mach/mach/test/test_mach.py31
-rw-r--r--python/mach/mach/test/test_site.py56
-rw-r--r--python/mach/mach/test/test_site_activation.py463
-rw-r--r--python/mach/mach/test/test_site_compatibility.py189
-rw-r--r--python/mach/mach/test/zero_microseconds.py12
-rw-r--r--python/mach/mach/util.py110
-rw-r--r--python/mach/metrics.yaml206
-rw-r--r--python/mach/pings.yaml22
-rw-r--r--python/mach/setup.cfg2
-rw-r--r--python/mach/setup.py42
67 files changed, 9303 insertions, 0 deletions
diff --git a/python/mach/.ruff.toml b/python/mach/.ruff.toml
new file mode 100644
index 0000000000..82b1a04648
--- /dev/null
+++ b/python/mach/.ruff.toml
@@ -0,0 +1,4 @@
+extend = "../../pyproject.toml"
+
+[isort]
+known-first-party = ["mach"]
diff --git a/python/mach/README.rst b/python/mach/README.rst
new file mode 100644
index 0000000000..7c2e00becb
--- /dev/null
+++ b/python/mach/README.rst
@@ -0,0 +1,13 @@
+====
+mach
+====
+
+Mach (German for *do*) is a generic command dispatcher for the command
+line.
+
+To use mach, you install the mach core (a Python package), create an
+executable *driver* script (named whatever you want), and write mach
+commands. When the *driver* is executed, mach dispatches to the
+requested command handler automatically.
+
+To learn more, read the docs in ``docs/``.
diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh
new file mode 100644
index 0000000000..13935cf88c
--- /dev/null
+++ b/python/mach/bash-completion.sh
@@ -0,0 +1,18 @@
+function _mach()
+{
+ local cur targets
+ COMPREPLY=()
+
+ # Calling `mach-completion` with -h/--help would result in the
+ # help text being used as the completion targets.
+ if [[ $COMP_LINE == *"-h"* || $COMP_LINE == *"--help"* ]]; then
+ return 0
+ fi
+
+ # Load the list of targets
+ targets=`"${COMP_WORDS[0]}" mach-completion ${COMP_LINE}`
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ COMPREPLY=( $(compgen -W "$targets" -- ${cur}) )
+ return 0
+}
+complete -o default -F _mach mach
diff --git a/python/mach/docs/commands.rst b/python/mach/docs/commands.rst
new file mode 100644
index 0000000000..7547193000
--- /dev/null
+++ b/python/mach/docs/commands.rst
@@ -0,0 +1,129 @@
+.. _mach_commands:
+
+=====================
+Implementing Commands
+=====================
+
+Mach commands are defined via Python decorators.
+
+All the relevant decorators are defined in the *mach.decorators* module.
+The important decorators are as follows:
+
+:py:func:`Command <mach.decorators.Command>`
+ A function decorator that denotes that the function should be called when
+ the specified command is requested. The decorator takes a command name
+ as its first argument and a number of additional arguments to
+ configure the behavior of the command. The decorated function must take a
+ ``command_context`` argument as its first.
+ ``command_context`` is a properly configured instance of a ``MozbuildObject``
+ subclass, meaning it can be used for accessing things like the current config
+ and running processes.
+
+:py:func:`CommandArgument <mach.decorators.CommandArgument>`
+ A function decorator that defines an argument to the command. Its
+ arguments are essentially proxied to ArgumentParser.add_argument()
+
+:py:func:`SubCommand <mach.decorators.SubCommand>`
+ A function decorator that denotes that the function should be a
+ sub-command to an existing ``@Command``. The decorator takes the
+ parent command name as its first argument and the sub-command name
+ as its second argument.
+
+ ``@CommandArgument`` can be used on ``@SubCommand`` instances just
+ like they can on ``@Command`` instances.
+
+
+Here is a complete example:
+
+.. code-block:: python
+
+ from mach.decorators import (
+ CommandArgument,
+ Command,
+ )
+
+ @Command('doit', help='Do ALL OF THE THINGS.')
+ @CommandArgument('--force', '-f', action='store_true',
+ help='Force doing it.')
+ def doit(command_context, force=False):
+ # Do stuff here.
+
+When the module is loaded, the decorators tell mach about all handlers.
+When mach runs, it takes the assembled metadata from these handlers and
+hooks it up to the command line driver. Under the hood, arguments passed
+to the decorators are being used to help mach parse command arguments,
+formulate arguments to the methods, etc. See the documentation in the
+:py:mod:`mach.base` module for more.
+
+The Python modules defining mach commands do not need to live inside the
+main mach source tree.
+
+Conditionally Filtering Commands
+================================
+
+Sometimes it might only make sense to run a command given a certain
+context. For example, running tests only makes sense if the product
+they are testing has been built, and said build is available. To make
+sure a command is only runnable from within a correct context, you can
+define a series of conditions on the
+:py:func:`Command <mach.decorators.Command>` decorator.
+
+A condition is simply a function that takes an instance of the
+:py:func:`mozbuild.base.MachCommandBase` class as an argument, and
+returns ``True`` or ``False``. If any of the conditions defined on a
+command return ``False``, the command will not be runnable. The
+docstring of a condition function is used in error messages, to explain
+why the command cannot currently be run.
+
+Here is an example:
+
+.. code-block:: python
+
+ from mach.decorators import (
+ Command,
+ )
+
+ def build_available(cls):
+ """The build needs to be available."""
+ return cls.build_path is not None
+
+ @Command('run_tests', conditions=[build_available])
+ def run_tests(command_context):
+ # Do stuff here.
+
+By default all commands without any conditions applied will be runnable,
+but it is possible to change this behaviour by setting
+``require_conditions`` to ``True``:
+
+.. code-block:: python
+
+ m = mach.main.Mach()
+ m.require_conditions = True
+
+Minimizing Code in Commands
+===========================
+
+Mach command modules, classes, and methods work best when they are
+minimal dispatchers. The reason is import bloat. Currently, the mach
+core needs to import every Python file potentially containing mach
+commands for every command invocation. If you have dozens of commands or
+commands in modules that import a lot of Python code, these imports
+could slow mach down and waste memory.
+
+It is thus recommended that mach modules, classes, and methods do as
+little work as possible. Ideally the module should only import from
+the :py:mod:`mach` package. If you need external modules, you should
+import them from within the command method.
+
+To keep code size small, the body of a command method should be limited
+to:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Of course, these recommendations can be ignored if you want to risk
+slower performance.
+
+In the future, the mach driver may cache the dispatching information or
+have it intelligently loaded to facilitate lazy loading.
diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst
new file mode 100644
index 0000000000..8a2a99a2f5
--- /dev/null
+++ b/python/mach/docs/driver.rst
@@ -0,0 +1,32 @@
+.. _mach_driver:
+
+=======
+Drivers
+=======
+
+Entry Points
+============
+
+It is possible to use setuptools' entry points to load commands
+directly from python packages. A mach entry point is a function which
+returns a list of files or directories containing mach command
+providers. e.g.:
+
+.. code-block:: python
+
+ def list_providers():
+ providers = []
+ here = os.path.abspath(os.path.dirname(__file__))
+ for p in os.listdir(here):
+ if p.endswith('.py'):
+ providers.append(os.path.join(here, p))
+ return providers
+
+See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
+for more information on creating an entry point. To search for entry
+point plugins, you can call
+:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.:
+
+.. code-block:: python
+
+ mach.load_commands_from_entry_point("mach.external.providers")
diff --git a/python/mach/docs/faq.rst b/python/mach/docs/faq.rst
new file mode 100644
index 0000000000..a640f83e87
--- /dev/null
+++ b/python/mach/docs/faq.rst
@@ -0,0 +1,152 @@
+.. _mach_faq:
+
+==========================
+Frequently Asked Questions
+==========================
+
+How do I report bugs?
+---------------------
+
+Bugs against the ``mach`` core can be filed in Bugzilla in the `Firefox
+Build System::Mach
+Core <https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Mach%20Core>`__ component.
+
+.. note::
+
+ Most ``mach`` bugs are bugs in individual commands, not bugs in the core
+ ``mach`` code. Bugs for individual commands should be filed against the
+ component that command is related to. For example, bugs in the
+ *build* command should be filed against *Firefox Build System ::
+ General*. Bugs against testing commands should be filed somewhere in
+ the *Testing* product.
+
+How do I debug a command failing with a Python exception?
+---------------------------------------------------------
+
+You can run a command and break into ``pdb``, the Python debugger,
+when the command is invoked with:
+
+.. code-block:: shell
+
+ ./mach --debug-command FAILING-COMMAND ARGS ...
+
+How do I debug ``mach`` itself?
+-------------------------------
+
+If you are editing the mach code, or other Python modules you can
+open the terminal and start debugging with pdb with the following:
+
+.. code-block:: shell
+
+ python3 -m pdb ./mach <command>
+
+How do I debug ``pytest`` tests?
+--------------------------------
+
+First, before debugging, run ``./mach python-test`` once to ensure that
+the testing virtualenv is up-to-date:
+
+.. code-block:: shell
+
+ ./mach python-test path/to/test.py
+
+Then, using the testing virtualenv, debug the test file:
+
+.. code-block:: shell
+
+ <objdir>/_virtualenvs/python-test/bin/python -m pdb path/to/test.py
+
+How do I profile a slow command?
+--------------------------------
+
+To diagnose bottlenecks, you can collect a performance profile:
+
+.. code-block:: shell
+
+ ./mach --profile-command SLOW-COMMAND ARGS ...
+
+Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using
+`snakeviz <https://jiffyclub.github.io/snakeviz/>`__:
+
+.. code-block:: shell
+
+ # If you don't have snakeviz installed yet:
+ python3 -m pip install snakeviz
+ python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile
+
+How do I profile ``mach`` itself?
+---------------------------------
+
+Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile``
+directly to profile ``mach`` itself:
+
+.. code-block:: shell
+
+ python3 -m cProfile -o mach.cProfile ./mach ...
+ python3 -m snakeviz mach.cProfile
+
+Is ``mach`` a build system?
+---------------------------
+
+No. ``mach`` is just a generic command dispatching tool that happens to have
+a few commands that interact with the real build system. Historically,
+``mach`` *was* born to become a better interface to the build system.
+However, its potential beyond just build system interaction was quickly
+realized and ``mach`` grew to fit those needs.
+
+How do I add features to ``mach``?
+----------------------------------
+If you would like to add a new feature to ``mach`` that cannot be implemented as
+a ``mach`` command, the first step is to file a bug in the
+``Firefox Build System :: Mach Core`` component.
+
+Should I implement X as a ``mach`` command?
+-------------------------------------------
+
+There are no hard or fast rules. Generally speaking, if you have some
+piece of functionality or action that is useful to multiple people
+(especially if it results in productivity wins), then you should
+consider implementing a ``mach`` command for it.
+
+Some other cases where you should consider implementing something as a
+``mach`` command:
+
+- When your tool is a random script in the tree. Random scripts are
+ hard to find and may not conform to coding conventions or best
+ practices. ``Mach`` provides a framework in which your tool can live that
+ will put it in a better position to succeed than if it were on its
+ own.
+- When the alternative is a ``make`` target. The build team generally does
+ not like one-off ``make`` targets that aren't part of building (read:
+ compiling) the tree. This includes things related to testing and
+ packaging. These weigh down ``Makefiles`` and add to the burden of
+ maintaining the build system. Instead, you are encouraged to
+ implement ancillary functionality in Python. If you do implement something
+ in Python, hooking it up to ``mach`` is often trivial.
+
+How do I use 3rd-party Python packages in my ``mach`` command?
+--------------------------------------------------------------
+
+See :ref:`Using third-party Python packages`.
+
+How does ``mach`` fit into the modules system?
+----------------------------------------------
+
+Mozilla operates with a `modules governance
+system <https://www.mozilla.org/about/governance/policies/module-ownership/>`__ where
+there are different components with different owners. There is not
+currently a ``mach`` module. There may or may never be one; currently ``mach``
+is owned by the build team.
+
+Even if a ``mach`` module were established, ``mach`` command modules would
+likely never belong to it. Instead, ``mach`` command modules are owned by the
+team/module that owns the system they interact with. In other words, ``mach``
+is not a power play to consolidate authority for tooling. Instead, it aims to
+expose that tooling through a common, shared interface.
+
+
+Who do I contact for help or to report issues?
+----------------------------------------------
+
+You can ask questions in
+`#build <https://chat.mozilla.org/#/room/#build:mozilla.org>`__.
diff --git a/python/mach/docs/index.rst b/python/mach/docs/index.rst
new file mode 100644
index 0000000000..752fe93219
--- /dev/null
+++ b/python/mach/docs/index.rst
@@ -0,0 +1,89 @@
+====
+Mach
+====
+
+Mach (German for *do*) is a generic command dispatcher for the command
+line.
+
+To use mach, you install the mach core (a Python package), create an
+executable *driver* script (named whatever you want), and write mach
+commands. When the *driver* is executed, mach dispatches to the
+requested command handler automatically.
+
+.. raw:: html
+
+ <h2>Features</h2>
+
+----
+
+On a high level, mach is similar to using argparse with subparsers (for
+command handling). When you dig deeper, mach offers a number of
+additional features:
+
+Distributed command definitions
+ With optparse/argparse, you have to define your commands on a central
+ parser instance. With mach, you annotate your command methods with
+ decorators and mach finds and dispatches to them automatically.
+
+Command categories
+ Mach commands can be grouped into categories when displayed in help.
+ This is currently not possible with argparse.
+
+Logging management
+ Mach provides a facility for logging (both classical text and
+ structured) that is available to any command handler.
+
+Settings files
+ Mach provides a facility for reading settings from an ini-like file
+ format.
+
+.. raw:: html
+
+ <h2>Components</h2>
+
+----
+
+Mach is conceptually composed of the following components:
+
+core
+ The mach core is the core code powering mach. This is a Python package
+ that contains all the business logic that makes mach work. The mach
+ core is common to all mach deployments.
+
+commands
+ These are what mach dispatches to. Commands are simply Python methods
+ registered as command names. The set of commands is unique to the
+ environment mach is deployed in.
+
+driver
+ The *driver* is the entry-point to mach. It is simply an executable
+ script that loads the mach core, tells it where commands can be found,
+ then asks the mach core to handle the current request. The driver is
+ unique to the deployed environment. But, it's usually based on an
+ example from this source tree.
+
+.. raw:: html
+
+ <h2> Project State</h2>
+
+----
+
+mach was originally written as a command dispatching framework to aid
+Firefox development. While the code is mostly generic, there are still
+some pieces that closely tie it to Mozilla/Firefox. The goal is for
+these to eventually be removed and replaced with generic features so
+mach is suitable for anybody to use. Until then, mach may not be the
+best fit for you.
+
+.. toctree::
+ :maxdepth: 1
+ :hidden:
+
+ usage
+ commands
+ driver
+ logging
+ settings
+ telemetry
+ windows-usage-outside-mozillabuild
+ faq
diff --git a/python/mach/docs/logging.rst b/python/mach/docs/logging.rst
new file mode 100644
index 0000000000..ff245cf032
--- /dev/null
+++ b/python/mach/docs/logging.rst
@@ -0,0 +1,100 @@
+.. _mach_logging:
+
+=======
+Logging
+=======
+
+Mach configures a built-in logging facility so commands can easily log
+data.
+
+What sets the logging facility apart from most loggers you've seen is
+that it encourages structured logging. Instead of conventional logging
+where simple strings are logged, the internal logging mechanism logs all
+events with the following pieces of information:
+
+* A string *action*
+* A dict of log message fields
+* A formatting string
+
+Essentially, instead of assembling a human-readable string at
+logging-time, you create an object holding all the pieces of data that
+will constitute your logged event. For each unique type of logged event,
+you assign an *action* name.
+
+Depending on how logging is configured, your logged event could get
+written a couple of different ways.
+
+JSON Logging
+============
+
+Where machines are the intended target of the logging data, a JSON
+logger is configured. The JSON logger assembles an array consisting of
+the following elements:
+
+* Decimal wall clock time in seconds since UNIX epoch
+* String *action* of message
+* Object with structured message data
+
+The JSON-serialized array is written to a configured file handle.
+Consumers of this logging stream can just perform a readline() then feed
+that into a JSON deserializer to reconstruct the original logged
+message. They can key off the *action* element to determine how to
+process individual events. There is no need to invent a parser.
+Convenient, isn't it?
+
+Logging for Humans
+==================
+
+Where humans are the intended consumer of a log message, the structured
+log message are converted to more human-friendly form. This is done by
+utilizing the *formatting* string provided at log time. The logger
+simply calls the *format* method of the formatting string, passing the
+dict containing the message's fields.
+
+When *mach* is used in a terminal that supports it, the logging facility
+also supports terminal features such as colorization. This is done
+automatically in the logging layer - there is no need to control this at
+logging time.
+
+In addition, messages intended for humans typically prepends every line
+with the time passed since the application started.
+
+Logging HOWTO
+=============
+
+Structured logging piggybacks on top of Python's built-in logging
+infrastructure provided by the *logging* package. We accomplish this by
+taking advantage of *logging.Logger.log()*'s *extra* argument. To this
+argument, we pass a dict with the fields *action* and *params*. These
+are the string *action* and dict of message fields, respectively. The
+formatting string is passed as the *msg* argument, like normal.
+
+If you were logging to a logger directly, you would do something like:
+
+.. code-block:: python
+
+ logger.log(logging.INFO, 'My name is {name}',
+ extra={'action': 'my_name', 'params': {'name': 'Gregory'}})
+
+The JSON logging would produce something like::
+
+ [1339985554.306338, "my_name", {"name": "Gregory"}]
+
+Human logging would produce something like::
+
+ 0.52 My name is Gregory
+
+Since there is a lot of complexity using logger.log directly, it is
+recommended to go through a wrapping layer that hides part of the
+complexity for you. The easiest way to do this is by utilizing the
+LoggingMixin:
+
+.. code-block:: python
+
+ import logging
+ from mach.mixin.logging import LoggingMixin
+
+ class MyClass(LoggingMixin):
+ def foo(self):
+ self.log(logging.INFO, 'foo_start', {'bar': True},
+ 'Foo performed. Bar: {bar}')
diff --git a/python/mach/docs/metrics.md b/python/mach/docs/metrics.md
new file mode 100644
index 0000000000..8c826f54a9
--- /dev/null
+++ b/python/mach/docs/metrics.md
@@ -0,0 +1,55 @@
+<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. -->
+
+# Metrics
+This document enumerates the metrics collected by this project using the [Glean SDK](https://mozilla.github.io/glean/book/index.html).
+This project may depend on other projects which also collect metrics.
+This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project.
+
+# Pings
+
+ - [usage](#usage)
+
+
+## usage
+
+Sent when the mach invocation is completed (regardless of result). Contains information about the mach invocation that was made, its result, and some details about the current environment and hardware.
+
+
+This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section).
+
+**Data reviews for this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34>
+
+**Bugs related to this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053>
+
+The following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| mach.argv |[string_list](https://mozilla.github.io/glean/book/user/metrics/string_list.html) |Parameters provided to mach. Absolute paths are sanitized to be relative to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" would be replaced with "$topsrcdir/python/mozbuild". If a valid replacement base path cannot be found, the path is replaced with "<path omitted>". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.command |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the mach command that was invoked, such as "build", "doc", or "try". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.duration |[timespan](https://mozilla.github.io/glean/book/user/metrics/timespan.html) |How long it took for the command to complete. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.success |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the mach invocation succeeded. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.cpu_brand |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |CPU brand string from CPUID. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.distro |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the operating system distribution. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3)||never | |
+| mach.system.distro_version |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The high-level OS version. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3)||never | |
+| mach.system.logical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of logical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.memory |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |Amount of system memory. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.physical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of physical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.artifact |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-artifact-builds`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.ccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--with-ccache`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.clobber |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the build was a clobber/full build. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15)||never | |
+| mozbuild.compiler |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The compiler type in use (CC_TYPE), such as "clang" or "gcc". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.debug |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-debug`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.icecream |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if icecream in use. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.opt |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-optimize`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.project |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The project being built. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1654084#c2)||never | |
+| mozbuild.sccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if ccache in use is sccache. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+
+
+Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection).
+
+<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. -->
diff --git a/python/mach/docs/settings.rst b/python/mach/docs/settings.rst
new file mode 100644
index 0000000000..4daba37472
--- /dev/null
+++ b/python/mach/docs/settings.rst
@@ -0,0 +1,138 @@
+.. _mach_settings:
+
+========
+Settings
+========
+
+Mach can read settings in from a set of configuration files. These
+configuration files are either named ``machrc`` or ``.machrc`` and
+are specified by the bootstrap script. In mozilla-central, these files
+can live in ``~/.mozbuild`` and/or ``topsrcdir``.
+
+Settings can be specified anywhere, and used both by mach core or
+individual commands.
+
+
+Core Settings
+=============
+
+These settings are implemented by mach core.
+
+* alias - Create a command alias. This is useful if you want to alias a command to something else, optionally including some defaults. It can either be used to create an entire new command, or provide defaults for an existing one. For example:
+
+.. parsed-literal::
+
+ [alias]
+ mochitest = mochitest -f browser
+ browser-test = mochitest -f browser
+
+
+Defining Settings
+=================
+
+Settings need to be explicitly defined, along with their type,
+otherwise mach will throw when trying to access them.
+
+To define settings, use the :func:`~decorators.SettingsProvider`
+decorator in an existing mach command module. E.g:
+
+.. code-block:: python
+
+ from mach.decorators import SettingsProvider
+ from mozbuild.base import MachCommandBase
+
+ @SettingsProvider
+ class ArbitraryClassName(MachCommandBase):
+ config_settings = [
+ ('foo.bar', 'string', "A helpful description"),
+ ('foo.baz', 'int', "Another description", 0, {'choices': set([0,1,2])}),
+ ]
+
+``@SettingsProvider``'s must specify a variable called ``config_settings``
+that returns a list of tuples. Alternatively, it can specify a function
+called ``config_settings`` that returns a list of tuples.
+
+Each tuple is of the form:
+
+.. code-block:: python
+
+ ('<section>.<option>', '<type>', '<description>', default, extra)
+
+``type`` is a string and can be one of:
+string, boolean, int, pos_int, path
+
+``description`` is a string explaining how to define the settings and
+where they get used. Descriptions should ideally be multi-line paragraphs
+where the first line acts as a short description.
+
+``default`` is optional, and provides a default value in case none was
+specified by any of the configuration files.
+
+``extra`` is also optional and is a dict containing additional key/value
+pairs to add to the setting's metadata. The following keys may be specified
+in the ``extra`` dict:
+
+ * ``choices`` - A set of allowed values for the setting.
+
+Wildcards
+---------
+
+Sometimes a section should allow arbitrarily defined options from the user, such
+as the ``alias`` section mentioned above. To define a section like this, use ``*``
+as the option name. For example:
+
+.. parsed-literal::
+
+ ('foo.*', 'string', 'desc')
+
+This allows configuration files like this:
+
+.. parsed-literal::
+
+ [foo]
+ arbitrary1 = some string
+ arbitrary2 = some other string
+
+
+Finding Settings
+================
+
+You can see which settings are available as well as their description and
+expected values by running:
+
+.. parsed-literal::
+
+ ./mach settings # or
+ ./mach settings --list
+
+
+Accessing Settings
+==================
+
+Now that the settings are defined and documented, they're accessible from
+individual mach commands from the mach command context.
+For example:
+
+.. code-block:: python
+
+ from mach.decorators import (
+ Command,
+ SettingsProvider,
+ )
+ from mozbuild.base import MachCommandBase
+
+ @SettingsProvider
+ class ExampleSettings(object):
+ config_settings = [
+ ('a.b', 'string', 'desc', 'default'),
+ ('foo.bar', 'string', 'desc',),
+ ('foo.baz', 'int', 'desc', 0, {'choices': set([0,1,2])}),
+ ]
+
+ @Command('command', category='misc',
+ description='Prints a setting')
+ def command(command_context):
+ settings = command_context._mach_context.settings
+ print(settings.a.b)
+ for option in settings.foo:
+ print(settings.foo[option])
diff --git a/python/mach/docs/telemetry.rst b/python/mach/docs/telemetry.rst
new file mode 100644
index 0000000000..2d185a970e
--- /dev/null
+++ b/python/mach/docs/telemetry.rst
@@ -0,0 +1,37 @@
+.. _mach_telemetry:
+
+==============
+Mach Telemetry
+==============
+
+`Glean <https://mozilla.github.io/glean/>`_ is used to collect telemetry, and uses the metrics
+defined in the ``metrics.yaml`` files in-tree.
+These files are all documented in a single :ref:`generated file here<metrics>`.
+
+.. toctree::
+ :maxdepth: 1
+
+ metrics
+
+Adding Metrics to a new Command
+===============================
+
+If you would like to submit telemetry metrics from your mach ``@Command``, you should take two steps:
+
+#. Parameterize your ``@Command`` annotation with ``metrics_path``.
+#. Use the ``command_context.metrics`` handle provided by ``MachCommandBase``
+
+For example::
+
+ METRICS_PATH = os.path.abspath(os.path.join(__file__, '..', '..', 'metrics.yaml'))
+
+ @Command('custom-command', metrics_path=METRICS_PATH)
+ def custom_command(command_context):
+ command_context.metrics.custom.foo.set('bar')
+
+Updating Generated Metrics Docs
+===============================
+
+When a ``metrics.yaml`` is added/changed/removed, :ref:`the metrics document<metrics>` will need to be updated::
+
+ ./mach doc mach-telemetry
diff --git a/python/mach/docs/usage.rst b/python/mach/docs/usage.rst
new file mode 100644
index 0000000000..a32b35395c
--- /dev/null
+++ b/python/mach/docs/usage.rst
@@ -0,0 +1,150 @@
+.. _mach_usage:
+
+==========
+User Guide
+==========
+
+Mach is the central entry point for most operations that can be performed in
+mozilla-central.
+
+
+Command Help
+------------
+
+To see an overview of all the available commands, run:
+
+.. code-block:: shell
+
+ $ ./mach help
+
+For more detailed information on a specific command, run:
+
+.. code-block:: shell
+
+ $ ./mach help <command>
+
+If a command has subcommands listed, you can see more details on the subcommand
+by running:
+
+.. code-block:: shell
+
+ $ ./mach help <command> <subcommand>
+
+Alternatively, you can pass ``-h/--help``. For example, all of the
+following are valid:
+
+.. code-block:: shell
+
+ $ ./mach help try
+ $ ./mach help try fuzzy
+ $ ./mach try -h
+ $ ./mach try fuzzy --help
+
+
+Tab Completion
+--------------
+
+There are commands built-in to ``mach`` that can generate a fast tab completion
+script for various shells. Supported shells are currently ``bash``, ``zsh`` and
+``fish``. These generated scripts will slowly become out of date over time, so
+you may want to create a cron task to periodically re-generate them.
+
+See below for installation instructions:
+
+Bash
+~~~~
+
+.. code-block:: shell
+
+ $ mach mach-completion bash -f _mach
+ $ sudo mv _mach /etc/bash_completion.d
+
+Bash (homebrew)
+~~~~~~~~~~~~~~~
+
+.. code-block:: shell
+
+ $ mach mach-completion bash -f $(brew --prefix)/etc/bash_completion.d/mach.bash-completion
+
+Zsh
+~~~
+
+.. code-block:: shell
+
+ $ mkdir ~/.zfunc
+ $ mach mach-completion zsh -f ~/.zfunc/_mach
+
+then edit ~/.zshrc and add:
+
+.. code-block:: shell
+
+ fpath+=~/.zfunc
+ autoload -U compinit && compinit
+
+You can use any directory of your choosing.
+
+Zsh (oh-my-zsh)
+~~~~~~~~~~~~~~~
+
+.. code-block:: shell
+
+ $ mkdir $ZSH/plugins/mach
+ $ mach mach-completion zsh -f $ZSH/plugins/mach/_mach
+
+then edit ~/.zshrc and add 'mach' to your enabled plugins:
+
+.. code-block:: shell
+
+ plugins(mach ...)
+
+Zsh (prezto)
+~~~~~~~~~~~~
+
+.. code-block:: shell
+
+ $ mach mach-completion zsh -f ~/.zprezto/modules/completion/external/src/_mach
+
+Fish
+~~~~
+
+.. code-block:: shell
+
+ $ ./mach mach-completion fish -f ~/.config/fish/completions/mach.fish
+
+Fish (homebrew)
+~~~~~~~~~~~~~~~
+
+.. code-block:: shell
+
+ $ ./mach mach-completion fish -f (brew --prefix)/share/fish/vendor_completions.d/mach.fish
+
+
+User Settings
+-------------
+
+Some mach commands can read configuration from a ``machrc`` file. The default
+location for this file is ``~/.mozbuild/machrc`` (you'll need to create it).
+This can also be set to a different location by setting the ``MACHRC``
+environment variable.
+
+For a list of all the available settings, run:
+
+.. code-block:: shell
+
+ $ ./mach settings
+
+The settings file follows the ``ini`` format, e.g:
+
+.. code-block:: ini
+
+ [alias]
+ eslint = lint -l eslint
+
+ [build]
+ telemetry = true
+
+ [try]
+ default = fuzzy
+
+
+.. _bash completion: https://searchfox.org/mozilla-central/source/python/mach/bash-completion.sh
diff --git a/python/mach/docs/windows-usage-outside-mozillabuild.rst b/python/mach/docs/windows-usage-outside-mozillabuild.rst
new file mode 100644
index 0000000000..6a034fd384
--- /dev/null
+++ b/python/mach/docs/windows-usage-outside-mozillabuild.rst
@@ -0,0 +1,124 @@
+==========================================
+Using Mach on Windows Outside MozillaBuild
+==========================================
+
+.. note::
+
+ These docs still require that you've followed the :ref:`Building Firefox On Windows` guide.
+
+`MozillaBuild <https://wiki.mozilla.org/MozillaBuild>`__ is required to build
+Firefox on Windows, because it provides necessary unix-y tools such as ``sh`` and ``awk``.
+
+Traditionally, to interact with Mach and the Firefox Build System, Windows
+developers would have to do so from within the MozillaBuild shell. This could be
+disadvantageous for two main reasons:
+
+1. The MozillaBuild environment is unix-y and based on ``bash``, which may be unfamiliar
+ for developers used to the Windows Command Prompt or Powershell.
+2. There have been long-standing stability issues with MozillaBuild - this is due to
+ the fragile interface point between the underlying "MSYS" tools and "native Windows"
+ binaries.
+
+It is now (experimentally!) possible to invoke Mach directly from other command line
+environments, such as Powershell, Command Prompt, or even a developer-managed MSYS2
+environment. Windows Terminal should work as well, for those on the "cutting edge".
+
+.. note::
+
+ If you're using a Cygwin-based environment such as MSYS2, it'll probably be
+ best to use the Windows-native version of Python (as described below) instead of a Python
+ distribution provided by the environment's package manager. Otherwise you'll likely run into
+ compatibility issues:
+
+ * Cygwin/MSYS Python will run into compatibility issues with Mach due to its unexpected Unix-y
+ conventions despite Mach assuming it's on a "Windows" platform. Additionally, there may
+ be performance issues.
+ * MinGW Python will encounter issues building native packages because they'll expect the
+ MSVC toolchain.
+
+.. warning::
+
+ This is only recommended for more advanced Windows developers: this work is experimental
+ and may run into unexpected failures!
+
+Following are steps for preparing Windows-native (Command Prompt/Powershell) usage of Mach:
+
+1. Install Python
+~~~~~~~~~~~~~~~~~
+
+Download Python from the `the official website <https://www.python.org/downloads/windows/>`__.
+
+.. note::
+
+ To avoid Mach compatibility issues with recent Python releases, it's recommended to install
+ the 2nd-most recent "major version". For example, at time of writing, the current modern Python
+ version is 3.10.1, so a safe version to install would be the most recent 3.9 release.
+
+You'll want to download the "Windows installer (64-bit)" associated with the release you've chosen.
+During installation, ensure that you check the "Add Python 3.x to PATH" option, otherwise you might
+`encounter issues running Mercurial <https://bz.mercurial-scm.org/show_bug.cgi?id=6635>`__.
+
+.. note::
+
+ Due to issues with Python DLL import failures with pip-installed binaries, it's not
+ recommended to use the Windows Store release of Python.
+
+2. Modify your PATH
+~~~~~~~~~~~~~~~~~~~
+
+The Python "user site-packages directory" needs to be added to your ``PATH`` so that packages
+installed via ``pip install --user`` (such as ``hg``) can be invoked from the command-line.
+
+1. From the Start menu, go to the Control Panel entry for "Edit environment variables
+ for your account".
+2. Double-click the ``Path`` row in the top list of variables. Click "New" to add a new item to
+ the list.
+3. In a Command Prompt window, resolve the Python directory with the command
+ ``python -c "import site; import os; print(os.path.abspath(os.path.join(site.getusersitepackages(), '..', 'Scripts')))"``.
+4. Paste the output into the new item entry in the "Edit environment variable" window.
+5. Click "New" again, and add the ``bin`` folder of MozillaBuild: probably ``C:\mozilla-build\bin``.
+6. Click "OK".
+
+3. Install Version Control System
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you're using Mercurial, you'll need to install it to your Windows-native Python:
+
+.. code-block:: shell
+
+ pip3 install --user mercurial windows-curses
+
+If you're using Git with Cinnabar, follow its `setup instructions <https://github.com/glandium/git-cinnabar#setup>`__.
+
+4. Set Powershell Execution Policy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you're using Powershell, Windows will raise an error by default when you try to invoke
+``.\mach.ps1``:
+
+.. code::
+
+ .\mach : File <topsrcdir>\mach.ps1 cannot be loaded because running scripts is disabled on this system. For
+ more information, see about_Execution_Policies at https:/go.microsoft.com/fwlink/?LinkID=135170.
+ At line:1 char:1
+
+To work around this:
+
+1. From the Start menu, type in "Powershell", then right-click on the best match and click
+ "Run as administrator"
+2. Run the command ``Set-ExecutionPolicy RemoteSigned``
+3. Close the Administrator Powershell window, and open a regular Powershell window
+4. Go to your Firefox checkout (likely ``C:\mozilla-source\mozilla-unified``)
+5. Test the new execution policy by running ``.\mach bootstrap``. If it doesn't immediately fail
+ with the error about "Execution Policies", then the problem is resolved.
+
+Success!
+~~~~~~~~
+
+At this point, you should be able to invoke Mach and manage your version control system outside
+of MozillaBuild.
+
+.. tip::
+
+ `See here <https://crisal.io/words/2022/11/22/msys2-firefox-development.html>`__ for a detailed guide on
+ installing and customizing a development environment with MSYS2, zsh, and Windows Terminal.
diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mach/mach/__init__.py
diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py
new file mode 100644
index 0000000000..fac17e9b03
--- /dev/null
+++ b/python/mach/mach/base.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+class CommandContext(object):
+ """Holds run-time state so it can easily be passed to command providers."""
+
+ def __init__(
+ self, cwd: str, settings=None, log_manager=None, commands=None, **kwargs
+ ):
+ self.cwd = cwd
+ self.settings = settings
+ self.log_manager = log_manager
+ self.commands = commands
+ self.is_interactive = None # Filled in after args are parsed
+ self.telemetry = None # Filled in after args are parsed
+ self.command_attrs = {}
+
+ for k, v in kwargs.items():
+ setattr(self, k, v)
+
+
+class MachError(Exception):
+ """Base class for all errors raised by mach itself."""
+
+
+class NoCommandError(MachError):
+ """No command was passed into mach."""
+
+ def __init__(self, namespace):
+ MachError.__init__(self)
+ self.namespace = namespace
+
+
+class UnknownCommandError(MachError):
+ """Raised when we attempted to execute an unknown command."""
+
+ def __init__(self, command, verb, suggested_commands=None):
+ MachError.__init__(self)
+
+ self.command = command
+ self.verb = verb
+ self.suggested_commands = suggested_commands or []
+
+
+class UnrecognizedArgumentError(MachError):
+ """Raised when an unknown argument is passed to mach."""
+
+ def __init__(self, command, arguments):
+ MachError.__init__(self)
+
+ self.command = command
+ self.arguments = arguments
+
+
+class FailedCommandError(Exception):
+ """Raised by commands to signal a handled failure to be printed by mach
+
+ When caught by mach a FailedCommandError will print message and exit
+ with ''exit_code''. The optional ''reason'' is a string in cases where
+ other scripts may wish to handle the exception, though this is generally
+ intended to communicate failure to mach.
+ """
+
+ def __init__(self, message, exit_code=1, reason=""):
+ Exception.__init__(self, message)
+ self.exit_code = exit_code
+ self.reason = reason
+
+
+class MissingFileError(MachError):
+ """Attempted to load a mach commands file that doesn't exist."""
diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mach/mach/commands/__init__.py
diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py
new file mode 100644
index 0000000000..12c4b240ea
--- /dev/null
+++ b/python/mach/mach/commands/commandinfo.py
@@ -0,0 +1,487 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import re
+import subprocess
+import sys
+from itertools import chain
+from pathlib import Path
+
+import attr
+from mozbuild.util import memoize
+
+from mach.decorators import Command, CommandArgument, SubCommand
+
+COMPLETION_TEMPLATES_DIR = Path(__file__).resolve().parent / "completion_templates"
+
+
+@attr.s
+class CommandInfo(object):
+ name = attr.ib(type=str)
+ description = attr.ib(type=str)
+ subcommands = attr.ib(type=list)
+ options = attr.ib(type=dict)
+ subcommand = attr.ib(type=str, default=None)
+
+
+def render_template(shell, context):
+ filename = "{}.template".format(shell)
+ with open(COMPLETION_TEMPLATES_DIR / filename) as fh:
+ template = fh.read()
+ return template % context
+
+
+@memoize
+def command_handlers(command_context):
+ """A dictionary of command handlers keyed by command name."""
+ return command_context._mach_context.commands.command_handlers
+
+
+@memoize
+def commands(command_context):
+ """A sorted list of all command names."""
+ return sorted(command_handlers(command_context))
+
+
+def _get_parser_options(parser):
+ options = {}
+ for action in parser._actions:
+ # ignore positional args
+ if not action.option_strings:
+ continue
+
+ # ignore suppressed args
+ if action.help == argparse.SUPPRESS:
+ continue
+
+ options[tuple(action.option_strings)] = action.help or ""
+ return options
+
+
+@memoize
+def global_options(command_context):
+ """Return a dict of global options.
+
+ Of the form `{("-o", "--option"): "description"}`.
+ """
+ for group in command_context._mach_context.global_parser._action_groups:
+ if group.title == "Global Arguments":
+ return _get_parser_options(group)
+
+
+@memoize
+def _get_handler_options(handler):
+ """Return a dict of options for the given handler.
+
+ Of the form `{("-o", "--option"): "description"}`.
+ """
+ options = {}
+ for option_strings, val in handler.arguments:
+ # ignore positional args
+ if option_strings[0][0] != "-":
+ continue
+
+ options[tuple(option_strings)] = val.get("help", "")
+
+ if handler._parser:
+ options.update(_get_parser_options(handler.parser))
+
+ return options
+
+
+def _get_handler_info(handler):
+ try:
+ options = _get_handler_options(handler)
+ except (Exception, SystemExit):
+ # We don't want misbehaving commands to break tab completion,
+ # ignore any exceptions.
+ options = {}
+
+ subcommands = []
+ for sub in sorted(handler.subcommand_handlers):
+ subcommands.append(_get_handler_info(handler.subcommand_handlers[sub]))
+
+ return CommandInfo(
+ name=handler.name,
+ description=handler.description or "",
+ options=options,
+ subcommands=subcommands,
+ subcommand=handler.subcommand,
+ )
+
+
+@memoize
+def commands_info(command_context):
+ """Return a list of CommandInfo objects for each command."""
+ commands_info = []
+ # Loop over self.commands() rather than self.command_handlers().items() for
+ # alphabetical order.
+ for c in commands(command_context):
+ commands_info.append(_get_handler_info(command_handlers(command_context)[c]))
+ return commands_info
+
+
+@Command("mach-commands", category="misc", description="List all mach commands.")
+def run_commands(command_context):
+ print("\n".join(commands(command_context)))
+
+
+@Command(
+ "mach-debug-commands",
+ category="misc",
+ description="Show info about available mach commands.",
+)
+@CommandArgument(
+ "match",
+ metavar="MATCH",
+ default=None,
+ nargs="?",
+ help="Only display commands containing given substring.",
+)
+def run_debug_commands(command_context, match=None):
+ import inspect
+
+ for command, handler in command_handlers(command_context).items():
+ if match and match not in command:
+ continue
+
+ func = handler.func
+
+ print(command)
+ print("=" * len(command))
+ print("")
+ print("File: %s" % inspect.getsourcefile(func))
+ print("Function: %s" % func.__name__)
+ print("")
+
+
+@Command(
+ "mach-completion",
+ category="misc",
+ description="Prints a list of completion strings for the specified command.",
+)
+@CommandArgument(
+ "args", default=None, nargs=argparse.REMAINDER, help="Command to complete."
+)
+def run_completion(command_context, args):
+ if not args:
+ print("\n".join(commands(command_context)))
+ return
+
+ is_help = "help" in args
+ command = None
+ for i, arg in enumerate(args):
+ if arg in commands(command_context):
+ command = arg
+ args = args[i + 1 :]
+ break
+
+ # If no command is typed yet, just offer the commands.
+ if not command:
+ print("\n".join(commands(command_context)))
+ return
+
+ handler = command_handlers(command_context)[command]
+ # If a subcommand was typed, update the handler.
+ for arg in args:
+ if arg in handler.subcommand_handlers:
+ handler = handler.subcommand_handlers[arg]
+ break
+
+ targets = sorted(handler.subcommand_handlers.keys())
+ if is_help:
+ print("\n".join(targets))
+ return
+
+ targets.append("help")
+ targets.extend(chain(*_get_handler_options(handler).keys()))
+ print("\n".join(targets))
+
+
+def _zsh_describe(value, description=None):
+ value = '"' + value.replace(":", "\\:")
+ if description:
+ description = subprocess.list2cmdline(
+ [re.sub(r'(["\'#&;`|*?~<>^()\[\]{}$\\\x0A\xFF])', r"\\\1", description)]
+ ).lstrip('"')
+
+ if description.endswith('"') and not description.endswith(r"\""):
+ description = description[:-1]
+
+ value += ":{}".format(description)
+
+ value += '"'
+
+ return value
+
+
+@SubCommand(
+ "mach-completion",
+ "bash",
+ description="Print mach completion script for bash shell",
+)
+@CommandArgument(
+ "-f",
+ "--file",
+ dest="outfile",
+ default=None,
+ help="File path to save completion script.",
+)
+def completion_bash(command_context, outfile):
+ commands_subcommands = []
+ case_options = []
+ case_subcommands = []
+ for i, cmd in enumerate(commands_info(command_context)):
+ # Build case statement for options.
+ options = []
+ for opt_strs, description in cmd.options.items():
+ for opt in opt_strs:
+ options.append(_zsh_describe(opt, None).strip('"'))
+
+ if options:
+ case_options.append(
+ "\n".join(
+ [
+ " ({})".format(cmd.name),
+ ' opts="${{opts}} {}"'.format(" ".join(options)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ # Build case statement for subcommand options.
+ for sub in cmd.subcommands:
+ options = []
+ for opt_strs, description in sub.options.items():
+ for opt in opt_strs:
+ options.append(_zsh_describe(opt, None))
+
+ if options:
+ case_options.append(
+ "\n".join(
+ [
+ ' ("{} {}")'.format(sub.name, sub.subcommand),
+ ' opts="${{opts}} {}"'.format(" ".join(options)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ # Build case statement for subcommands.
+ subcommands = [_zsh_describe(s.subcommand, None) for s in cmd.subcommands]
+ if subcommands:
+ commands_subcommands.append(
+ '[{}]=" {} "'.format(
+ cmd.name, " ".join([h.subcommand for h in cmd.subcommands])
+ )
+ )
+
+ case_subcommands.append(
+ "\n".join(
+ [
+ " ({})".format(cmd.name),
+ ' subs="${{subs}} {}"'.format(" ".join(subcommands)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ globalopts = [
+ opt for opt_strs in global_options(command_context) for opt in opt_strs
+ ]
+ context = {
+ "case_options": "\n".join(case_options),
+ "case_subcommands": "\n".join(case_subcommands),
+ "commands": " ".join(commands(command_context)),
+ "commands_subcommands": " ".join(sorted(commands_subcommands)),
+ "globalopts": " ".join(sorted(globalopts)),
+ }
+
+ outfile = open(outfile, "w") if outfile else sys.stdout
+ print(render_template("bash", context), file=outfile)
+
+
+@SubCommand(
+ "mach-completion",
+ "zsh",
+ description="Print mach completion script for zsh shell",
+)
+@CommandArgument(
+ "-f",
+ "--file",
+ dest="outfile",
+ default=None,
+ help="File path to save completion script.",
+)
+def completion_zsh(command_context, outfile):
+ commands_descriptions = []
+ commands_subcommands = []
+ case_options = []
+ case_subcommands = []
+ for i, cmd in enumerate(commands_info(command_context)):
+ commands_descriptions.append(_zsh_describe(cmd.name, cmd.description))
+
+ # Build case statement for options.
+ options = []
+ for opt_strs, description in cmd.options.items():
+ for opt in opt_strs:
+ options.append(_zsh_describe(opt, description))
+
+ if options:
+ case_options.append(
+ "\n".join(
+ [
+ " ({})".format(cmd.name),
+ " opts+=({})".format(" ".join(options)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ # Build case statement for subcommand options.
+ for sub in cmd.subcommands:
+ options = []
+ for opt_strs, description in sub.options.items():
+ for opt in opt_strs:
+ options.append(_zsh_describe(opt, description))
+
+ if options:
+ case_options.append(
+ "\n".join(
+ [
+ " ({} {})".format(sub.name, sub.subcommand),
+ " opts+=({})".format(" ".join(options)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ # Build case statement for subcommands.
+ subcommands = [
+ _zsh_describe(s.subcommand, s.description) for s in cmd.subcommands
+ ]
+ if subcommands:
+ commands_subcommands.append(
+ '[{}]=" {} "'.format(
+ cmd.name, " ".join([h.subcommand for h in cmd.subcommands])
+ )
+ )
+
+ case_subcommands.append(
+ "\n".join(
+ [
+ " ({})".format(cmd.name),
+ " subs+=({})".format(" ".join(subcommands)),
+ " ;;",
+ "",
+ ]
+ )
+ )
+
+ globalopts = []
+ for opt_strings, description in global_options(command_context).items():
+ for opt in opt_strings:
+ globalopts.append(_zsh_describe(opt, description))
+
+ context = {
+ "case_options": "\n".join(case_options),
+ "case_subcommands": "\n".join(case_subcommands),
+ "commands": " ".join(sorted(commands_descriptions)),
+ "commands_subcommands": " ".join(sorted(commands_subcommands)),
+ "globalopts": " ".join(sorted(globalopts)),
+ }
+
+ outfile = open(outfile, "w") if outfile else sys.stdout
+ print(render_template("zsh", context), file=outfile)
+
+
+@SubCommand(
+ "mach-completion",
+ "fish",
+ description="Print mach completion script for fish shell",
+)
+@CommandArgument(
+ "-f",
+ "--file",
+ dest="outfile",
+ default=None,
+ help="File path to save completion script.",
+)
+def completion_fish(command_context, outfile):
+ def _append_opt_strs(comp, opt_strs):
+ for opt in opt_strs:
+ if opt.startswith("--"):
+ comp += " -l {}".format(opt[2:])
+ elif opt.startswith("-"):
+ comp += " -s {}".format(opt[1:])
+ return comp
+
+ globalopts = []
+ for opt_strs, description in global_options(command_context).items():
+ comp = (
+ "complete -c mach -n '__fish_mach_complete_no_command' "
+ "-d '{}'".format(description.replace("'", "\\'"))
+ )
+ comp = _append_opt_strs(comp, opt_strs)
+ globalopts.append(comp)
+
+ cmds = []
+ cmds_opts = []
+ for i, cmd in enumerate(commands_info(command_context)):
+ cmds.append(
+ "complete -c mach -f -n '__fish_mach_complete_no_command' "
+ "-a {} -d '{}'".format(cmd.name, cmd.description.replace("'", "\\'"))
+ )
+
+ cmds_opts += ["# {}".format(cmd.name)]
+
+ subcommands = " ".join([s.subcommand for s in cmd.subcommands])
+ for opt_strs, description in cmd.options.items():
+ comp = (
+ "complete -c mach -A -n '__fish_mach_complete_command {} {}' "
+ "-d '{}'".format(cmd.name, subcommands, description.replace("'", "\\'"))
+ )
+ comp = _append_opt_strs(comp, opt_strs)
+ cmds_opts.append(comp)
+
+ for sub in cmd.subcommands:
+
+ for opt_strs, description in sub.options.items():
+ comp = (
+ "complete -c mach -A -n '__fish_mach_complete_subcommand {} {}' "
+ "-d '{}'".format(
+ sub.name, sub.subcommand, description.replace("'", "\\'")
+ )
+ )
+ comp = _append_opt_strs(comp, opt_strs)
+ cmds_opts.append(comp)
+
+ description = sub.description or ""
+ description = description.replace("'", "\\'")
+ comp = (
+ "complete -c mach -A -n '__fish_mach_complete_command {} {}' "
+ "-d '{}' -a {}".format(
+ cmd.name, subcommands, description, sub.subcommand
+ )
+ )
+ cmds_opts.append(comp)
+
+ if i < len(commands(command_context)) - 1:
+ cmds_opts.append("")
+
+ context = {
+ "commands": " ".join(commands(command_context)),
+ "command_completions": "\n".join(cmds),
+ "command_option_completions": "\n".join(cmds_opts),
+ "global_option_completions": "\n".join(globalopts),
+ }
+
+ outfile = open(outfile, "w") if outfile else sys.stdout
+ print(render_template("fish", context), file=outfile)
diff --git a/python/mach/mach/commands/completion_templates/bash.template b/python/mach/mach/commands/completion_templates/bash.template
new file mode 100644
index 0000000000..5372308702
--- /dev/null
+++ b/python/mach/mach/commands/completion_templates/bash.template
@@ -0,0 +1,62 @@
+_mach_complete()
+{
+ local com coms comsubs cur opts script sub subs
+ COMPREPLY=()
+ declare -A comsubs=( %(commands_subcommands)s )
+
+ _get_comp_words_by_ref -n : cur words
+ # for an alias, get the real script behind it
+ if [[ $(type -t ${words[0]}) == "alias" ]]; then
+ script=$(alias ${words[0]} | sed -E "s/alias ${words[0]}='(.*)'/\\1/")
+ else
+ script=${words[0]}
+ fi
+ # lookup for command and subcommand
+ for word in ${words[@]:1}; do
+ if [[ $word == -* ]]; then
+ continue
+ fi
+
+ if [[ -z $com ]]; then
+ com=$word
+ elif [[ "${comsubs[$com]}" == *" $word "* ]]; then
+ sub=$word
+ break
+ fi
+ done
+ # completing for an option
+ if [[ ${cur} == -* ]] ; then
+ if [[ -n $com ]]; then
+ if [[ -n $sub ]]; then
+ optkey="$com $sub"
+ else
+ optkey="$com"
+ fi
+ case $optkey in
+%(case_options)s
+ esac
+ else
+ # no command, complete global options
+ opts="%(globalopts)s"
+ fi
+ COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+ __ltrim_colon_completions "$cur"
+ return 0;
+ # completing for a command
+ elif [[ $cur == $com ]]; then
+ coms="%(commands)s"
+ COMPREPLY=($(compgen -W "${coms}" -- ${cur}))
+ __ltrim_colon_completions "$cur"
+ return 0
+ else
+ if [[ -z $sub ]]; then
+ case "$com" in
+%(case_subcommands)s
+ esac
+ COMPREPLY=($(compgen -W "${subs}" -- ${cur}))
+ __ltrim_colon_completions "$cur"
+ fi
+ return 0
+ fi
+}
+complete -o default -F _mach_complete mach
diff --git a/python/mach/mach/commands/completion_templates/fish.template b/python/mach/mach/commands/completion_templates/fish.template
new file mode 100644
index 0000000000..8373ee4080
--- /dev/null
+++ b/python/mach/mach/commands/completion_templates/fish.template
@@ -0,0 +1,64 @@
+function __fish_mach_complete_no_command
+ for i in (commandline -opc)
+ if contains -- $i %(commands)s
+ return 1
+ end
+ end
+ return 0
+end
+
+function __fish_mach_complete_command_matches
+ for i in (commandline -opc)
+ if contains -- $i %(commands)s
+ set com $i
+ break
+ end
+ end
+
+ if not set -q com
+ return 1
+ end
+
+ if test "$com" != "$argv"
+ return 1
+ end
+ return 0
+end
+
+function __fish_mach_complete_command
+ __fish_mach_complete_command_matches $argv[1]
+ if test $status -ne 0
+ return 1
+ end
+
+ # If a subcommand is already entered, don't complete, we should defer to
+ # '__fish_mach_complete_subcommand'.
+ for i in (commandline -opc)
+ if contains -- $i $argv[2..-1]
+ return 1
+ end
+ end
+ return 0
+end
+
+function __fish_mach_complete_subcommand
+ __fish_mach_complete_command_matches $argv[1]
+ if test $status -ne 0
+ return 1
+ end
+
+ # Command matches, now check for subcommand
+ for i in (commandline -opc)
+ if contains -- $i $argv[2]
+ return 0
+ end
+ end
+ return 1
+end
+
+# global options
+%(global_option_completions)s
+# commands
+%(command_completions)s
+# command options
+%(command_option_completions)s
diff --git a/python/mach/mach/commands/completion_templates/zsh.template b/python/mach/mach/commands/completion_templates/zsh.template
new file mode 100644
index 0000000000..21677841ef
--- /dev/null
+++ b/python/mach/mach/commands/completion_templates/zsh.template
@@ -0,0 +1,62 @@
+#compdef mach
+_mach_complete()
+{
+ local com coms comsubs cur optkey opts state sub subs
+ cur=${words[${#words[@]}]}
+ typeset -A comsubs
+ comsubs=( %(commands_subcommands)s )
+
+ # lookup for command and subcommand
+ for word in ${words[@]:1}; do
+ if [[ $word == -* ]]; then
+ continue
+ fi
+
+ if [[ -z $com ]]; then
+ com=$word
+ elif [[ ${comsubs[$com]} == *" $word "* ]]; then
+ sub=$word
+ break
+ fi
+ done
+
+ # check for a subcommand
+ if [[ $cur == $com ]]; then
+ state="command"
+ coms=(%(commands)s)
+ elif [[ ${cur} == -* ]]; then
+ state="option"
+ if [[ -z $com ]]; then
+ # no command, use global options
+ opts=(%(globalopts)s)
+ fi
+ fi
+ case $state in
+ (command)
+ _describe 'command' coms
+ ;;
+ (option)
+ if [[ -n $sub ]]; then
+ optkey="$com $sub"
+ else
+ optkey="$com"
+ fi
+ case $optkey in
+%(case_options)s
+ esac
+ _describe 'option' opts
+ ;;
+ *)
+ if [[ -z $sub ]]; then
+ # if we're completing a command with subcommands, add them here
+ case "$com" in
+%(case_subcommands)s
+ esac
+ _describe 'subcommand' subs
+ fi
+ # also fallback to file completion
+ _arguments '*:file:_files'
+ esac
+}
+_mach_complete "$@"
+compdef _mach_complete mach
diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py
new file mode 100644
index 0000000000..8e168a3921
--- /dev/null
+++ b/python/mach/mach/commands/settings.py
@@ -0,0 +1,51 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from textwrap import TextWrapper
+
+from mach.config import TYPE_CLASSES
+from mach.decorators import Command, CommandArgument
+
+
+# Interact with settings for mach.
+
+# Currently, we only provide functionality to view what settings are
+# available. In the future, this module will be used to modify settings, help
+# people create configs via a wizard, etc.
+
+
+@Command("settings", category="devenv", description="Show available config settings.")
+@CommandArgument(
+ "-l",
+ "--list",
+ dest="short",
+ action="store_true",
+ help="Show settings in a concise list",
+)
+def run_settings(command_context, short=None):
+ """List available settings."""
+ types = {v: k for k, v in TYPE_CLASSES.items()}
+ wrapper = TextWrapper(initial_indent="# ", subsequent_indent="# ")
+ for i, section in enumerate(sorted(command_context._mach_context.settings)):
+ if not short:
+ print("%s[%s]" % ("" if i == 0 else "\n", section))
+
+ for option in sorted(command_context._mach_context.settings[section]._settings):
+ meta = command_context._mach_context.settings[section].get_meta(option)
+ desc = meta["description"]
+
+ if short:
+ print("%s.%s -- %s" % (section, option, desc.splitlines()[0]))
+ continue
+
+ if option == "*":
+ option = "<option>"
+
+ if "choices" in meta:
+ value = "{%s}" % ", ".join(meta["choices"])
+ else:
+ value = "<%s>" % types[meta["type_cls"]]
+
+ print(wrapper.fill(desc))
+ print(";%s=%s" % (option, value))
diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py
new file mode 100644
index 0000000000..5428a9edad
--- /dev/null
+++ b/python/mach/mach/config.py
@@ -0,0 +1,415 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""
+This file defines classes for representing config data/settings.
+
+Config data is modeled as key-value pairs. Keys are grouped together into named
+sections. Individual config settings (options) have metadata associated with
+them. This metadata includes type, default value, valid values, etc.
+
+The main interface to config data is the ConfigSettings class. 1 or more
+ConfigProvider classes are associated with ConfigSettings and define what
+settings are available.
+"""
+
+import collections
+import collections.abc
+import sys
+from functools import wraps
+from pathlib import Path
+from typing import List, Union
+
+import six
+from six import string_types
+from six.moves.configparser import NoSectionError, RawConfigParser
+
+
+class ConfigException(Exception):
+ pass
+
+
+class ConfigType(object):
+ """Abstract base class for config values."""
+
+ @staticmethod
+ def validate(value):
+ """Validates a Python value conforms to this type.
+
+ Raises a TypeError or ValueError if it doesn't conform. Does not do
+ anything if the value is valid.
+ """
+
+ @staticmethod
+ def from_config(config, section, option):
+ """Obtain the value of this type from a RawConfigParser.
+
+ Receives a RawConfigParser instance, a str section name, and the str
+ option in that section to retrieve.
+
+ The implementation may assume the option exists in the RawConfigParser
+ instance.
+
+ Implementations are not expected to validate the value. But, they
+ should return the appropriate Python type.
+ """
+
+ @staticmethod
+ def to_config(value):
+ return value
+
+
+class StringType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, string_types):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.get(section, option)
+
+
+class BooleanType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, bool):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.getboolean(section, option)
+
+ @staticmethod
+ def to_config(value):
+ return "true" if value else "false"
+
+
+class IntegerType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, int):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.getint(section, option)
+
+
+class PositiveIntegerType(IntegerType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, int):
+ raise TypeError()
+
+ if value < 0:
+ raise ValueError()
+
+
+class PathType(StringType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, string_types):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.get(section, option)
+
+
+TYPE_CLASSES = {
+ "string": StringType,
+ "boolean": BooleanType,
+ "int": IntegerType,
+ "pos_int": PositiveIntegerType,
+ "path": PathType,
+}
+
+
+class DefaultValue(object):
+ pass
+
+
+def reraise_attribute_error(func):
+ """Used to make sure __getattr__ wrappers around __getitem__
+ raise AttributeError instead of KeyError.
+ """
+
+ @wraps(func)
+ def _(*args, **kwargs):
+ try:
+ return func(*args, **kwargs)
+ except KeyError:
+ exc_class, exc, tb = sys.exc_info()
+ six.reraise(AttributeError().__class__, exc, tb)
+
+ return _
+
+
+class ConfigSettings(collections.abc.Mapping):
+ """Interface for configuration settings.
+
+ This is the main interface to the configuration.
+
+ A configuration is a collection of sections. Each section contains
+ key-value pairs.
+
+ When an instance is created, the caller first registers ConfigProvider
+ instances with it. This tells the ConfigSettings what individual settings
+ are available and defines extra metadata associated with those settings.
+ This is used for validation, etc.
+
+ Once ConfigProvider instances are registered, a config is populated. It can
+ be loaded from files or populated by hand.
+
+ ConfigSettings instances are accessed like dictionaries or by using
+ attributes. e.g. the section "foo" is accessed through either
+ settings.foo or settings['foo'].
+
+ Sections are modeled by the ConfigSection class which is defined inside
+ this one. They look just like dicts or classes with attributes. To access
+ the "bar" option in the "foo" section:
+
+ value = settings.foo.bar
+ value = settings['foo']['bar']
+ value = settings.foo['bar']
+
+ Assignment is similar:
+
+ settings.foo.bar = value
+ settings['foo']['bar'] = value
+ settings['foo'].bar = value
+
+ You can even delete user-assigned values:
+
+ del settings.foo.bar
+ del settings['foo']['bar']
+
+ If there is a default, it will be returned.
+
+ When settings are mutated, they are validated against the registered
+ providers. Setting unknown settings or setting values to illegal values
+ will result in exceptions being raised.
+ """
+
+ class ConfigSection(collections.abc.MutableMapping, object):
+ """Represents an individual config section."""
+
+ def __init__(self, config, name, settings):
+ object.__setattr__(self, "_config", config)
+ object.__setattr__(self, "_name", name)
+ object.__setattr__(self, "_settings", settings)
+
+ wildcard = any(s == "*" for s in self._settings)
+ object.__setattr__(self, "_wildcard", wildcard)
+
+ @property
+ def options(self):
+ try:
+ return self._config.options(self._name)
+ except NoSectionError:
+ return []
+
+ def get_meta(self, option):
+ if option in self._settings:
+ return self._settings[option]
+ if self._wildcard:
+ return self._settings["*"]
+ raise KeyError("Option not registered with provider: %s" % option)
+
+ def _validate(self, option, value):
+ meta = self.get_meta(option)
+ meta["type_cls"].validate(value)
+
+ if "choices" in meta and value not in meta["choices"]:
+ raise ValueError(
+ "Value '%s' must be one of: %s"
+ % (value, ", ".join(sorted(meta["choices"])))
+ )
+
+ # MutableMapping interface
+ def __len__(self):
+ return len(self.options)
+
+ def __iter__(self):
+ return iter(self.options)
+
+ def __contains__(self, k):
+ return self._config.has_option(self._name, k)
+
+ def __getitem__(self, k):
+ meta = self.get_meta(k)
+
+ if self._config.has_option(self._name, k):
+ v = meta["type_cls"].from_config(self._config, self._name, k)
+ else:
+ v = meta.get("default", DefaultValue)
+
+ if v == DefaultValue:
+ raise KeyError("No default value registered: %s" % k)
+
+ self._validate(k, v)
+ return v
+
+ def __setitem__(self, k, v):
+ self._validate(k, v)
+ meta = self.get_meta(k)
+
+ if not self._config.has_section(self._name):
+ self._config.add_section(self._name)
+
+ self._config.set(self._name, k, meta["type_cls"].to_config(v))
+
+ def __delitem__(self, k):
+ self._config.remove_option(self._name, k)
+
+ # Prune empty sections.
+ if not len(self._config.options(self._name)):
+ self._config.remove_section(self._name)
+
+ @reraise_attribute_error
+ def __getattr__(self, k):
+ return self.__getitem__(k)
+
+ @reraise_attribute_error
+ def __setattr__(self, k, v):
+ self.__setitem__(k, v)
+
+ @reraise_attribute_error
+ def __delattr__(self, k):
+ self.__delitem__(k)
+
+ def __init__(self):
+ self._config = RawConfigParser()
+ self._config.optionxform = str
+
+ self._settings = {}
+ self._sections = {}
+ self._finalized = False
+
+ def load_file(self, filename: Union[str, Path]):
+ self.load_files([Path(filename)])
+
+ def load_files(self, filenames: List[Path]):
+ """Load a config from files specified by their paths.
+
+ Files are loaded in the order given. Subsequent files will overwrite
+ values from previous files. If a file does not exist, it will be
+ ignored.
+ """
+ filtered = [f for f in filenames if f.exists()]
+
+ fps = [open(f, "rt") for f in filtered]
+ self.load_fps(fps)
+ for fp in fps:
+ fp.close()
+
+ def load_fps(self, fps):
+ """Load config data by reading file objects."""
+
+ for fp in fps:
+ self._config.readfp(fp)
+
+ def write(self, fh):
+ """Write the config to a file object."""
+ self._config.write(fh)
+
+ @classmethod
+ def _format_metadata(cls, type_cls, description, default=DefaultValue, extra=None):
+ """Formats and returns the metadata for a setting.
+
+ Each setting must have:
+
+ type_cls -- a ConfigType-derived type defining the type of the setting.
+
+ description -- str describing how to use the setting and where it
+ applies.
+
+ Each setting has the following optional parameters:
+
+ default -- The default value for the setting. If None (the default)
+ there is no default.
+
+ extra -- A dict of additional key/value pairs to add to the
+ setting metadata.
+ """
+ if isinstance(type_cls, string_types):
+ type_cls = TYPE_CLASSES[type_cls]
+
+ meta = {"description": description, "type_cls": type_cls}
+
+ if default != DefaultValue:
+ meta["default"] = default
+
+ if extra:
+ meta.update(extra)
+
+ return meta
+
+ def register_provider(self, provider):
+ """Register a SettingsProvider with this settings interface."""
+
+ if self._finalized:
+ raise ConfigException("Providers cannot be registered after finalized.")
+
+ settings = provider.config_settings
+ if callable(settings):
+ settings = settings()
+
+ config_settings = collections.defaultdict(dict)
+ for setting in settings:
+ section, option = setting[0].split(".")
+
+ if option in config_settings[section]:
+ raise ConfigException(
+ "Setting has already been registered: %s.%s" % (section, option)
+ )
+
+ meta = self._format_metadata(*setting[1:])
+ config_settings[section][option] = meta
+
+ for section_name, settings in config_settings.items():
+ section = self._settings.get(section_name, {})
+
+ for k, v in settings.items():
+ if k in section:
+ raise ConfigException(
+ "Setting already registered: %s.%s" % (section_name, k)
+ )
+
+ section[k] = v
+
+ self._settings[section_name] = section
+
+ def _finalize(self):
+ if self._finalized:
+ return
+
+ for section, settings in self._settings.items():
+ s = ConfigSettings.ConfigSection(self._config, section, settings)
+ self._sections[section] = s
+
+ self._finalized = True
+
+ # Mapping interface.
+ def __len__(self):
+ return len(self._settings)
+
+ def __iter__(self):
+ self._finalize()
+
+ return iter(self._sections.keys())
+
+ def __contains__(self, k):
+ return k in self._settings
+
+ def __getitem__(self, k):
+ self._finalize()
+
+ return self._sections[k]
+
+ # Allow attribute access because it looks nice.
+ @reraise_attribute_error
+ def __getattr__(self, k):
+ return self.__getitem__(k)
diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py
new file mode 100644
index 0000000000..fe4443e168
--- /dev/null
+++ b/python/mach/mach/decorators.py
@@ -0,0 +1,340 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import collections
+import collections.abc
+from typing import Optional
+
+from mozbuild.base import MachCommandBase
+
+from .base import MachError
+from .registrar import Registrar
+
+
+class _MachCommand(object):
+ """Container for mach command metadata."""
+
+ __slots__ = (
+ # Content from decorator arguments to define the command.
+ "name",
+ "subcommand",
+ "category",
+ "description",
+ "conditions",
+ "_parser",
+ "arguments",
+ "argument_group_names",
+ "virtualenv_name",
+ "ok_if_tests_disabled",
+ # By default, subcommands will be sorted. If this is set to
+ # 'declaration', they will be left in declaration order.
+ "order",
+ # This is the function or callable that will be called when
+ # the command is invoked
+ "func",
+ # The path to the `metrics.yaml` file that describes data that telemetry will
+ # gather for this command. This path is optional.
+ "metrics_path",
+ # Dict of string to _MachCommand defining sub-commands for this
+ # command.
+ "subcommand_handlers",
+ # For subcommands, the global order that the subcommand's declaration
+ # was seen.
+ "decl_order",
+ # Whether to disable automatic logging to last_log.json for the command.
+ "no_auto_log",
+ )
+
+ def __init__(
+ self,
+ name=None,
+ subcommand=None,
+ category=None,
+ description=None,
+ conditions=None,
+ parser=None,
+ order=None,
+ virtualenv_name=None,
+ ok_if_tests_disabled=False,
+ no_auto_log=False,
+ ):
+ self.name = name
+ self.subcommand = subcommand
+ self.category = category
+ self.description = description
+ self.conditions = conditions or []
+ self._parser = parser
+ self.arguments = []
+ self.argument_group_names = []
+ self.virtualenv_name = virtualenv_name
+ self.order = order
+ if ok_if_tests_disabled and category != "testing":
+ raise ValueError(
+ "ok_if_tests_disabled should only be set for " "`testing` mach commands"
+ )
+ self.ok_if_tests_disabled = ok_if_tests_disabled
+
+ self.func = None
+ self.metrics_path = None
+ self.subcommand_handlers = {}
+ self.decl_order = None
+ self.no_auto_log = no_auto_log
+
+ def create_instance(self, context, virtualenv_name):
+ metrics = None
+ if self.metrics_path:
+ metrics = context.telemetry.metrics(self.metrics_path)
+
+ # This ensures the resulting class is defined inside `mach` so that logging
+ # works as expected, and has a meaningful name
+ subclass = type(self.name, (MachCommandBase,), {})
+ return subclass(
+ context,
+ virtualenv_name=virtualenv_name,
+ metrics=metrics,
+ no_auto_log=self.no_auto_log,
+ )
+
+ @property
+ def parser(self):
+ # Creating CLI parsers at command dispatch time can be expensive. Make
+ # it possible to lazy load them by using functions.
+ if callable(self._parser):
+ self._parser = self._parser()
+
+ return self._parser
+
+ @property
+ def docstring(self):
+ return self.func.__doc__
+
+ def __ior__(self, other):
+ if not isinstance(other, _MachCommand):
+ raise ValueError("can only operate on _MachCommand instances")
+
+ for a in self.__slots__:
+ if not getattr(self, a):
+ setattr(self, a, getattr(other, a))
+
+ return self
+
+ def register(self, func):
+ """Register the command in the Registrar with the function to be called on invocation."""
+ if not self.subcommand:
+ if not self.conditions and Registrar.require_conditions:
+ return
+
+ msg = (
+ "Mach command '%s' implemented incorrectly. "
+ + "Conditions argument must take a list "
+ + "of functions. Found %s instead."
+ )
+
+ if not isinstance(self.conditions, collections.abc.Iterable):
+ msg = msg % (self.name, type(self.conditions))
+ raise MachError(msg)
+
+ for c in self.conditions:
+ if not hasattr(c, "__call__"):
+ msg = msg % (self.name, type(c))
+ raise MachError(msg)
+
+ self.func = func
+
+ Registrar.register_command_handler(self)
+
+ else:
+ if self.name not in Registrar.command_handlers:
+ raise MachError(
+ "Command referenced by sub-command does not exist: %s" % self.name
+ )
+
+ self.func = func
+ parent = Registrar.command_handlers[self.name]
+
+ if self.subcommand in parent.subcommand_handlers:
+ raise MachError("sub-command already defined: %s" % self.subcommand)
+
+ parent.subcommand_handlers[self.subcommand] = self
+
+
+class Command(object):
+ """Decorator for functions or methods that provide a mach command.
+
+ The decorator accepts arguments that define basic attributes of the
+ command. The following arguments are recognized:
+
+ category -- The string category to which this command belongs. Mach's
+ help will group commands by category.
+
+ description -- A brief description of what the command does.
+
+ parser -- an optional argparse.ArgumentParser instance or callable
+ that returns an argparse.ArgumentParser instance to use as the
+ basis for the command arguments.
+
+ For example:
+
+ .. code-block:: python
+
+ @Command('foo', category='misc', description='Run the foo action')
+ def foo(self, command_context):
+ pass
+ """
+
+ def __init__(self, name, metrics_path: Optional[str] = None, **kwargs):
+ self._mach_command = _MachCommand(name=name, **kwargs)
+ self._mach_command.metrics_path = metrics_path
+
+ def __call__(self, func):
+ if not hasattr(func, "_mach_command"):
+ func._mach_command = _MachCommand()
+
+ func._mach_command |= self._mach_command
+ func._mach_command.register(func)
+
+ return func
+
+
+class SubCommand(object):
+ """Decorator for functions or methods that provide a sub-command.
+
+ Mach commands can have sub-commands. e.g. ``mach command foo`` or
+ ``mach command bar``. Each sub-command has its own parser and is
+ effectively its own mach command.
+
+ The decorator accepts arguments that define basic attributes of the
+ sub command:
+
+ command -- The string of the command this sub command should be
+ attached to.
+
+ subcommand -- The string name of the sub command to register.
+
+ description -- A textual description for this sub command.
+ """
+
+ global_order = 0
+
+ def __init__(
+ self,
+ command,
+ subcommand,
+ description=None,
+ parser=None,
+ metrics_path: Optional[str] = None,
+ virtualenv_name: Optional[str] = None,
+ ):
+ self._mach_command = _MachCommand(
+ name=command,
+ subcommand=subcommand,
+ description=description,
+ parser=parser,
+ virtualenv_name=virtualenv_name,
+ )
+ self._mach_command.decl_order = SubCommand.global_order
+ SubCommand.global_order += 1
+
+ self._mach_command.metrics_path = metrics_path
+
+ def __call__(self, func):
+ if not hasattr(func, "_mach_command"):
+ func._mach_command = _MachCommand()
+
+ func._mach_command |= self._mach_command
+ func._mach_command.register(func)
+
+ return func
+
+
+class CommandArgument(object):
+ """Decorator for additional arguments to mach subcommands.
+
+ This decorator should be used to add arguments to mach commands. Arguments
+ to the decorator are proxied to ArgumentParser.add_argument().
+
+ For example:
+
+ .. code-block:: python
+
+ @Command('foo', help='Run the foo action')
+ @CommandArgument('-b', '--bar', action='store_true', default=False,
+ help='Enable bar mode.')
+ def foo(self, command_context):
+ pass
+ """
+
+ def __init__(self, *args, **kwargs):
+ if kwargs.get("nargs") == argparse.REMAINDER:
+ # These are the assertions we make in dispatcher.py about
+ # those types of CommandArguments.
+ assert len(args) == 1
+ assert all(
+ k in ("default", "nargs", "help", "group", "metavar") for k in kwargs
+ )
+ self._command_args = (args, kwargs)
+
+ def __call__(self, func):
+ if not hasattr(func, "_mach_command"):
+ func._mach_command = _MachCommand()
+
+ func._mach_command.arguments.insert(0, self._command_args)
+
+ return func
+
+
+class CommandArgumentGroup(object):
+ """Decorator for additional argument groups to mach commands.
+
+ This decorator should be used to add arguments groups to mach commands.
+ Arguments to the decorator are proxied to
+ ArgumentParser.add_argument_group().
+
+ For example:
+
+ .. code-block: python
+
+ @Command('foo', helps='Run the foo action')
+ @CommandArgumentGroup('group1')
+ @CommandArgument('-b', '--bar', group='group1', action='store_true',
+ default=False, help='Enable bar mode.')
+ def foo(self, command_context):
+ pass
+
+ The name should be chosen so that it makes sense as part of the phrase
+ 'Command Arguments for <name>' because that's how it will be shown in the
+ help message.
+ """
+
+ def __init__(self, group_name):
+ self._group_name = group_name
+
+ def __call__(self, func):
+ if not hasattr(func, "_mach_command"):
+ func._mach_command = _MachCommand()
+
+ func._mach_command.argument_group_names.insert(0, self._group_name)
+
+ return func
+
+
+def SettingsProvider(cls):
+ """Class decorator to denote that this class provides Mach settings.
+
+ When this decorator is encountered, the underlying class will automatically
+ be registered with the Mach registrar and will (likely) be hooked up to the
+ mach driver.
+ """
+ if not hasattr(cls, "config_settings"):
+ raise MachError(
+ "@SettingsProvider must contain a config_settings attribute. It "
+ "may either be a list of tuples, or a callable that returns a list "
+ "of tuples. Each tuple must be of the form:\n"
+ "(<section>.<option>, <type_cls>, <description>, <default>, <choices>)\n"
+ "as specified by ConfigSettings._format_metadata."
+ )
+
+ Registrar.register_settings_provider(cls)
+ return cls
diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py
new file mode 100644
index 0000000000..95287eac40
--- /dev/null
+++ b/python/mach/mach/dispatcher.py
@@ -0,0 +1,516 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import difflib
+import shlex
+import sys
+from operator import itemgetter
+
+from .base import NoCommandError, UnknownCommandError, UnrecognizedArgumentError
+from .decorators import SettingsProvider
+
+
+@SettingsProvider
+class DispatchSettings:
+ config_settings = [
+ (
+ "alias.*",
+ "string",
+ """
+Create a command alias of the form `<alias>=<command> <args>`.
+Aliases can also be used to set default arguments:
+<command>=<command> <args>
+""".strip(),
+ ),
+ ]
+
+
+class CommandFormatter(argparse.HelpFormatter):
+ """Custom formatter to format just a subcommand."""
+
+ def add_usage(self, *args):
+ pass
+
+
+class CommandAction(argparse.Action):
+ """An argparse action that handles mach commands.
+
+ This class is essentially a reimplementation of argparse's sub-parsers
+ feature. We first tried to use sub-parsers. However, they were missing
+ features like grouping of commands (http://bugs.python.org/issue14037).
+
+ The way this works involves light magic and a partial understanding of how
+ argparse works.
+
+ Arguments registered with an argparse.ArgumentParser have an action
+ associated with them. An action is essentially a class that when called
+ does something with the encountered argument(s). This class is one of those
+ action classes.
+
+ An instance of this class is created doing something like:
+
+ parser.add_argument('command', action=CommandAction, registrar=r)
+
+ Note that a mach.registrar.Registrar instance is passed in. The Registrar
+ holds information on all the mach commands that have been registered.
+
+ When this argument is registered with the ArgumentParser, an instance of
+ this class is instantiated. One of the subtle but important things it does
+ is tell the argument parser that it's interested in *all* of the remaining
+ program arguments. So, when the ArgumentParser calls this action, we will
+ receive the command name plus all of its arguments.
+
+ For more, read the docs in __call__.
+ """
+
+ def __init__(
+ self,
+ option_strings,
+ dest,
+ required=True,
+ default=None,
+ registrar=None,
+ context=None,
+ ):
+ # A proper API would have **kwargs here. However, since we are a little
+ # hacky, we intentionally omit it as a way of detecting potentially
+ # breaking changes with argparse's implementation.
+ #
+ # In a similar vein, default is passed in but is not needed, so we drop
+ # it.
+ argparse.Action.__init__(
+ self,
+ option_strings,
+ dest,
+ required=required,
+ help=argparse.SUPPRESS,
+ nargs=argparse.REMAINDER,
+ )
+
+ self._mach_registrar = registrar
+ self._context = context
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ """This is called when the ArgumentParser has reached our arguments.
+
+ Since we always register ourselves with nargs=argparse.REMAINDER,
+ values should be a list of remaining arguments to parse. The first
+ argument should be the name of the command to invoke and all remaining
+ arguments are arguments for that command.
+
+ The gist of the flow is that we look at the command being invoked. If
+ it's *help*, we handle that specially (because argparse's default help
+ handler isn't satisfactory). Else, we create a new, independent
+ ArgumentParser instance for just the invoked command (based on the
+ information contained in the command registrar) and feed the arguments
+ into that parser. We then merge the results with the main
+ ArgumentParser.
+ """
+ if namespace.help:
+ # -h or --help is in the global arguments.
+ self._handle_main_help(parser, namespace.verbose)
+ sys.exit(0)
+ elif values:
+ command = values[0].lower()
+ args = values[1:]
+ if command == "help":
+ if args and args[0] not in ["-h", "--help"]:
+ # Make sure args[0] is indeed a command.
+ self._handle_command_help(parser, args[0], args)
+ else:
+ self._handle_main_help(parser, namespace.verbose)
+ sys.exit(0)
+ elif "-h" in args or "--help" in args:
+ # -h or --help is in the command arguments.
+ if "--" in args:
+ # -- is in command arguments
+ if (
+ "-h" in args[: args.index("--")]
+ or "--help" in args[: args.index("--")]
+ ):
+ # Honor -h or --help only if it appears before --
+ self._handle_command_help(parser, command, args)
+ sys.exit(0)
+ else:
+ self._handle_command_help(parser, command, args)
+ sys.exit(0)
+ else:
+ raise NoCommandError(namespace)
+
+ # First see if the this is a user-defined alias
+ if command in self._context.settings.alias:
+ alias = self._context.settings.alias[command]
+ defaults = shlex.split(alias)
+ command = defaults.pop(0)
+ args = defaults + args
+
+ if command not in self._mach_registrar.command_handlers:
+ # Try to find similar commands, may raise UnknownCommandError.
+ command = self._suggest_command(command)
+
+ handler = self._mach_registrar.command_handlers.get(command)
+
+ prog = command
+ usage = "%(prog)s [global arguments] " + command + " [command arguments]"
+
+ subcommand = None
+
+ # If there are sub-commands, parse the intent out immediately.
+ if handler.subcommand_handlers and args:
+ # mach <command> help <subcommand>
+ if set(args[: args.index("--")] if "--" in args else args).intersection(
+ ("help", "--help")
+ ):
+ self._handle_subcommand_help(parser, handler, args)
+ sys.exit(0)
+ # mach <command> <subcommand> ...
+ elif args[0] in handler.subcommand_handlers:
+ subcommand = args[0]
+ handler = handler.subcommand_handlers[subcommand]
+ prog = prog + " " + subcommand
+ usage = (
+ "%(prog)s [global arguments] "
+ + command
+ + " "
+ + subcommand
+ + " [command arguments]"
+ )
+ args.pop(0)
+
+ # We create a new parser, populate it with the command's arguments,
+ # then feed all remaining arguments to it, merging the results
+ # with ourselves. This is essentially what argparse subparsers
+ # do.
+
+ parser_args = {
+ "add_help": False,
+ "usage": usage,
+ }
+
+ remainder = None
+
+ if handler.parser:
+ subparser = handler.parser
+ subparser.context = self._context
+ subparser.prog = subparser.prog + " " + prog
+ for arg in subparser._actions[:]:
+ if arg.nargs == argparse.REMAINDER:
+ subparser._actions.remove(arg)
+ remainder = (
+ (arg.dest,),
+ {"default": arg.default, "nargs": arg.nargs, "help": arg.help},
+ )
+ else:
+ subparser = argparse.ArgumentParser(**parser_args)
+
+ for arg in handler.arguments:
+ # Remove our group keyword; it's not needed here.
+ group_name = arg[1].get("group")
+ if group_name:
+ del arg[1]["group"]
+
+ if arg[1].get("nargs") == argparse.REMAINDER:
+ # parse_known_args expects all argparse.REMAINDER ('...')
+ # arguments to be all stuck together. Instead, we want them to
+ # pick any extra argument, wherever they are.
+ # Assume a limited CommandArgument for those arguments.
+ assert len(arg[0]) == 1
+ assert all(k in ("default", "nargs", "help", "metavar") for k in arg[1])
+ remainder = arg
+ else:
+ subparser.add_argument(*arg[0], **arg[1])
+
+ # We define the command information on the main parser result so as to
+ # not interfere with arguments passed to the command.
+ setattr(namespace, "mach_handler", handler)
+ setattr(namespace, "command", command)
+ setattr(namespace, "subcommand", subcommand)
+
+ command_namespace, extra = subparser.parse_known_args(args)
+ setattr(namespace, "command_args", command_namespace)
+ if remainder:
+ (name,), options = remainder
+ # parse_known_args usefully puts all arguments after '--' in
+ # extra, but also puts '--' there. We don't want to pass it down
+ # to the command handler. Note that if multiple '--' are on the
+ # command line, only the first one is removed, so that subsequent
+ # ones are passed down.
+ if "--" in extra:
+ extra.remove("--")
+
+ # Commands with argparse.REMAINDER arguments used to force the
+ # other arguments to be '+' prefixed. If a user now passes such
+ # an argument, if will silently end up in extra. So, check if any
+ # of the allowed arguments appear in a '+' prefixed form, and error
+ # out if that's the case.
+ for args, _ in handler.arguments:
+ for arg in args:
+ arg = arg.replace("-", "+", 1)
+ if arg in extra:
+ raise UnrecognizedArgumentError(command, [arg])
+
+ if extra:
+ setattr(command_namespace, name, extra)
+ else:
+ setattr(command_namespace, name, options.get("default", []))
+ elif extra:
+ raise UnrecognizedArgumentError(command, extra)
+
+ def _handle_main_help(self, parser, verbose):
+ # Since we don't need full sub-parser support for the main help output,
+ # we create groups in the ArgumentParser and populate each group with
+ # arguments corresponding to command names. This has the side-effect
+ # that argparse renders it nicely.
+ r = self._mach_registrar
+ disabled_commands = []
+
+ cats = [(k, v[2]) for k, v in r.categories.items()]
+ sorted_cats = sorted(cats, key=itemgetter(1), reverse=True)
+ for category, priority in sorted_cats:
+ group = None
+
+ for command in sorted(r.commands_by_category[category]):
+ handler = r.command_handlers[command]
+
+ # Instantiate a handler class to see if it should be filtered
+ # out for the current context or not. Condition functions can be
+ # applied to the command's decorator.
+ if handler.conditions:
+ instance = handler.create_instance(
+ self._context, handler.virtualenv_name
+ )
+
+ is_filtered = False
+ for c in handler.conditions:
+ if not c(instance):
+ is_filtered = True
+ break
+ if is_filtered:
+ description = handler.description
+ disabled_command = {
+ "command": command,
+ "description": description,
+ }
+ disabled_commands.append(disabled_command)
+ continue
+
+ if group is None:
+ title, description, _priority = r.categories[category]
+ group = parser.add_argument_group(title, description)
+
+ description = handler.description
+ group.add_argument(command, help=description, action="store_true")
+
+ if disabled_commands and "disabled" in r.categories:
+ title, description, _priority = r.categories["disabled"]
+ group = parser.add_argument_group(title, description)
+ if verbose:
+ for c in disabled_commands:
+ group.add_argument(
+ c["command"], help=c["description"], action="store_true"
+ )
+
+ parser.print_help()
+
+ def _populate_command_group(self, parser, handler, group):
+ extra_groups = {}
+ for group_name in handler.argument_group_names:
+ group_full_name = "Command Arguments for " + group_name
+ extra_groups[group_name] = parser.add_argument_group(group_full_name)
+
+ for arg in handler.arguments:
+ # Apply our group keyword.
+ group_name = arg[1].get("group")
+ if group_name:
+ del arg[1]["group"]
+ group = extra_groups[group_name]
+ group.add_argument(*arg[0], **arg[1])
+
+ def _get_command_arguments_help(self, handler):
+ # This code is worth explaining. Because we are doing funky things with
+ # argument registration to allow the same option in both global and
+ # command arguments, we can't simply put all arguments on the same
+ # parser instance because argparse would complain. We can't register an
+ # argparse subparser here because it won't properly show help for
+ # global arguments. So, we employ a strategy similar to command
+ # execution where we construct a 2nd, independent ArgumentParser for
+ # just the command data then supplement the main help's output with
+ # this 2nd parser's. We use a custom formatter class to ignore some of
+ # the help output.
+ parser_args = {
+ "formatter_class": CommandFormatter,
+ "add_help": False,
+ }
+
+ if handler.parser:
+ c_parser = handler.parser
+ c_parser.context = self._context
+ c_parser.formatter_class = NoUsageFormatter
+ # Accessing _action_groups is a bit shady. We are highly dependent
+ # on the argparse implementation not changing. We fail fast to
+ # detect upstream changes so we can intelligently react to them.
+ group = c_parser._action_groups[1]
+
+ # By default argparse adds two groups called "positional arguments"
+ # and "optional arguments". We want to rename these to reflect standard
+ # mach terminology.
+ c_parser._action_groups[0].title = "Command Parameters"
+ c_parser._action_groups[1].title = "Command Arguments"
+
+ if not handler.description:
+ handler.description = c_parser.description
+ c_parser.description = None
+ else:
+ c_parser = argparse.ArgumentParser(**parser_args)
+ group = c_parser.add_argument_group("Command Arguments")
+
+ self._populate_command_group(c_parser, handler, group)
+
+ return c_parser
+
+ def _handle_command_help(self, parser, command, args):
+ handler = self._mach_registrar.command_handlers.get(command)
+
+ if not handler:
+ raise UnknownCommandError(command, "query")
+
+ if handler.subcommand_handlers:
+ self._handle_subcommand_help(parser, handler, args)
+ return
+
+ c_parser = self._get_command_arguments_help(handler)
+
+ # Set the long help of the command to the docstring (if present) or
+ # the command decorator description argument (if present).
+ if handler.docstring:
+ parser.description = format_docstring(handler.docstring)
+ elif handler.description:
+ parser.description = handler.description
+
+ parser.usage = "%(prog)s [global arguments] " + command + " [command arguments]"
+
+ # This is needed to preserve line endings in the description field,
+ # which may be populated from a docstring.
+ parser.formatter_class = argparse.RawDescriptionHelpFormatter
+ parser.print_help()
+ print("")
+ c_parser.print_help()
+
+ def _handle_subcommand_main_help(self, parser, handler):
+ parser.usage = (
+ "%(prog)s [global arguments] "
+ + handler.name
+ + " subcommand [subcommand arguments]"
+ )
+ group = parser.add_argument_group("Sub Commands")
+
+ def by_decl_order(item):
+ return item[1].decl_order
+
+ def by_name(item):
+ return item[1].subcommand
+
+ subhandlers = handler.subcommand_handlers.items()
+ for subcommand, subhandler in sorted(
+ subhandlers,
+ key=by_decl_order if handler.order == "declaration" else by_name,
+ ):
+ group.add_argument(
+ subcommand, help=subhandler.description, action="store_true"
+ )
+
+ if handler.docstring:
+ parser.description = format_docstring(handler.docstring)
+
+ c_parser = self._get_command_arguments_help(handler)
+
+ parser.formatter_class = argparse.RawDescriptionHelpFormatter
+
+ parser.print_help()
+ print("")
+ c_parser.print_help()
+
+ def _handle_subcommand_help(self, parser, handler, args):
+ subcommand = set(args).intersection(list(handler.subcommand_handlers.keys()))
+ if not subcommand:
+ return self._handle_subcommand_main_help(parser, handler)
+
+ subcommand = subcommand.pop()
+ subhandler = handler.subcommand_handlers[subcommand]
+
+ # Initialize the parser if necessary
+ subhandler.parser
+
+ c_parser = subhandler.parser or argparse.ArgumentParser(add_help=False)
+ c_parser.formatter_class = CommandFormatter
+
+ group = c_parser.add_argument_group("Sub Command Arguments")
+ self._populate_command_group(c_parser, subhandler, group)
+
+ if subhandler.docstring:
+ parser.description = format_docstring(subhandler.docstring)
+
+ parser.formatter_class = argparse.RawDescriptionHelpFormatter
+ parser.usage = (
+ "%(prog)s [global arguments] "
+ + handler.name
+ + " "
+ + subcommand
+ + " [command arguments]"
+ )
+
+ parser.print_help()
+ print("")
+ c_parser.print_help()
+
+ def _suggest_command(self, command):
+ names = [h.name for h in self._mach_registrar.command_handlers.values()]
+ # We first try to look for a valid command that is very similar to the given command.
+ suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8)
+ # If we find more than one matching command, or no command at all,
+ # we give command suggestions instead (with a lower matching threshold).
+ # All commands that start with the given command (for instance:
+ # 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-')
+ # are also included.
+ if len(suggested_commands) != 1:
+ suggested_commands = set(
+ difflib.get_close_matches(command, names, cutoff=0.5)
+ )
+ suggested_commands |= {cmd for cmd in names if cmd.startswith(command)}
+ raise UnknownCommandError(command, "run", suggested_commands)
+ sys.stderr.write(
+ "We're assuming the '%s' command is '%s' and we're "
+ "executing it for you.\n\n" % (command, suggested_commands[0])
+ )
+ return suggested_commands[0]
+
+
+class NoUsageFormatter(argparse.HelpFormatter):
+ def _format_usage(self, *args, **kwargs):
+ return ""
+
+
+def format_docstring(docstring):
+ """Format a raw docstring into something suitable for presentation.
+
+ This function is based on the example function in PEP-0257.
+ """
+ if not docstring:
+ return ""
+ lines = docstring.expandtabs().splitlines()
+ indent = sys.maxsize
+ for line in lines[1:]:
+ stripped = line.lstrip()
+ if stripped:
+ indent = min(indent, len(line) - len(stripped))
+ trimmed = [lines[0].strip()]
+ if indent < sys.maxsize:
+ for line in lines[1:]:
+ trimmed.append(line[indent:].rstrip())
+ while trimmed and not trimmed[-1]:
+ trimmed.pop()
+ while trimmed and not trimmed[0]:
+ trimmed.pop(0)
+ return "\n".join(trimmed)
diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py
new file mode 100644
index 0000000000..d39f336cc0
--- /dev/null
+++ b/python/mach/mach/logging.py
@@ -0,0 +1,398 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains logging functionality for mach. It essentially provides
+# support for a structured logging framework built on top of Python's built-in
+# logging framework.
+
+import codecs
+import json
+import logging
+import os
+import sys
+import time
+
+import blessed
+import six
+from mozbuild.util import mozilla_build_version
+from packaging.version import Version
+
+IS_WINDOWS = sys.platform.startswith("win")
+
+if IS_WINDOWS:
+ import msvcrt
+ from ctypes import byref, windll
+ from ctypes.wintypes import DWORD
+
+ ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
+
+ def enable_virtual_terminal_processing(file_descriptor):
+ handle = msvcrt.get_osfhandle(file_descriptor)
+ try:
+ mode = DWORD()
+ windll.kernel32.GetConsoleMode(handle, byref(mode))
+ mode.value |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
+ windll.kernel32.SetConsoleMode(handle, mode.value)
+ except Exception as e:
+ raise e
+
+
+def enable_blessed():
+ # Only Windows has issues with enabling blessed
+ # and interpreting ANSI escape sequences
+ if not IS_WINDOWS:
+ return True
+
+ if os.environ.get("NO_ANSI"):
+ return False
+
+ # MozillaBuild 4.0.2 is the first Release that supports
+ # ANSI escape sequences, so if we're greater than that
+ # version, we can enable them (via Blessed).
+ return mozilla_build_version() >= Version("4.0.2")
+
+
+# stdout and stderr may not necessarily be set up to write Unicode output, so
+# reconfigure them if necessary.
+def _wrap_stdstream(fh):
+ if fh in (sys.stderr, sys.stdout):
+ encoding = sys.getdefaultencoding()
+ encoding = "utf-8" if encoding in ("ascii", "charmap") else encoding
+ if six.PY2:
+ return codecs.getwriter(encoding)(fh, errors="replace")
+ else:
+ return codecs.getwriter(encoding)(fh.buffer, errors="replace")
+ else:
+ return fh
+
+
+def format_seconds(total):
+ """Format number of seconds to MM:SS.DD form."""
+
+ minutes, seconds = divmod(total, 60)
+
+ return "%2d:%05.2f" % (minutes, seconds)
+
+
+class ConvertToStructuredFilter(logging.Filter):
+ """Filter that converts unstructured records into structured ones."""
+
+ def filter(self, record):
+ if hasattr(record, "action") and hasattr(record, "params"):
+ return True
+
+ record.action = "unstructured"
+ record.params = {"msg": record.getMessage()}
+ record.msg = "{msg}"
+
+ return True
+
+
+class StructuredJSONFormatter(logging.Formatter):
+ """Log formatter that writes a structured JSON entry."""
+
+ def format(self, record):
+ action = getattr(record, "action", "UNKNOWN")
+ params = getattr(record, "params", {})
+
+ return json.dumps([record.created, action, params])
+
+
+class StructuredHumanFormatter(logging.Formatter):
+ """Log formatter that writes structured messages for humans.
+
+ It is important that this formatter never be added to a logger that
+ produces unstructured/classic log messages. If it is, the call to format()
+ could fail because the string could contain things (like JSON) that look
+ like formatting character sequences.
+
+ Because of this limitation, format() will fail with a KeyError if an
+ unstructured record is passed or if the structured message is malformed.
+ """
+
+ def __init__(self, start_time, write_interval=False, write_times=True):
+ self.start_time = start_time
+ self.write_interval = write_interval
+ self.write_times = write_times
+ self.last_time = None
+
+ def format(self, record):
+ formatted_msg = record.msg.format(**getattr(record, "params", {}))
+
+ elapsed_time = (
+ format_seconds(self._time(record)) + " " if self.write_times else ""
+ )
+
+ rv = elapsed_time + formatted_msg
+ formatted_stack_trace_result = formatted_stack_trace(record, self)
+
+ if formatted_stack_trace_result != "":
+ stack_trace = "\n" + elapsed_time + formatted_stack_trace_result
+ rv += stack_trace.replace("\n", f"\n{elapsed_time}")
+
+ return rv
+
+ def _time(self, record):
+ t = record.created - self.start_time
+
+ if self.write_interval and self.last_time is not None:
+ t = record.created - self.last_time
+
+ self.last_time = record.created
+
+ return t
+
+
+class StructuredTerminalFormatter(StructuredHumanFormatter):
+ """Log formatter for structured messages writing to a terminal."""
+
+ def set_terminal(self, terminal):
+ self.terminal = terminal
+ self._sgr0 = terminal.normal if terminal else ""
+
+ def format(self, record):
+ formatted_msg = record.msg.format(**getattr(record, "params", {}))
+ elapsed_time = (
+ self.terminal.blue(format_seconds(self._time(record))) + " "
+ if self.write_times
+ else ""
+ )
+
+ rv = elapsed_time + self._colorize(formatted_msg) + self._sgr0
+ formatted_stack_trace_result = formatted_stack_trace(record, self)
+
+ if formatted_stack_trace_result != "":
+ stack_trace = "\n" + elapsed_time + formatted_stack_trace_result
+ rv += stack_trace.replace("\n", f"\n{elapsed_time}")
+
+ # Some processes (notably Clang) don't reset terminal attributes after
+ # printing newlines. This can lead to terminal attributes getting in a
+ # wonky state. Work around this by sending the sgr0 sequence after every
+ # line to reset all attributes. For programs that rely on the next line
+ # inheriting the same attributes, this will prevent that from happening.
+ # But that's better than "corrupting" the terminal.
+ return rv + self._sgr0
+
+ def _colorize(self, s):
+ if not self.terminal:
+ return s
+
+ result = s
+
+ reftest = s.startswith("REFTEST ")
+ if reftest:
+ s = s[8:]
+
+ if s.startswith("TEST-PASS"):
+ result = self.terminal.green(s[0:9]) + s[9:]
+ elif s.startswith("TEST-UNEXPECTED"):
+ result = self.terminal.red(s[0:20]) + s[20:]
+ elif s.startswith("TEST-START"):
+ result = self.terminal.yellow(s[0:10]) + s[10:]
+ elif s.startswith("TEST-INFO"):
+ result = self.terminal.yellow(s[0:9]) + s[9:]
+
+ if reftest:
+ result = "REFTEST " + result
+
+ return result
+
+
+def formatted_stack_trace(record, formatter):
+ """
+ Formatting behavior here intended to mimic a portion of the
+ standard library's logging.Formatter::format function
+ """
+ rv = ""
+
+ if record.exc_info:
+ # Cache the traceback text to avoid converting it multiple times
+ # (it's constant anyway)
+ if not record.exc_text:
+ record.exc_text = formatter.formatException(record.exc_info)
+ if record.exc_text:
+ rv = record.exc_text
+ if record.stack_info:
+ if rv[-1:] != "\n":
+ rv = rv + "\n"
+ rv = rv + formatter.formatStack(record.stack_info)
+
+ return rv
+
+
+class LoggingManager(object):
+ """Holds and controls global logging state.
+
+ An application should instantiate one of these and configure it as needed.
+
+ This class provides a mechanism to configure the output of logging data
+ both from mach and from the overall logging system (e.g. from other
+ modules).
+ """
+
+ def __init__(self):
+ self.start_time = time.time()
+
+ self.json_handlers = []
+ self.terminal_handler = None
+ self.terminal_formatter = None
+
+ self.root_logger = logging.getLogger()
+ self.root_logger.setLevel(logging.DEBUG)
+
+ # Installing NullHandler on the root logger ensures that *all* log
+ # messages have at least one handler. This prevents Python from
+ # complaining about "no handlers could be found for logger XXX."
+ self.root_logger.addHandler(logging.NullHandler())
+
+ mach_logger = logging.getLogger("mach")
+ mach_logger.setLevel(logging.DEBUG)
+
+ self.structured_filter = ConvertToStructuredFilter()
+
+ self.structured_loggers = [mach_logger]
+
+ self._terminal = None
+
+ def create_terminal(self):
+ if enable_blessed():
+ # Sometimes blessed fails to set up the terminal, in that case, silently fail.
+ try:
+ terminal = blessed.Terminal(stream=_wrap_stdstream(sys.stdout))
+
+ if terminal.is_a_tty:
+ self._terminal = terminal
+ except Exception:
+ pass
+
+ @property
+ def terminal(self):
+ return self._terminal
+
+ def add_json_handler(self, fh):
+ """Enable JSON logging on the specified file object."""
+
+ # Configure the consumer of structured messages.
+ handler = logging.StreamHandler(stream=fh)
+ handler.setFormatter(StructuredJSONFormatter())
+ handler.setLevel(logging.DEBUG)
+
+ # And hook it up.
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.json_handlers.append(handler)
+
+ def add_terminal_logging(
+ self, fh=sys.stdout, level=logging.INFO, write_interval=False, write_times=True
+ ):
+ """Enable logging to the terminal."""
+ self.create_terminal()
+
+ if IS_WINDOWS:
+ try:
+ # fileno() can raise in some cases, like unit tests.
+ # so we can try to enable this but if we fail it's fine
+ enable_virtual_terminal_processing(sys.stdout.fileno())
+ enable_virtual_terminal_processing(sys.stderr.fileno())
+ except Exception:
+ pass
+
+ fh = _wrap_stdstream(fh)
+ formatter = StructuredHumanFormatter(
+ self.start_time, write_interval=write_interval, write_times=write_times
+ )
+
+ if self.terminal:
+ formatter = StructuredTerminalFormatter(
+ self.start_time, write_interval=write_interval, write_times=write_times
+ )
+ formatter.set_terminal(self.terminal)
+
+ handler = logging.StreamHandler(stream=fh)
+ handler.setFormatter(formatter)
+ handler.setLevel(level)
+
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.terminal_handler = handler
+ self.terminal_formatter = formatter
+
+ def replace_terminal_handler(self, handler):
+ """Replace the installed terminal handler.
+
+ Returns the old handler or None if none was configured.
+ If the new handler is None, removes any existing handler and disables
+ logging to the terminal.
+ """
+ old = self.terminal_handler
+
+ if old:
+ for logger in self.structured_loggers:
+ logger.removeHandler(old)
+
+ if handler:
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.terminal_handler = handler
+
+ return old
+
+ def enable_unstructured(self):
+ """Enable logging of unstructured messages."""
+ if self.terminal_handler:
+ self.terminal_handler.addFilter(self.structured_filter)
+ self.root_logger.addHandler(self.terminal_handler)
+
+ def disable_unstructured(self):
+ """Disable logging of unstructured messages."""
+ if self.terminal_handler:
+ self.terminal_handler.removeFilter(self.structured_filter)
+ self.root_logger.removeHandler(self.terminal_handler)
+
+ def register_structured_logger(self, logger, terminal=True, json=True):
+ """Register a structured logger.
+
+ This needs to be called for all structured loggers that don't chain up
+ to the mach logger in order for their output to be captured.
+ """
+ self.structured_loggers.append(logger)
+
+ if terminal and self.terminal_handler:
+ logger.addHandler(self.terminal_handler)
+
+ if json:
+ for handler in self.json_handlers:
+ logger.addHandler(handler)
+
+ def enable_all_structured_loggers(self, terminal=True, json=True):
+ """Enable logging of all structured messages from all loggers.
+
+ ``terminal`` and ``json`` determine which log handlers to operate
+ on. By default, all known handlers are operated on.
+ """
+
+ # Glean makes logs that we're not interested in, so we squelch them.
+ logging.getLogger("glean").setLevel(logging.CRITICAL)
+
+ # Remove current handlers from all loggers so we don't double
+ # register handlers.
+ for logger in self.root_logger.manager.loggerDict.values():
+ # Some entries might be logging.PlaceHolder.
+ if not isinstance(logger, logging.Logger):
+ continue
+
+ if terminal:
+ logger.removeHandler(self.terminal_handler)
+
+ if json:
+ for handler in self.json_handlers:
+ logger.removeHandler(handler)
+
+ # Wipe out existing registered structured loggers since they
+ # all propagate to root logger.
+ self.structured_loggers = []
+ self.register_structured_logger(self.root_logger, terminal=terminal, json=json)
diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py
new file mode 100644
index 0000000000..9ab880341d
--- /dev/null
+++ b/python/mach/mach/main.py
@@ -0,0 +1,735 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+import argparse
+import codecs
+import errno
+import imp
+import logging
+import os
+import sys
+import traceback
+import uuid
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Dict, List, Union
+
+from .base import (
+ CommandContext,
+ FailedCommandError,
+ MachError,
+ MissingFileError,
+ NoCommandError,
+ UnknownCommandError,
+ UnrecognizedArgumentError,
+)
+from .config import ConfigSettings
+from .dispatcher import CommandAction
+from .logging import LoggingManager
+from .registrar import Registrar
+from .sentry import NoopErrorReporter, register_sentry
+from .telemetry import create_telemetry_from_environment, report_invocation_metrics
+from .util import UserError, setenv
+
+SUGGEST_MACH_BUSTED_TEMPLATE = r"""
+You can invoke ``./mach busted`` to check if this issue is already on file. If it
+isn't, please use ``./mach busted file %s`` to report it. If ``./mach busted`` is
+misbehaving, you can also inspect the dependencies of bug 1543241.
+""".lstrip()
+
+MACH_ERROR_TEMPLATE = (
+ r"""
+The error occurred in mach itself. This is likely a bug in mach itself or a
+fundamental problem with a loaded module.
+
+""".lstrip()
+ + SUGGEST_MACH_BUSTED_TEMPLATE
+)
+
+ERROR_FOOTER = r"""
+If filing a bug, please include the full output of mach, including this error
+message.
+
+The details of the failure are as follows:
+""".lstrip()
+
+USER_ERROR = r"""
+This is a user error and does not appear to be a bug in mach.
+""".lstrip()
+
+COMMAND_ERROR_TEMPLATE = (
+ r"""
+The error occurred in the implementation of the invoked mach command.
+
+This should never occur and is likely a bug in the implementation of that
+command.
+""".lstrip()
+ + SUGGEST_MACH_BUSTED_TEMPLATE
+)
+
+MODULE_ERROR_TEMPLATE = (
+ r"""
+The error occurred in code that was called by the mach command. This is either
+a bug in the called code itself or in the way that mach is calling it.
+""".lstrip()
+ + SUGGEST_MACH_BUSTED_TEMPLATE
+)
+
+NO_COMMAND_ERROR = r"""
+It looks like you tried to run mach without a command.
+
+Run ``mach help`` to show a list of commands.
+""".lstrip()
+
+UNKNOWN_COMMAND_ERROR = r"""
+It looks like you are trying to %s an unknown mach command: %s
+%s
+Run ``mach help`` to show a list of commands.
+""".lstrip()
+
+SUGGESTED_COMMANDS_MESSAGE = r"""
+Did you want to %s any of these commands instead: %s?
+"""
+
+UNRECOGNIZED_ARGUMENT_ERROR = r"""
+It looks like you passed an unrecognized argument into mach.
+
+The %s command does not accept the arguments: %s
+""".lstrip()
+
+INVALID_ENTRY_POINT = r"""
+Entry points should return a list of command providers or directories
+containing command providers. The following entry point is invalid:
+
+ %s
+
+You are seeing this because there is an error in an external module attempting
+to implement a mach command. Please fix the error, or uninstall the module from
+your system.
+""".lstrip()
+
+
+class ArgumentParser(argparse.ArgumentParser):
+ """Custom implementation argument parser to make things look pretty."""
+
+ def error(self, message):
+ """Custom error reporter to give more helpful text on bad commands."""
+ if not message.startswith("argument command: invalid choice"):
+ argparse.ArgumentParser.error(self, message)
+ assert False
+
+ print("Invalid command specified. The list of commands is below.\n")
+ self.print_help()
+ sys.exit(1)
+
+ def format_help(self):
+ text = argparse.ArgumentParser.format_help(self)
+
+ # Strip out the silly command list that would preceed the pretty list.
+ #
+ # Commands:
+ # {foo,bar}
+ # foo Do foo.
+ # bar Do bar.
+ search = "Commands:\n {"
+ start = text.find(search)
+
+ if start != -1:
+ end = text.find("}\n", start)
+ assert end != -1
+
+ real_start = start + len("Commands:\n")
+ real_end = end + len("}\n")
+
+ text = text[0:real_start] + text[real_end:]
+
+ return text
+
+
+class ContextWrapper(object):
+ def __init__(self, context, handler):
+ object.__setattr__(self, "_context", context)
+ object.__setattr__(self, "_handler", handler)
+
+ def __getattribute__(self, key):
+ try:
+ return getattr(object.__getattribute__(self, "_context"), key)
+ except AttributeError as e:
+ try:
+ ret = object.__getattribute__(self, "_handler")(key)
+ except (AttributeError, TypeError):
+ # TypeError is in case the handler comes from old code not
+ # taking a key argument.
+ raise e
+ setattr(self, key, ret)
+ return ret
+
+ def __setattr__(self, key, value):
+ setattr(object.__getattribute__(self, "_context"), key, value)
+
+
+class MachCommandReference:
+ """A reference to a mach command.
+
+ Holds the metadata for a mach command.
+ """
+
+ module: Path
+
+ def __init__(self, module: Union[str, Path]):
+ self.module = Path(module)
+
+
+class Mach(object):
+ """Main mach driver type.
+
+ This type is responsible for holding global mach state and dispatching
+ a command from arguments.
+
+ The following attributes may be assigned to the instance to influence
+ behavior:
+
+ populate_context_handler -- If defined, it must be a callable. The
+ callable signature is the following:
+ populate_context_handler(key=None)
+ It acts as a fallback getter for the mach.base.CommandContext
+ instance.
+ This allows to augment the context instance with arbitrary data
+ for use in command handlers.
+
+ require_conditions -- If True, commands that do not have any condition
+ functions applied will be skipped. Defaults to False.
+
+ settings_paths -- A list of files or directories in which to search
+ for settings files to load.
+
+ """
+
+ USAGE = """%(prog)s [global arguments] command [command arguments]
+
+mach (German for "do") is the main interface to the Mozilla build system and
+common developer tasks.
+
+You tell mach the command you want to perform and it does it for you.
+
+Some common commands are:
+
+ %(prog)s build Build/compile the source tree.
+ %(prog)s help Show full help, including the list of all commands.
+
+To see more help for a specific command, run:
+
+ %(prog)s help <command>
+"""
+
+ def __init__(self, cwd: str):
+ assert Path(cwd).is_dir()
+
+ self.cwd = cwd
+ self.log_manager = LoggingManager()
+ self.logger = logging.getLogger(__name__)
+ self.settings = ConfigSettings()
+ self.settings_paths = []
+
+ if "MACHRC" in os.environ:
+ self.settings_paths.append(os.environ["MACHRC"])
+
+ self.log_manager.register_structured_logger(self.logger)
+ self.populate_context_handler = None
+
+ def load_commands_from_directory(self, path: Path):
+ """Scan for mach commands from modules in a directory.
+
+ This takes a path to a directory, loads the .py files in it, and
+ registers and found mach command providers with this mach instance.
+ """
+ for f in sorted(path.iterdir()):
+ if not f.suffix == ".py" or f.name == "__init__.py":
+ continue
+
+ full_path = path / f
+ module_name = f"mach.commands.{str(f)[0:-3]}"
+
+ self.load_commands_from_file(full_path, module_name=module_name)
+
+ def load_commands_from_file(self, path: Union[str, Path], module_name=None):
+ """Scan for mach commands from a file.
+
+ This takes a path to a file and loads it as a Python module under the
+ module name specified. If no name is specified, a random one will be
+ chosen.
+ """
+ if module_name is None:
+ # Ensure parent module is present otherwise we'll (likely) get
+ # an error due to unknown parent.
+ if "mach.commands" not in sys.modules:
+ mod = imp.new_module("mach.commands")
+ sys.modules["mach.commands"] = mod
+
+ module_name = f"mach.commands.{uuid.uuid4().hex}"
+
+ try:
+ imp.load_source(module_name, str(path))
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ raise MissingFileError(f"{path} does not exist")
+
+ def load_commands_from_spec(
+ self, spec: Dict[str, MachCommandReference], topsrcdir: str, missing_ok=False
+ ):
+ """Load mach commands based on the given spec.
+
+ Takes a dictionary mapping command names to their metadata.
+ """
+ modules = set(spec[command].module for command in spec)
+
+ for path in modules:
+ try:
+ self.load_commands_from_file(topsrcdir / path)
+ except MissingFileError:
+ if not missing_ok:
+ raise
+
+ def load_commands_from_entry_point(self, group="mach.providers"):
+ """Scan installed packages for mach command provider entry points. An
+ entry point is a function that returns a list of paths to files or
+ directories containing command providers.
+
+ This takes an optional group argument which specifies the entry point
+ group to use. If not specified, it defaults to 'mach.providers'.
+ """
+ try:
+ import pkg_resources
+ except ImportError:
+ print(
+ "Could not find setuptools, ignoring command entry points",
+ file=sys.stderr,
+ )
+ return
+
+ for entry in pkg_resources.iter_entry_points(group=group, name=None):
+ paths = entry.load()()
+ if not isinstance(paths, Iterable):
+ print(INVALID_ENTRY_POINT % entry)
+ sys.exit(1)
+
+ for path in paths:
+ path = Path(path)
+ if path.is_file():
+ self.load_commands_from_file(path)
+ elif path.is_dir():
+ self.load_commands_from_directory(path)
+ else:
+ print(f"command provider '{path}' does not exist")
+
+ def define_category(self, name, title, description, priority=50):
+ """Provide a description for a named command category."""
+
+ Registrar.register_category(name, title, description, priority)
+
+ @property
+ def require_conditions(self):
+ return Registrar.require_conditions
+
+ @require_conditions.setter
+ def require_conditions(self, value):
+ Registrar.require_conditions = value
+
+ def run(self, argv, stdin=None, stdout=None, stderr=None):
+ """Runs mach with arguments provided from the command line.
+
+ Returns the integer exit code that should be used. 0 means success. All
+ other values indicate failure.
+ """
+ sentry = NoopErrorReporter()
+
+ # If no encoding is defined, we default to UTF-8 because without this
+ # Python 2.7 will assume the default encoding of ASCII. This will blow
+ # up with UnicodeEncodeError as soon as it encounters a non-ASCII
+ # character in a unicode instance. We simply install a wrapper around
+ # the streams and restore once we have finished.
+ stdin = sys.stdin if stdin is None else stdin
+ stdout = sys.stdout if stdout is None else stdout
+ stderr = sys.stderr if stderr is None else stderr
+
+ orig_stdin = sys.stdin
+ orig_stdout = sys.stdout
+ orig_stderr = sys.stderr
+
+ sys.stdin = stdin
+ sys.stdout = stdout
+ sys.stderr = stderr
+
+ orig_env = dict(os.environ)
+
+ try:
+ # Load settings as early as possible so things in dispatcher.py
+ # can use them.
+ for provider in Registrar.settings_providers:
+ self.settings.register_provider(provider)
+
+ setting_paths_to_pass = [Path(path) for path in self.settings_paths]
+ self.load_settings(setting_paths_to_pass)
+
+ if sys.version_info < (3, 0):
+ if stdin.encoding is None:
+ sys.stdin = codecs.getreader("utf-8")(stdin)
+
+ if stdout.encoding is None:
+ sys.stdout = codecs.getwriter("utf-8")(stdout)
+
+ if stderr.encoding is None:
+ sys.stderr = codecs.getwriter("utf-8")(stderr)
+
+ # Allow invoked processes (which may not have a handle on the
+ # original stdout file descriptor) to know if the original stdout
+ # is a TTY. This provides a mechanism to allow said processes to
+ # enable emitting code codes, for example.
+ if os.isatty(orig_stdout.fileno()):
+ setenv("MACH_STDOUT_ISATTY", "1")
+
+ return self._run(argv)
+ except KeyboardInterrupt:
+ print("mach interrupted by signal or user action. Stopping.")
+ return 1
+
+ except Exception:
+ # _run swallows exceptions in invoked handlers and converts them to
+ # a proper exit code. So, the only scenario where we should get an
+ # exception here is if _run itself raises. If _run raises, that's a
+ # bug in mach (or a loaded command module being silly) and thus
+ # should be reported differently.
+ self._print_error_header(argv, sys.stdout)
+ print(MACH_ERROR_TEMPLATE % "general")
+
+ exc_type, exc_value, exc_tb = sys.exc_info()
+ stack = traceback.extract_tb(exc_tb)
+
+ sentry_event_id = sentry.report_exception(exc_value)
+ self._print_exception(
+ sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id
+ )
+
+ return 1
+
+ finally:
+ os.environ.clear()
+ os.environ.update(orig_env)
+
+ sys.stdin = orig_stdin
+ sys.stdout = orig_stdout
+ sys.stderr = orig_stderr
+
+ def _run(self, argv):
+ if self.populate_context_handler:
+ topsrcdir = Path(self.populate_context_handler("topdir"))
+ sentry = register_sentry(argv, self.settings, topsrcdir)
+ else:
+ sentry = NoopErrorReporter()
+
+ context = CommandContext(
+ cwd=self.cwd,
+ settings=self.settings,
+ log_manager=self.log_manager,
+ commands=Registrar,
+ )
+
+ if self.populate_context_handler:
+ context = ContextWrapper(context, self.populate_context_handler)
+
+ parser = self.get_argument_parser(context)
+ context.global_parser = parser
+
+ if not len(argv):
+ # We don't register the usage until here because if it is globally
+ # registered, argparse always prints it. This is not desired when
+ # running with --help.
+ parser.usage = Mach.USAGE
+ parser.print_usage()
+ return 0
+
+ try:
+ args = parser.parse_args(argv)
+ except NoCommandError:
+ print(NO_COMMAND_ERROR)
+ return 1
+ except UnknownCommandError as e:
+ suggestion_message = (
+ SUGGESTED_COMMANDS_MESSAGE % (e.verb, ", ".join(e.suggested_commands))
+ if e.suggested_commands
+ else ""
+ )
+ print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message))
+ return 1
+ except UnrecognizedArgumentError as e:
+ print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, " ".join(e.arguments)))
+ return 1
+
+ if not hasattr(args, "mach_handler"):
+ raise MachError("ArgumentParser result missing mach handler info.")
+
+ context.is_interactive = (
+ args.is_interactive
+ and sys.__stdout__.isatty()
+ and sys.__stderr__.isatty()
+ and not os.environ.get("MOZ_AUTOMATION", None)
+ )
+ context.telemetry = create_telemetry_from_environment(self.settings)
+
+ handler = getattr(args, "mach_handler")
+ report_invocation_metrics(context.telemetry, handler.name)
+
+ # Add JSON logging to a file if requested.
+ if args.logfile:
+ self.log_manager.add_json_handler(args.logfile)
+
+ # Up the logging level if requested.
+ log_level = logging.INFO
+ if args.verbose:
+ log_level = logging.DEBUG
+
+ self.log_manager.register_structured_logger(logging.getLogger("mach"))
+
+ write_times = True
+ if (
+ args.log_no_times
+ or "MACH_NO_WRITE_TIMES" in os.environ
+ or "MOZ_AUTOMATION" in os.environ
+ ):
+ write_times = False
+
+ # Always enable terminal logging. The log manager figures out if we are
+ # actually in a TTY or are a pipe and does the right thing.
+ self.log_manager.add_terminal_logging(
+ level=log_level, write_interval=args.log_interval, write_times=write_times
+ )
+
+ if args.settings_file:
+ # Argument parsing has already happened, so settings that apply
+ # to command line handling (e.g alias, defaults) will be ignored.
+ self.load_settings([Path(args.settings_file)])
+
+ try:
+ return Registrar._run_command_handler(
+ handler,
+ context,
+ debug_command=args.debug_command,
+ profile_command=args.profile_command,
+ **vars(args.command_args),
+ )
+ except KeyboardInterrupt as ki:
+ raise ki
+ except FailedCommandError as e:
+ print(e)
+ return e.exit_code
+ except UserError:
+ # We explicitly don't report UserErrors to Sentry.
+ exc_type, exc_value, exc_tb = sys.exc_info()
+ # The first two frames are us and are never used.
+ stack = traceback.extract_tb(exc_tb)[2:]
+ self._print_error_header(argv, sys.stdout)
+ print(USER_ERROR)
+ self._print_exception(sys.stdout, exc_type, exc_value, stack)
+ return 1
+ except Exception:
+ exc_type, exc_value, exc_tb = sys.exc_info()
+ sentry_event_id = sentry.report_exception(exc_value)
+
+ # The first two frames are us and are never used.
+ stack = traceback.extract_tb(exc_tb)[2:]
+
+ # If we have nothing on the stack, the exception was raised as part
+ # of calling the @Command method itself. This likely means a
+ # mismatch between @CommandArgument and arguments to the method.
+ # e.g. there exists a @CommandArgument without the corresponding
+ # argument on the method. We handle that here until the module
+ # loader grows the ability to validate better.
+ if not len(stack):
+ print(COMMAND_ERROR_TEMPLATE % handler.name)
+ self._print_exception(
+ sys.stdout,
+ exc_type,
+ exc_value,
+ traceback.extract_tb(exc_tb),
+ sentry_event_id=sentry_event_id,
+ )
+ return 1
+
+ # Split the frames into those from the module containing the
+ # command and everything else.
+ command_frames = []
+ other_frames = []
+
+ initial_file = stack[0][0]
+
+ for frame in stack:
+ if frame[0] == initial_file:
+ command_frames.append(frame)
+ else:
+ other_frames.append(frame)
+
+ # If the exception was in the module providing the command, it's
+ # likely the bug is in the mach command module, not something else.
+ # If there are other frames, the bug is likely not the mach
+ # command's fault.
+ self._print_error_header(argv, sys.stdout)
+
+ if len(other_frames):
+ print(MODULE_ERROR_TEMPLATE % handler.name)
+ else:
+ print(COMMAND_ERROR_TEMPLATE % handler.name)
+
+ self._print_exception(
+ sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id
+ )
+
+ return 1
+
+ def log(self, level, action, params, format_str):
+ """Helper method to record a structured log event."""
+ self.logger.log(level, format_str, extra={"action": action, "params": params})
+
+ def _print_error_header(self, argv, fh):
+ fh.write("Error running mach:\n\n")
+ fh.write(" ")
+ fh.write(repr(argv))
+ fh.write("\n\n")
+
+ def _print_exception(self, fh, exc_type, exc_value, stack, sentry_event_id=None):
+ fh.write(ERROR_FOOTER)
+ fh.write("\n")
+
+ for l in traceback.format_exception_only(exc_type, exc_value):
+ fh.write(l)
+
+ fh.write("\n")
+ for l in traceback.format_list(stack):
+ fh.write(l)
+
+ if not sentry_event_id:
+ return
+
+ fh.write("\nSentry event ID: {}\n".format(sentry_event_id))
+
+ def load_settings(self, paths: List[Path]):
+ """Load the specified settings files.
+
+ If a directory is specified, the following basenames will be
+ searched for in this order:
+
+ machrc, .machrc
+ """
+ valid_names = ("machrc", ".machrc")
+
+ def find_in_dir(base: Path):
+ if base.is_file():
+ return base
+
+ for name in valid_names:
+ path = base / name
+ if path.is_file():
+ return path
+
+ files = map(find_in_dir, paths)
+ files = filter(bool, files)
+
+ self.settings.load_files(list(files))
+
+ def get_argument_parser(self, context):
+ """Returns an argument parser for the command-line interface."""
+
+ parser = ArgumentParser(
+ add_help=False,
+ usage="%(prog)s [global arguments] " "command [command arguments]",
+ )
+
+ # WARNING!!! If you add a global argument here, also add it to the
+ # global argument handling in the top-level `mach` script.
+ # Order is important here as it dictates the order the auto-generated
+ # help messages are printed.
+ global_group = parser.add_argument_group("Global Arguments")
+
+ global_group.add_argument(
+ "-v",
+ "--verbose",
+ dest="verbose",
+ action="store_true",
+ default=False,
+ help="Print verbose output.",
+ )
+ global_group.add_argument(
+ "-l",
+ "--log-file",
+ dest="logfile",
+ metavar="FILENAME",
+ type=argparse.FileType("a"),
+ help="Filename to write log data to.",
+ )
+ global_group.add_argument(
+ "--log-interval",
+ dest="log_interval",
+ action="store_true",
+ default=False,
+ help="Prefix log line with interval from last message rather "
+ "than relative time. Note that this is NOT execution time "
+ "if there are parallel operations.",
+ )
+ global_group.add_argument(
+ "--no-interactive",
+ dest="is_interactive",
+ action="store_false",
+ help="Automatically selects the default option on any "
+ "interactive prompts. If the output is not a terminal, "
+ "then --no-interactive is assumed.",
+ )
+ suppress_log_by_default = False
+ if "INSIDE_EMACS" in os.environ:
+ suppress_log_by_default = True
+ global_group.add_argument(
+ "--log-no-times",
+ dest="log_no_times",
+ action="store_true",
+ default=suppress_log_by_default,
+ help="Do not prefix log lines with times. By default, "
+ "mach will prefix each output line with the time since "
+ "command start.",
+ )
+ global_group.add_argument(
+ "-h",
+ "--help",
+ dest="help",
+ action="store_true",
+ default=False,
+ help="Show this help message.",
+ )
+ global_group.add_argument(
+ "--debug-command",
+ action="store_true",
+ help="Start a Python debugger when command is dispatched.",
+ )
+ global_group.add_argument(
+ "--profile-command",
+ action="store_true",
+ help="Capture a Python profile of the mach process as command is dispatched.",
+ )
+ global_group.add_argument(
+ "--settings",
+ dest="settings_file",
+ metavar="FILENAME",
+ default=None,
+ help="Path to settings file.",
+ )
+
+ # We need to be last because CommandAction swallows all remaining
+ # arguments and argparse parses arguments in the order they were added.
+ parser.add_argument(
+ "command", action=CommandAction, registrar=Registrar, context=context
+ )
+
+ return parser
diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mach/mach/mixin/__init__.py
diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py
new file mode 100644
index 0000000000..4ba6955a2d
--- /dev/null
+++ b/python/mach/mach/mixin/logging.py
@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+
+class LoggingMixin(object):
+ """Provides functionality to control logging."""
+
+ def populate_logger(self, name=None):
+ """Ensure this class instance has a logger associated with it.
+
+ Users of this mixin that call log() will need to ensure self._logger is
+ a logging.Logger instance before they call log(). This function ensures
+ self._logger is defined by populating it if it isn't.
+ """
+ if hasattr(self, "_logger"):
+ return
+
+ if name is None:
+ name = ".".join([self.__module__, self.__class__.__name__])
+
+ self._logger = logging.getLogger(name)
+
+ def log(self, level, action, params, format_str):
+ """Log a structured log event.
+
+ A structured log event consists of a logging level, a string action, a
+ dictionary of attributes, and a formatting string.
+
+ The logging level is one of the logging.* constants, such as
+ logging.INFO.
+
+ The action string is essentially the enumeration of the event. Each
+ different type of logged event should have a different action.
+
+ The params dict is the metadata constituting the logged event.
+
+ The formatting string is used to convert the structured message back to
+ human-readable format. Conversion back to human-readable form is
+ performed by calling format() on this string, feeding into it the dict
+ of attributes constituting the event.
+
+ Example Usage:
+
+ .. code-block:: python
+
+ self.log(logging.DEBUG, 'login', {'username': 'johndoe'},
+ 'User login: {username}')
+ """
+ self._logger.log(level, format_str, extra={"action": action, "params": params})
diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py
new file mode 100644
index 0000000000..d5fd733a17
--- /dev/null
+++ b/python/mach/mach/mixin/process.py
@@ -0,0 +1,217 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides mixins to perform process execution.
+
+import logging
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+from mozprocess.processhandler import ProcessHandlerMixin
+
+from .logging import LoggingMixin
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if "SHELL" in os.environ:
+ _current_shell = os.environ["SHELL"]
+elif "MOZILLABUILD" in os.environ:
+ mozillabuild = os.environ["MOZILLABUILD"]
+ if (Path(mozillabuild) / "msys2").exists():
+ _current_shell = mozillabuild + "/msys2/usr/bin/sh.exe"
+ else:
+ _current_shell = mozillabuild + "/msys/bin/sh.exe"
+elif "COMSPEC" in os.environ:
+ _current_shell = os.environ["COMSPEC"]
+elif sys.platform != "win32":
+ # Fall back to a standard shell.
+ _current_shell = "/bin/sh"
+else:
+ raise Exception("Could not detect environment shell!")
+
+_in_msys = False
+
+if (
+ os.environ.get("MSYSTEM", None) in ("MINGW32", "MINGW64")
+ or "MOZILLABUILD" in os.environ
+):
+ _in_msys = True
+
+ if not _current_shell.lower().endswith(".exe"):
+ _current_shell += ".exe"
+
+
+class LineHandlingEarlyReturn(Exception):
+ pass
+
+
+class ProcessExecutionMixin(LoggingMixin):
+ """Mix-in that provides process execution functionality."""
+
+ def run_process(
+ self,
+ args=None,
+ cwd: Optional[str] = None,
+ append_env=None,
+ explicit_env=None,
+ log_name=None,
+ log_level=logging.INFO,
+ line_handler=None,
+ require_unix_environment=False,
+ ensure_exit_code=0,
+ ignore_children=False,
+ pass_thru=False,
+ python_unbuffered=True,
+ ):
+ """Runs a single process to completion.
+
+ Takes a list of arguments to run where the first item is the
+ executable. Runs the command in the specified directory and
+ with optional environment variables.
+
+ append_env -- Dict of environment variables to append to the current
+ set of environment variables.
+ explicit_env -- Dict of environment variables to set for the new
+ process. Any existing environment variables will be ignored.
+
+ require_unix_environment if True will ensure the command is executed
+ within a UNIX environment. Basically, if we are on Windows, it will
+ execute the command via an appropriate UNIX-like shell.
+
+ ignore_children is proxied to mozprocess's ignore_children.
+
+ ensure_exit_code is used to ensure the exit code of a process matches
+ what is expected. If it is an integer, we raise an Exception if the
+ exit code does not match this value. If it is True, we ensure the exit
+ code is 0. If it is False, we don't perform any exit code validation.
+
+ pass_thru is a special execution mode where the child process inherits
+ this process's standard file handles (stdin, stdout, stderr) as well as
+ additional file descriptors. It should be used for interactive processes
+ where buffering from mozprocess could be an issue. pass_thru does not
+ use mozprocess. Therefore, arguments like log_name, line_handler,
+ and ignore_children have no effect.
+
+ When python_unbuffered is set, the PYTHONUNBUFFERED environment variable
+ will be set in the child process. This is normally advantageous (see bug
+ 1627873) but is detrimental in certain circumstances (specifically, we
+ have seen issues when using pass_thru mode to open a Python subshell, as
+ in bug 1628838). This variable should be set to False to avoid bustage
+ in those circumstances.
+ """
+ args = self._normalize_command(args, require_unix_environment)
+
+ self.log(logging.INFO, "new_process", {"args": " ".join(args)}, "{args}")
+
+ def handleLine(line):
+ # Converts str to unicode on Python 2 and bytes to str on Python 3.
+ if isinstance(line, bytes):
+ line = line.decode(sys.stdout.encoding or "utf-8", "replace")
+
+ if line_handler:
+ try:
+ line_handler(line)
+ except LineHandlingEarlyReturn:
+ return
+
+ if line.startswith("BUILDTASK") or not log_name:
+ return
+
+ self.log(log_level, log_name, {"line": line.rstrip()}, "{line}")
+
+ use_env = {}
+ if explicit_env:
+ use_env = explicit_env
+ else:
+ use_env.update(os.environ)
+
+ if append_env:
+ use_env.update(append_env)
+
+ if python_unbuffered:
+ use_env["PYTHONUNBUFFERED"] = "1"
+
+ self.log(logging.DEBUG, "process", {"env": str(use_env)}, "Environment: {env}")
+
+ if pass_thru:
+ proc = subprocess.Popen(args, cwd=cwd, env=use_env, close_fds=False)
+ status = None
+ # Leave it to the subprocess to handle Ctrl+C. If it terminates as
+ # a result of Ctrl+C, proc.wait() will return a status code, and,
+ # we get out of the loop. If it doesn't, like e.g. gdb, we continue
+ # waiting.
+ while status is None:
+ try:
+ status = proc.wait()
+ except KeyboardInterrupt:
+ pass
+ else:
+ p = ProcessHandlerMixin(
+ args,
+ cwd=cwd,
+ env=use_env,
+ processOutputLine=[handleLine],
+ universal_newlines=True,
+ ignore_children=ignore_children,
+ )
+ p.run()
+ p.processOutput()
+ status = None
+ sig = None
+ while status is None:
+ try:
+ if sig is None:
+ status = p.wait()
+ else:
+ status = p.kill(sig=sig)
+ except KeyboardInterrupt:
+ if sig is None:
+ sig = signal.SIGINT
+ elif sig == signal.SIGINT:
+ # If we've already tried SIGINT, escalate.
+ sig = signal.SIGKILL
+
+ if ensure_exit_code is False:
+ return status
+
+ if ensure_exit_code is True:
+ ensure_exit_code = 0
+
+ if status != ensure_exit_code:
+ raise Exception(
+ "Process executed with non-0 exit code %d: %s" % (status, args)
+ )
+
+ return status
+
+ def _normalize_command(self, args, require_unix_environment):
+ """Adjust command arguments to run in the necessary environment.
+
+ This exists mainly to facilitate execution of programs requiring a *NIX
+ shell when running on Windows. The caller specifies whether a shell
+ environment is required. If it is and we are running on Windows but
+ aren't running in the UNIX-like msys environment, then we rewrite the
+ command to execute via a shell.
+ """
+ assert isinstance(args, list) and len(args)
+
+ if not require_unix_environment or not _in_msys:
+ return args
+
+ # Always munge Windows-style into Unix style for the command.
+ prog = args[0].replace("\\", "/")
+
+ # PyMake removes the C: prefix. But, things seem to work here
+ # without it. Not sure what that's about.
+
+ # We run everything through the msys shell. We need to use
+ # '-c' and pass all the arguments as one argument because that is
+ # how sh works.
+ cline = subprocess.list2cmdline([prog] + args[1:])
+ return [_current_shell, "-c", cline]
diff --git a/python/mach/mach/python_lockfile.py b/python/mach/mach/python_lockfile.py
new file mode 100644
index 0000000000..78f201d4ed
--- /dev/null
+++ b/python/mach/mach/python_lockfile.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import toml
+from packaging.requirements import Requirement
+
+
+class PoetryLockfiles:
+ def __init__(
+ self,
+ poetry_lockfile: Path,
+ pip_lockfile: Path,
+ ):
+ self.poetry_lockfile = poetry_lockfile
+ self.pip_lockfile = pip_lockfile
+
+
+class PoetryHandle:
+ def __init__(self, work_dir: Path):
+ self._work_dir = work_dir
+ self._dependencies = {}
+
+ def add_requirement(self, requirement: Requirement):
+ self._dependencies[requirement.name] = str(requirement.specifier)
+
+ def add_requirements_in_file(self, requirements_in: Path):
+ with open(requirements_in) as requirements_in:
+ for line in requirements_in.readlines():
+ if line.startswith("#"):
+ continue
+
+ req = Requirement(line)
+ self.add_requirement(req)
+
+ def reuse_existing_lockfile(self, lockfile_path: Path):
+ """Make minimal number of changes to the lockfile to satisfy new requirements"""
+ shutil.copy(str(lockfile_path), str(self._work_dir / "poetry.lock"))
+
+ def generate_lockfiles(self, do_update):
+ """Generate pip-style lockfiles that satisfy provided requirements
+
+ One lockfile will be made for all mandatory requirements, and then an extra,
+ compatible lockfile will be created for each optional requirement.
+
+ Args:
+ do_update: if True, then implicitly upgrade the versions of transitive
+ dependencies
+ """
+
+ poetry_config = {
+ "name": "poetry-test",
+ "description": "",
+ "version": "0",
+ "authors": [],
+ "dependencies": {"python": "^3.7"},
+ }
+ poetry_config["dependencies"].update(self._dependencies)
+
+ pyproject = {"tool": {"poetry": poetry_config}}
+ with open(self._work_dir / "pyproject.toml", "w") as pyproject_file:
+ toml.dump(pyproject, pyproject_file)
+
+ self._run_poetry(["lock"] + (["--no-update"] if not do_update else []))
+ self._run_poetry(["export", "-o", "requirements.txt"])
+
+ return PoetryLockfiles(
+ self._work_dir / "poetry.lock",
+ self._work_dir / "requirements.txt",
+ )
+
+ def _run_poetry(self, args):
+ subprocess.check_call(
+ [sys.executable, "-m", "poetry"] + args, cwd=self._work_dir
+ )
diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py
new file mode 100644
index 0000000000..75481596f4
--- /dev/null
+++ b/python/mach/mach/registrar.py
@@ -0,0 +1,186 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import time
+from cProfile import Profile
+from pathlib import Path
+
+import six
+
+from .base import MachError
+
+INVALID_COMMAND_CONTEXT = r"""
+It looks like you tried to run a mach command from an invalid context. The %s
+command failed to meet the following conditions: %s
+
+Run |mach help| to show a list of all commands available to the current context.
+""".lstrip()
+
+
+class MachRegistrar(object):
+ """Container for mach command and config providers."""
+
+ def __init__(self):
+ self.command_handlers = {}
+ self.commands_by_category = {}
+ self.settings_providers = set()
+ self.categories = {}
+ self.require_conditions = False
+ self.command_depth = 0
+
+ def register_command_handler(self, handler):
+ name = handler.name
+
+ if not handler.category:
+ raise MachError(
+ "Cannot register a mach command without a " "category: %s" % name
+ )
+
+ if handler.category not in self.categories:
+ raise MachError(
+ "Cannot register a command to an undefined "
+ "category: %s -> %s" % (name, handler.category)
+ )
+
+ self.command_handlers[name] = handler
+ self.commands_by_category[handler.category].add(name)
+
+ def register_settings_provider(self, cls):
+ self.settings_providers.add(cls)
+
+ def register_category(self, name, title, description, priority=50):
+ self.categories[name] = (title, description, priority)
+ self.commands_by_category[name] = set()
+
+ @classmethod
+ def _condition_failed_message(cls, name, conditions):
+ msg = ["\n"]
+ for c in conditions:
+ part = [" %s" % getattr(c, "__name__", c)]
+ if c.__doc__ is not None:
+ part.append(c.__doc__)
+ msg.append(" - ".join(part))
+ return INVALID_COMMAND_CONTEXT % (name, "\n".join(msg))
+
+ @classmethod
+ def _instance(_, handler, context, **kwargs):
+ if context is None:
+ raise ValueError("Expected a non-None context.")
+
+ prerun = getattr(context, "pre_dispatch_handler", None)
+ if prerun:
+ prerun(context, handler, args=kwargs)
+
+ context.handler = handler
+ return handler.create_instance(context, handler.virtualenv_name)
+
+ @classmethod
+ def _fail_conditions(_, handler, instance):
+ fail_conditions = []
+ if handler.conditions:
+ for c in handler.conditions:
+ if not c(instance):
+ fail_conditions.append(c)
+
+ return fail_conditions
+
+ def _run_command_handler(
+ self, handler, context, debug_command=False, profile_command=False, **kwargs
+ ):
+ instance = MachRegistrar._instance(handler, context, **kwargs)
+ fail_conditions = MachRegistrar._fail_conditions(handler, instance)
+ if fail_conditions:
+ print(
+ MachRegistrar._condition_failed_message(handler.name, fail_conditions)
+ )
+ return 1
+
+ self.command_depth += 1
+ fn = handler.func
+ if handler.virtualenv_name:
+ instance.activate_virtualenv()
+
+ profile = None
+ if profile_command:
+ profile = Profile()
+ profile.enable()
+
+ start_time = time.monotonic()
+
+ if debug_command:
+ import pdb
+
+ result = pdb.runcall(fn, instance, **kwargs)
+ else:
+ result = fn(instance, **kwargs)
+
+ end_time = time.monotonic()
+
+ if profile_command:
+ profile.disable()
+ profile_file = (
+ Path(context.topdir) / f"mach_profile_{handler.name}.cProfile"
+ )
+ profile.dump_stats(profile_file)
+ print(
+ f'Mach command profile created at "{profile_file}". To visualize, use '
+ f"snakeviz:"
+ )
+ print("python3 -m pip install snakeviz")
+ print(f"python3 -m snakeviz {profile_file.name}")
+
+ result = result or 0
+ assert isinstance(result, six.integer_types)
+
+ if not debug_command:
+ postrun = getattr(context, "post_dispatch_handler", None)
+ if postrun:
+ postrun(
+ context,
+ handler,
+ instance,
+ not result,
+ start_time,
+ end_time,
+ self.command_depth,
+ args=kwargs,
+ )
+ self.command_depth -= 1
+
+ return result
+
+ def dispatch(self, name, context, argv=None, subcommand=None, **kwargs):
+ """Dispatch/run a command.
+
+ Commands can use this to call other commands.
+ """
+ handler = self.command_handlers[name]
+
+ if subcommand:
+ handler = handler.subcommand_handlers[subcommand]
+
+ if handler.parser:
+ parser = handler.parser
+
+ # save and restore existing defaults so **kwargs don't persist across
+ # subsequent invocations of Registrar.dispatch()
+ old_defaults = parser._defaults.copy()
+ parser.set_defaults(**kwargs)
+ kwargs, unknown = parser.parse_known_args(argv or [])
+ kwargs = vars(kwargs)
+ parser._defaults = old_defaults
+
+ if unknown:
+ if subcommand:
+ name = "{} {}".format(name, subcommand)
+ parser.error(
+ "unrecognized arguments for {}: {}".format(
+ name, ", ".join(["'{}'".format(arg) for arg in unknown])
+ )
+ )
+
+ return self._run_command_handler(handler, context, **kwargs)
+
+
+Registrar = MachRegistrar()
diff --git a/python/mach/mach/requirements.py b/python/mach/mach/requirements.py
new file mode 100644
index 0000000000..d5141e23f6
--- /dev/null
+++ b/python/mach/mach/requirements.py
@@ -0,0 +1,183 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+from pathlib import Path
+
+from packaging.requirements import Requirement
+
+THUNDERBIRD_PYPI_ERROR = """
+Thunderbird requirements definitions cannot include PyPI packages.
+""".strip()
+
+
+class PthSpecifier:
+ def __init__(self, path: str):
+ self.path = path
+
+
+class PypiSpecifier:
+ def __init__(self, requirement):
+ self.requirement = requirement
+
+
+class PypiOptionalSpecifier(PypiSpecifier):
+ def __init__(self, repercussion, requirement):
+ super().__init__(requirement)
+ self.repercussion = repercussion
+
+
+class MachEnvRequirements:
+ """Requirements associated with a "site dependency manifest", as
+ defined in "python/sites/".
+
+ Represents the dependencies of a site. The source files consist
+ of colon-delimited fields. The first field
+ specifies the action. The remaining fields are arguments to that
+ action. The following actions are supported:
+
+ pth -- Adds the path given as argument to "mach.pth" under
+ the virtualenv's site packages directory.
+
+ pypi -- Fetch the package, plus dependencies, from PyPI.
+
+ pypi-optional -- Attempt to install the package and dependencies from PyPI.
+ Continue using the site, even if the package could not be installed.
+
+ packages.txt -- Denotes that the specified path is a child manifest. It
+ will be read and processed as if its contents were concatenated
+ into the manifest being read.
+
+ thunderbird-packages.txt -- Denotes a Thunderbird child manifest.
+ Thunderbird child manifests are only activated when working on Thunderbird,
+ and they can cannot have "pypi" or "pypi-optional" entries.
+ """
+
+ def __init__(self):
+ self.requirements_paths = []
+ self.pth_requirements = []
+ self.pypi_requirements = []
+ self.pypi_optional_requirements = []
+ self.vendored_requirements = []
+
+ def pths_as_absolute(self, topsrcdir: str):
+ return [
+ os.path.normcase(Path(topsrcdir) / pth.path)
+ for pth in (self.pth_requirements + self.vendored_requirements)
+ ]
+
+ @classmethod
+ def from_requirements_definition(
+ cls,
+ topsrcdir: str,
+ is_thunderbird,
+ only_strict_requirements,
+ requirements_definition,
+ ):
+ requirements = cls()
+ _parse_mach_env_requirements(
+ requirements,
+ Path(requirements_definition),
+ Path(topsrcdir),
+ is_thunderbird,
+ only_strict_requirements,
+ )
+ return requirements
+
+
+def _parse_mach_env_requirements(
+ requirements_output,
+ root_requirements_path: Path,
+ topsrcdir: Path,
+ is_thunderbird,
+ only_strict_requirements,
+):
+ def _parse_requirements_line(
+ current_requirements_path: Path, line, line_number, is_thunderbird_packages_txt
+ ):
+ line = line.strip()
+ if not line or line.startswith("#"):
+ return
+
+ action, params = line.rstrip().split(":", maxsplit=1)
+ if action == "pth":
+ path = topsrcdir / params
+ if not path.exists():
+ # In sparse checkouts, not all paths will be populated.
+ return
+
+ requirements_output.pth_requirements.append(PthSpecifier(params))
+ elif action == "vendored":
+ requirements_output.vendored_requirements.append(PthSpecifier(params))
+ elif action == "packages.txt":
+ _parse_requirements_definition_file(
+ topsrcdir / params,
+ is_thunderbird_packages_txt,
+ )
+ elif action == "pypi":
+ if is_thunderbird_packages_txt:
+ raise Exception(THUNDERBIRD_PYPI_ERROR)
+
+ requirements_output.pypi_requirements.append(
+ PypiSpecifier(
+ _parse_package_specifier(params, only_strict_requirements)
+ )
+ )
+ elif action == "pypi-optional":
+ if is_thunderbird_packages_txt:
+ raise Exception(THUNDERBIRD_PYPI_ERROR)
+
+ if len(params.split(":", maxsplit=1)) != 2:
+ raise Exception(
+ "Expected pypi-optional package to have a repercussion "
+ 'description in the format "package:fallback explanation", '
+ 'found "{}"'.format(params)
+ )
+ raw_requirement, repercussion = params.split(":")
+ requirements_output.pypi_optional_requirements.append(
+ PypiOptionalSpecifier(
+ repercussion,
+ _parse_package_specifier(raw_requirement, only_strict_requirements),
+ )
+ )
+ elif action == "thunderbird-packages.txt":
+ if is_thunderbird:
+ _parse_requirements_definition_file(
+ topsrcdir / params, is_thunderbird_packages_txt=True
+ )
+ else:
+ raise Exception("Unknown requirements definition action: %s" % action)
+
+ def _parse_requirements_definition_file(
+ requirements_path: Path, is_thunderbird_packages_txt
+ ):
+ """Parse requirements file into list of requirements"""
+ if not requirements_path.is_file():
+ raise Exception(f'Missing requirements file at "{requirements_path}"')
+
+ requirements_output.requirements_paths.append(str(requirements_path))
+
+ with open(requirements_path, "r") as requirements_file:
+ lines = [line for line in requirements_file]
+
+ for number, line in enumerate(lines, start=1):
+ _parse_requirements_line(
+ requirements_path, line, number, is_thunderbird_packages_txt
+ )
+
+ _parse_requirements_definition_file(root_requirements_path, False)
+
+
+class UnexpectedFlexibleRequirementException(Exception):
+ def __init__(self, raw_requirement):
+ self.raw_requirement = raw_requirement
+
+
+def _parse_package_specifier(raw_requirement, only_strict_requirements):
+ requirement = Requirement(raw_requirement)
+
+ if only_strict_requirements and [
+ s for s in requirement.specifier if s.operator != "=="
+ ]:
+ raise UnexpectedFlexibleRequirementException(raw_requirement)
+ return requirement
diff --git a/python/mach/mach/sentry.py b/python/mach/mach/sentry.py
new file mode 100644
index 0000000000..5008f8a40c
--- /dev/null
+++ b/python/mach/mach/sentry.py
@@ -0,0 +1,222 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import abc
+import re
+from pathlib import Path
+from threading import Thread
+
+import sentry_sdk
+from mozversioncontrol import (
+ InvalidRepoPath,
+ MissingUpstreamRepo,
+ MissingVCSTool,
+ get_repository_object,
+)
+from six import string_types
+
+from mach.telemetry import is_telemetry_enabled
+from mach.util import get_state_dir
+
+# https://sentry.io/organizations/mozilla/projects/mach/
+_SENTRY_DSN = (
+ "https://5cfe351fb3a24e8d82c751252b48722b@o1069899.ingest.sentry.io/6250014"
+)
+
+
+class ErrorReporter(object):
+ @abc.abstractmethod
+ def report_exception(self, exception):
+ """Report the exception to remote error-tracking software."""
+
+
+class SentryErrorReporter(ErrorReporter):
+ """Reports errors using Sentry."""
+
+ def report_exception(self, exception):
+ return sentry_sdk.capture_exception(exception)
+
+
+class NoopErrorReporter(ErrorReporter):
+ """Drops errors instead of reporting them.
+
+ This is useful in cases where error-reporting is specifically disabled, such as
+ when telemetry hasn't been allowed.
+ """
+
+ def report_exception(self, exception):
+ return None
+
+
+def register_sentry(argv, settings, topsrcdir: Path):
+ if not is_telemetry_enabled(settings):
+ return NoopErrorReporter()
+
+ global _is_unmodified_mach_core_thread
+ _is_unmodified_mach_core_thread = Thread(
+ target=_is_unmodified_mach_core,
+ args=[topsrcdir],
+ daemon=True,
+ )
+ _is_unmodified_mach_core_thread.start()
+
+ sentry_sdk.init(
+ _SENTRY_DSN, before_send=lambda event, _: _process_event(event, topsrcdir)
+ )
+ sentry_sdk.add_breadcrumb(message="./mach {}".format(" ".join(argv)))
+ return SentryErrorReporter()
+
+
+def _process_event(sentry_event, topsrcdir: Path):
+ # Returning nothing causes the event to be dropped:
+ # https://docs.sentry.io/platforms/python/configuration/filtering/#using-beforesend
+ repo = _get_repository_object(topsrcdir)
+ if repo is None:
+ # We don't know the repo state, so we don't know if mach files are
+ # unmodified.
+ return
+
+ base_ref = repo.base_ref_as_hg()
+ if not base_ref:
+ # If we don't know which revision this exception is attached to, then it's
+ # not worth sending
+ return
+
+ _is_unmodified_mach_core_thread.join()
+ if not _is_unmodified_mach_core_result:
+ return
+
+ for map_fn in (_settle_mach_module_id, _patch_absolute_paths, _delete_server_name):
+ sentry_event = map_fn(sentry_event, topsrcdir)
+
+ sentry_event["release"] = "hg-rev-{}".format(base_ref)
+ return sentry_event
+
+
+def _settle_mach_module_id(sentry_event, _):
+ # Sentry groups issues according to the stack frames and their associated
+ # "module" properties. However, one of the modules is being reported
+ # like "mach.commands.26a828ef5164403eaff4305ab4cb0fab" (with a generated id).
+ # This function replaces that generated id with the static string "<generated>"
+ # so that grouping behaves as expected
+
+ stacktrace_frames = sentry_event["exception"]["values"][0]["stacktrace"]["frames"]
+ for frame in stacktrace_frames:
+ module = frame.get("module")
+ if not module:
+ continue
+
+ module = re.sub(
+ "mach\\.commands\\.[a-f0-9]{32}", "mach.commands.<generated>", module
+ )
+ frame["module"] = module
+ return sentry_event
+
+
+def _patch_absolute_paths(sentry_event, topsrcdir: Path):
+ # As discussed here (https://bugzilla.mozilla.org/show_bug.cgi?id=1636251#c28),
+ # we remove usernames from file names with a best-effort basis. The most likely
+ # place for usernames to manifest in Sentry information is within absolute paths,
+ # such as: "/home/mitch/dev/firefox/mach"
+ # We replace the state_dir, obj_dir, src_dir with "<...>" placeholders.
+ # Note that we also do a blanket find-and-replace of the user's name with "<user>",
+ # which may have ill effects if the user's name is, by happenstance, a substring
+ # of some other value within the Sentry event.
+ def recursive_patch(value, needle, replacement):
+ if isinstance(value, list):
+ return [recursive_patch(v, needle, replacement) for v in value]
+ elif isinstance(value, dict):
+ for key in list(value.keys()):
+ next_value = value.pop(key)
+ key = needle.sub(replacement, key)
+ value[key] = recursive_patch(next_value, needle, replacement)
+ return value
+ elif isinstance(value, string_types):
+ return needle.sub(replacement, value)
+ else:
+ return value
+
+ for (target_path, replacement) in (
+ (get_state_dir(), "<statedir>"),
+ (str(topsrcdir), "<topsrcdir>"),
+ (str(Path.home()), "~"),
+ ):
+ # Sentry converts "vars" to their "representations". When paths are in local
+ # variables on Windows, "C:\Users\MozillaUser\Desktop" becomes
+ # "'C:\\Users\\MozillaUser\\Desktop'". To still catch this case, we "repr"
+ # the home directory and scrub the beginning and end quotes, then
+ # find-and-replace on that.
+ repr_path = repr(target_path)[1:-1]
+
+ for target in (target_path, repr_path):
+ # Paths in the Sentry event aren't consistent:
+ # * On *nix, they're mostly forward slashes.
+ # * On *nix, not all absolute paths start with a leading forward slash.
+ # * On Windows, they're mostly backslashes.
+ # * On Windows, `.extra."sys.argv"` uses forward slashes.
+ # * The Python variables in-scope captured by the Sentry report may be
+ # inconsistent, even for a single path. For example, on
+ # Windows, Mach calculates the state_dir as "C:\Users\<user>/.mozbuild".
+
+ # Handle the case where not all absolute paths start with a leading
+ # forward slash: make the initial slash optional in the search string.
+ if target.startswith("/"):
+ target = "/?" + target[1:]
+
+ # Handle all possible slash variants: our search string should match
+ # both forward slashes and backslashes. This is done by dynamically
+ # replacing each "/" and "\" with the regex "[\/\\]" (match both).
+ slash_regex = re.compile(r"[\/\\]")
+ # The regex module parses string backslash escapes before compiling the
+ # regex, so we need to add more backslashes:
+ # "[\\/\\\\]" => [\/\\] => match "/" and "\"
+ target = slash_regex.sub(r"[\\/\\\\]", target)
+
+ # Compile the regex and patch the event.
+ needle_regex = re.compile(target, re.IGNORECASE)
+ sentry_event = recursive_patch(sentry_event, needle_regex, replacement)
+ return sentry_event
+
+
+def _delete_server_name(sentry_event, _):
+ sentry_event.pop("server_name")
+ return sentry_event
+
+
+def _get_repository_object(topsrcdir: Path):
+ try:
+ return get_repository_object(str(topsrcdir))
+ except (InvalidRepoPath, MissingVCSTool):
+ return None
+
+
+def _is_unmodified_mach_core(topsrcdir: Path):
+ """True if mach is unmodified compared to the public tree.
+
+ To avoid submitting Sentry events for errors caused by user's
+ local changes, we attempt to detect if mach (or code affecting mach)
+ has been modified in the user's local state:
+ * In a revision off of a "ancestor to central" revision, or:
+ * In the working, uncommitted state.
+
+ If "$topsrcdir/mach" and "*.py" haven't been touched, then we can be
+ pretty confident that the Mach behaviour that caused the exception
+ also exists in the public tree.
+ """
+ global _is_unmodified_mach_core_result
+
+ repo = _get_repository_object(topsrcdir)
+ try:
+ files = set(repo.get_outgoing_files()) | set(repo.get_changed_files())
+ _is_unmodified_mach_core_result = not any(
+ [file for file in files if file == "mach" or file.endswith(".py")]
+ )
+ except MissingUpstreamRepo:
+ # If we don't know the upstream state, we don't know if the mach files
+ # have been unmodified.
+ _is_unmodified_mach_core_result = False
+
+
+_is_unmodified_mach_core_result = None
+_is_unmodified_mach_core_thread = None
diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py
new file mode 100644
index 0000000000..58c1eac3fa
--- /dev/null
+++ b/python/mach/mach/site.py
@@ -0,0 +1,1405 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains code for managing the Python import scope for Mach. This
+# generally involves populating a Python virtualenv.
+
+import ast
+import enum
+import functools
+import json
+import os
+import platform
+import shutil
+import site
+import subprocess
+import sys
+import sysconfig
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Callable, Optional
+
+from mach.requirements import (
+ MachEnvRequirements,
+ UnexpectedFlexibleRequirementException,
+)
+
+PTH_FILENAME = "mach.pth"
+METADATA_FILENAME = "moz_virtualenv_metadata.json"
+# The following virtualenvs *may* be used in a context where they aren't allowed to
+# install pip packages over the network. In such a case, they must access unvendored
+# python packages via the system environment.
+PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS = ("mach", "build", "common")
+
+_is_windows = sys.platform == "cygwin" or (sys.platform == "win32" and os.sep == "\\")
+
+
+class VenvModuleNotFoundException(Exception):
+ def __init__(self):
+ msg = (
+ 'Mach was unable to find the "venv" module, which is needed '
+ "to create virtual environments in Python. You may need to "
+ "install it manually using the package manager for your system."
+ )
+ super(Exception, self).__init__(msg)
+
+
+class VirtualenvOutOfDateException(Exception):
+ pass
+
+
+class MozSiteMetadataOutOfDateError(Exception):
+ pass
+
+
+class InstallPipRequirementsException(Exception):
+ pass
+
+
+class SiteUpToDateResult:
+ def __init__(self, is_up_to_date, reason=None):
+ self.is_up_to_date = is_up_to_date
+ self.reason = reason
+
+
+class SitePackagesSource(enum.Enum):
+ NONE = "none"
+ SYSTEM = "system"
+ VENV = "pip"
+
+ @classmethod
+ def for_mach(cls):
+ source = os.environ.get("MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE", "").lower()
+ if source == "system":
+ source = SitePackagesSource.SYSTEM
+ elif source == "none":
+ source = SitePackagesSource.NONE
+ elif source == "pip":
+ source = SitePackagesSource.VENV
+ elif source:
+ raise Exception(
+ "Unexpected MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE value, expected one "
+ 'of "system", "pip", "none", or to not be set'
+ )
+
+ mach_use_system_python = bool(os.environ.get("MACH_USE_SYSTEM_PYTHON"))
+ if source:
+ if mach_use_system_python:
+ raise Exception(
+ "The MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE environment variable is "
+ "set, so the MACH_USE_SYSTEM_PYTHON variable is redundant and "
+ "should be unset."
+ )
+ return source
+
+ # Only print this warning once for the Mach site, so we don't spam it every
+ # time a site handle is created.
+ if mach_use_system_python:
+ print(
+ 'The "MACH_USE_SYSTEM_PYTHON" environment variable is deprecated, '
+ "please unset it or replace it with either "
+ '"MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE=system" or '
+ '"MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE=none"'
+ )
+
+ return (
+ SitePackagesSource.NONE
+ if (mach_use_system_python or os.environ.get("MOZ_AUTOMATION"))
+ else SitePackagesSource.VENV
+ )
+
+
+class MozSiteMetadata:
+ """Details about a Moz-managed python site
+
+ When a Moz-managed site is active, its associated metadata is available
+ at "MozSiteMetadata.current".
+
+ Sites that have associated virtualenvs (so, those that aren't strictly leaning on
+ the external python packages) will have their metadata written to
+ <prefix>/moz_virtualenv_metadata.json.
+ """
+
+ # Used to track which which virtualenv has been activated in-process.
+ current: Optional["MozSiteMetadata"] = None
+
+ def __init__(
+ self,
+ hex_version: int,
+ site_name: str,
+ mach_site_packages_source: SitePackagesSource,
+ original_python: "ExternalPythonSite",
+ prefix: str,
+ ):
+ """
+ Args:
+ hex_version: The python version number from sys.hexversion
+ site_name: The name of the site this metadata is associated with
+ site_packages_source: Where this site imports its
+ pip-installed dependencies from
+ mach_site_packages_source: Where the Mach site imports
+ its pip-installed dependencies from
+ original_python: The external Python site that was
+ used to invoke Mach. Usually the system Python, such as /usr/bin/python3
+ prefix: The same value as "sys.prefix" is when running within the
+ associated Python site. The same thing as the "virtualenv root".
+ """
+
+ self.hex_version = hex_version
+ self.site_name = site_name
+ self.mach_site_packages_source = mach_site_packages_source
+ # original_python is needed for commands that tweak the system, such
+ # as "./mach install-moz-phab".
+ self.original_python = original_python
+ self.prefix = prefix
+
+ def write(self, is_finalized):
+ raw = {
+ "hex_version": self.hex_version,
+ "virtualenv_name": self.site_name,
+ "mach_site_packages_source": self.mach_site_packages_source.name,
+ "original_python_executable": self.original_python.python_path,
+ "is_finalized": is_finalized,
+ }
+ with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file:
+ json.dump(raw, file)
+
+ def __eq__(self, other):
+ return (
+ type(self) == type(other)
+ and self.hex_version == other.hex_version
+ and self.site_name == other.site_name
+ and self.mach_site_packages_source == other.mach_site_packages_source
+ # On Windows, execution environment can lead to different cases. Normalize.
+ and Path(self.original_python.python_path)
+ == Path(other.original_python.python_path)
+ )
+
+ @classmethod
+ def from_runtime(cls):
+ if cls.current:
+ return cls.current
+
+ return cls.from_path(sys.prefix)
+
+ @classmethod
+ def from_path(cls, prefix):
+ metadata_path = os.path.join(prefix, METADATA_FILENAME)
+ out_of_date_exception = MozSiteMetadataOutOfDateError(
+ f'The virtualenv at "{prefix}" is out-of-date.'
+ )
+ try:
+ with open(metadata_path, "r") as file:
+ raw = json.load(file)
+
+ if not raw.get("is_finalized", False):
+ raise out_of_date_exception
+
+ return cls(
+ raw["hex_version"],
+ raw["virtualenv_name"],
+ SitePackagesSource[raw["mach_site_packages_source"]],
+ ExternalPythonSite(raw["original_python_executable"]),
+ metadata_path,
+ )
+ except FileNotFoundError:
+ return None
+ except KeyError:
+ raise out_of_date_exception
+
+ @contextmanager
+ def update_current_site(self, executable):
+ """Updates necessary global state when a site is activated
+
+ Due to needing to fetch some state before the actual activation happens, this
+ is represented as a context manager and should be used as follows:
+
+ with metadata.update_current_site(executable):
+ # Perform the actual implementation of changing the site, whether that is
+ # by exec-ing "activate_this.py" in a virtualenv, modifying the sys.path
+ # directly, or some other means
+ ...
+ """
+
+ try:
+ import pkg_resources
+ except ModuleNotFoundError:
+ pkg_resources = None
+
+ yield
+ MozSiteMetadata.current = self
+
+ sys.executable = executable
+
+ if pkg_resources:
+ # Rebuild the working_set based on the new sys.path.
+ pkg_resources._initialize_master_working_set()
+
+
+class MachSiteManager:
+ """Represents the activate-able "import scope" Mach needs
+
+ Whether running independently, using the system packages, or automatically managing
+ dependencies with "pip install", this class provides an easy handle to verify
+ that the "site" is up-to-date (whether than means that system packages don't
+ collide with vendored packages, or that the on-disk virtualenv needs rebuilding).
+
+ Note that, this is a *virtual* site: an on-disk Python virtualenv
+ is only created if there will be "pip installs" into the Mach site.
+ """
+
+ def __init__(
+ self,
+ topsrcdir: str,
+ virtualenv_root: Optional[str],
+ requirements: MachEnvRequirements,
+ original_python: "ExternalPythonSite",
+ site_packages_source: SitePackagesSource,
+ ):
+ """
+ Args:
+ topsrcdir: The path to the Firefox repo
+ virtualenv_root: The path to the the associated Mach virtualenv,
+ if any
+ requirements: The requirements associated with the Mach site, parsed from
+ the file at python/sites/mach.txt
+ original_python: The external Python site that was used to invoke Mach.
+ If Mach invocations are nested, then "original_python" refers to
+ Python site that was used to start Mach first.
+ Usually the system Python, such as /usr/bin/python3.
+ site_packages_source: Where the Mach site will import its pip-installed
+ dependencies from
+ """
+ self._topsrcdir = topsrcdir
+ self._site_packages_source = site_packages_source
+ self._requirements = requirements
+ self._virtualenv_root = virtualenv_root
+ self._metadata = MozSiteMetadata(
+ sys.hexversion,
+ "mach",
+ site_packages_source,
+ original_python,
+ self._virtualenv_root,
+ )
+
+ @classmethod
+ def from_environment(cls, topsrcdir: str, get_state_dir: Callable[[], str]):
+ """
+ Args:
+ topsrcdir: The path to the Firefox repo
+ get_state_dir: A function that resolves the path to the checkout-scoped
+ state_dir, generally ~/.mozbuild/srcdirs/<checkout-based-dir>/
+ """
+
+ requirements = resolve_requirements(topsrcdir, "mach")
+ # Mach needs to operate in environments in which no pip packages are installed
+ # yet, and the system isn't guaranteed to have the packages we need. For example,
+ # "./mach bootstrap" can't have any dependencies.
+ # So, all external dependencies of Mach's must be optional.
+ assert (
+ not requirements.pypi_requirements
+ ), "Mach pip package requirements must be optional."
+
+ # external_python is the Python interpreter that invoked Mach for this process.
+ external_python = ExternalPythonSite(sys.executable)
+
+ # original_python is the first Python interpreter that invoked the top-level
+ # Mach process. This is different from "external_python" when there's nested
+ # Mach invocations.
+ active_metadata = MozSiteMetadata.from_runtime()
+ if active_metadata:
+ original_python = active_metadata.original_python
+ else:
+ original_python = external_python
+
+ source = SitePackagesSource.for_mach()
+ virtualenv_root = (
+ _mach_virtualenv_root(get_state_dir())
+ if source == SitePackagesSource.VENV
+ else None
+ )
+ return cls(
+ topsrcdir,
+ virtualenv_root,
+ requirements,
+ original_python,
+ source,
+ )
+
+ def _up_to_date(self):
+ if self._site_packages_source == SitePackagesSource.NONE:
+ return SiteUpToDateResult(True)
+ elif self._site_packages_source == SitePackagesSource.SYSTEM:
+ _assert_pip_check(self._sys_path(), "mach", self._requirements)
+ return SiteUpToDateResult(True)
+ elif self._site_packages_source == SitePackagesSource.VENV:
+ environment = self._virtualenv()
+ return _is_venv_up_to_date(
+ environment,
+ self._pthfile_lines(environment),
+ self._requirements,
+ self._metadata,
+ )
+
+ def ensure(self, *, force=False):
+ result = self._up_to_date()
+ if force or not result.is_up_to_date:
+ if Path(sys.prefix) == Path(self._metadata.prefix):
+ # If the Mach virtualenv is already activated, then the changes caused
+ # by rebuilding the virtualenv won't take effect until the next time
+ # Mach is used, which can lead to confusing one-off errors.
+ # Instead, request that the user resolve the out-of-date situation,
+ # *then* come back and run the intended command.
+ raise VirtualenvOutOfDateException(result.reason)
+ self._build()
+
+ def attempt_populate_optional_packages(self):
+ if self._site_packages_source != SitePackagesSource.VENV:
+ pass
+
+ self._virtualenv().install_optional_packages(
+ self._requirements.pypi_optional_requirements
+ )
+
+ def activate(self):
+ assert not MozSiteMetadata.current
+
+ self.ensure()
+ with self._metadata.update_current_site(
+ self._virtualenv().python_path
+ if self._site_packages_source == SitePackagesSource.VENV
+ else sys.executable,
+ ):
+ # Reset the sys.path to insulate ourselves from the environment.
+ # This should be safe to do, since activation of the Mach site happens so
+ # early in the Mach lifecycle that no packages should have been imported
+ # from external sources yet.
+ sys.path = self._sys_path()
+ if self._site_packages_source == SitePackagesSource.VENV:
+ # Activate the Mach virtualenv in the current Python context. This
+ # automatically adds the virtualenv's "site-packages" to our scope, in
+ # addition to our first-party/vendored modules since they're specified
+ # in the "mach.pth" file.
+ activate_virtualenv(self._virtualenv())
+
+ def _build(self):
+ if self._site_packages_source != SitePackagesSource.VENV:
+ # The Mach virtualenv doesn't have a physical virtualenv on-disk if it won't
+ # be "pip install"-ing. So, there's no build work to do.
+ return
+
+ environment = self._virtualenv()
+ _create_venv_with_pthfile(
+ environment,
+ self._pthfile_lines(environment),
+ True,
+ self._requirements,
+ self._metadata,
+ )
+
+ def _sys_path(self):
+ if self._site_packages_source == SitePackagesSource.SYSTEM:
+ stdlib_paths, system_site_paths = self._metadata.original_python.sys_path()
+ return [
+ *stdlib_paths,
+ *self._requirements.pths_as_absolute(self._topsrcdir),
+ *system_site_paths,
+ ]
+ elif self._site_packages_source == SitePackagesSource.NONE:
+ stdlib_paths = self._metadata.original_python.sys_path_stdlib()
+ return [
+ *stdlib_paths,
+ *self._requirements.pths_as_absolute(self._topsrcdir),
+ ]
+ elif self._site_packages_source == SitePackagesSource.VENV:
+ stdlib_paths = self._metadata.original_python.sys_path_stdlib()
+ return [
+ *stdlib_paths,
+ # self._requirements will be added as part of the virtualenv activation.
+ ]
+
+ def _pthfile_lines(self, environment):
+ return [
+ # Prioritize vendored and first-party modules first.
+ *self._requirements.pths_as_absolute(self._topsrcdir),
+ # Then, include the virtualenv's site-packages.
+ *_deprioritize_venv_packages(
+ environment, self._site_packages_source == SitePackagesSource.VENV
+ ),
+ ]
+
+ def _virtualenv(self):
+ assert self._site_packages_source == SitePackagesSource.VENV
+ return PythonVirtualenv(self._metadata.prefix)
+
+
+class CommandSiteManager:
+ """Activate sites and ad-hoc-install pip packages
+
+ Provides tools to ensure that a command's scope will have expected, compatible
+ packages. Manages prioritization of the import scope, and ensures consistency
+ regardless of how a virtualenv is used (whether via in-process activation, or when
+ used standalone to invoke a script).
+
+ A few notes:
+
+ * The command environment always inherits Mach's import scope. This is
+ because "unloading" packages in Python is error-prone, so in-process activations
+ will always carry Mach's dependencies along with it. Accordingly, compatibility
+ between each command environment and the Mach environment must be maintained
+
+ * Unlike the Mach environment, command environments *always* have an associated
+ physical virtualenv on-disk. This is because some commands invoke child Python
+ processes, and that child process should have the same import scope.
+
+ """
+
+ def __init__(
+ self,
+ topsrcdir: str,
+ mach_virtualenv_root: Optional[str],
+ virtualenv_root: str,
+ site_name: str,
+ active_metadata: MozSiteMetadata,
+ populate_virtualenv: bool,
+ requirements: MachEnvRequirements,
+ ):
+ """
+ Args:
+ topsrcdir: The path to the Firefox repo
+ mach_virtualenv_root: The path to the Mach virtualenv, if any
+ virtualenv_root: The path to the virtualenv associated with this site
+ site_name: The name of this site, such as "build"
+ active_metadata: The currently-active moz-managed site
+ populate_virtualenv: True if packages should be installed to the on-disk
+ virtualenv with "pip". False if the virtualenv should only include
+ sys.path modifications, and all 3rd-party packages should be imported from
+ Mach's site packages source.
+ requirements: The requirements associated with this site, parsed from
+ the file at python/sites/<site_name>.txt
+ """
+ self._topsrcdir = topsrcdir
+ self._mach_virtualenv_root = mach_virtualenv_root
+ self.virtualenv_root = virtualenv_root
+ self._site_name = site_name
+ self._virtualenv = PythonVirtualenv(self.virtualenv_root)
+ self.python_path = self._virtualenv.python_path
+ self.bin_path = self._virtualenv.bin_path
+ self._populate_virtualenv = populate_virtualenv
+ self._mach_site_packages_source = active_metadata.mach_site_packages_source
+ self._requirements = requirements
+ self._metadata = MozSiteMetadata(
+ sys.hexversion,
+ site_name,
+ active_metadata.mach_site_packages_source,
+ active_metadata.original_python,
+ virtualenv_root,
+ )
+
+ @classmethod
+ def from_environment(
+ cls,
+ topsrcdir: str,
+ get_state_dir: Callable[[], Optional[str]],
+ site_name: str,
+ command_virtualenvs_dir: str,
+ ):
+ """
+ Args:
+ topsrcdir: The path to the Firefox repo
+ get_state_dir: A function that resolves the path to the checkout-scoped
+ state_dir, generally ~/.mozbuild/srcdirs/<checkout-based-dir>/
+ site_name: The name of this site, such as "build"
+ command_virtualenvs_dir: The location under which this site's virtualenv
+ should be created
+ """
+ active_metadata = MozSiteMetadata.from_runtime()
+ assert (
+ active_metadata
+ ), "A Mach-managed site must be active before doing work with command sites"
+
+ mach_site_packages_source = active_metadata.mach_site_packages_source
+ pip_restricted_site = site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS
+ if (
+ not pip_restricted_site
+ and mach_site_packages_source == SitePackagesSource.SYSTEM
+ ):
+ # Sites that aren't pip-network-install-restricted are likely going to be
+ # incompatible with the system. Besides, this use case shouldn't exist, since
+ # using the system packages is supposed to only be needed to lower risk of
+ # important processes like building Firefox.
+ raise Exception(
+ 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any '
+ f"sites other than {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS}. The "
+ f'current attempted site is "{site_name}".'
+ )
+
+ mach_virtualenv_root = (
+ _mach_virtualenv_root(get_state_dir())
+ if mach_site_packages_source == SitePackagesSource.VENV
+ else None
+ )
+ populate_virtualenv = (
+ mach_site_packages_source == SitePackagesSource.VENV
+ or not pip_restricted_site
+ )
+ return cls(
+ topsrcdir,
+ mach_virtualenv_root,
+ os.path.join(command_virtualenvs_dir, site_name),
+ site_name,
+ active_metadata,
+ populate_virtualenv,
+ resolve_requirements(topsrcdir, site_name),
+ )
+
+ def ensure(self):
+ """Ensure that this virtualenv is built, up-to-date, and ready for use
+ If using a virtualenv Python binary directly, it's useful to call this function
+ first to ensure that the virtualenv doesn't have obsolete references or packages.
+ """
+ result = self._up_to_date()
+ if not result.is_up_to_date:
+ print(f"Site not up-to-date reason: {result.reason}")
+ active_site = MozSiteMetadata.from_runtime()
+ if active_site.site_name == self._site_name:
+ print(result.reason, file=sys.stderr)
+ raise Exception(
+ f'The "{self._site_name}" site is out-of-date, even though it has '
+ f"already been activated. Was it modified while this Mach process "
+ f"was running?"
+ )
+
+ _create_venv_with_pthfile(
+ self._virtualenv,
+ self._pthfile_lines(),
+ self._populate_virtualenv,
+ self._requirements,
+ self._metadata,
+ )
+
+ def activate(self):
+ """Activate this site in the current Python context.
+
+ If you run a random Python script and wish to "activate" the
+ site, you can simply instantiate an instance of this class
+ and call .activate() to make the virtualenv active.
+ """
+
+ active_site = MozSiteMetadata.from_runtime()
+ site_is_already_active = active_site.site_name == self._site_name
+ if (
+ active_site.site_name not in ("mach", "common")
+ and not site_is_already_active
+ ):
+ raise Exception(
+ f'Activating from one command site ("{active_site.site_name}") to '
+ f'another ("{self._site_name}") is not allowed, because they may '
+ "be incompatible."
+ )
+
+ self.ensure()
+
+ if site_is_already_active:
+ return
+
+ with self._metadata.update_current_site(self._virtualenv.python_path):
+ activate_virtualenv(self._virtualenv)
+
+ def install_pip_package(self, package):
+ """Install a package via pip.
+
+ The supplied package is specified using a pip requirement specifier.
+ e.g. 'foo' or 'foo==1.0'.
+
+ If the package is already installed, this is a no-op.
+ """
+ if Path(sys.prefix) == Path(self.virtualenv_root):
+ # If we're already running in this interpreter, we can optimize in
+ # the case that the package requirement is already satisfied.
+ from pip._internal.req.constructors import install_req_from_line
+
+ req = install_req_from_line(package)
+ req.check_if_exists(use_user_site=False)
+ if req.satisfied_by is not None:
+ return
+
+ self._virtualenv.pip_install_with_constraints([package])
+
+ def install_pip_requirements(self, path, require_hashes=True, quiet=False):
+ """Install a pip requirements.txt file.
+
+ The supplied path is a text file containing pip requirement
+ specifiers.
+
+ If require_hashes is True, each specifier must contain the
+ expected hash of the downloaded package. See:
+ https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode
+ """
+
+ if not os.path.isabs(path):
+ path = os.path.join(self._topsrcdir, path)
+
+ args = ["--requirement", path]
+
+ if require_hashes:
+ args.append("--require-hashes")
+
+ install_result = self._virtualenv.pip_install(
+ args,
+ check=not quiet,
+ stdout=subprocess.PIPE if quiet else None,
+ )
+ if install_result.returncode:
+ print(install_result.stdout)
+ raise InstallPipRequirementsException(
+ f'Failed to install "{path}" into the "{self._site_name}" site.'
+ )
+
+ check_result = subprocess.run(
+ [self.python_path, "-m", "pip", "check"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+
+ if not check_result.returncode:
+ return
+
+ """
+ Some commands may use the "setup.py" script of first-party modules. This causes
+ a "*.egg-info" dir to be created for that module (which pip can then detect as
+ a package). Since we add all first-party module directories to the .pthfile for
+ the "mach" venv, these first-party modules are then detected by all venvs after
+ they are created. The problem is that these .egg-info directories can become
+ stale (since if the first-party module is updated it's not guaranteed that the
+ command that runs the "setup.py" was ran afterwards). This can cause
+ incompatibilities with the pip check (since the dependencies can change between
+ different versions).
+
+ These .egg-info dirs are in our VCS ignore lists (eg: ".hgignore") because they
+ are necessary to run some commands, so we don't want to always purge them, and we
+ also don't want to accidentally commit them. Given this, we can leverage our VCS
+ to find all the current first-party .egg-info dirs.
+
+ If we're in the case where 'pip check' fails, then we can try purging the
+ first-party .egg-info dirs, then run the 'pip check' again afterwards. If it's
+ still failing, then we know the .egg-info dirs weren't the problem. If that's
+ the case we can just raise the error encountered, which is the same as before.
+ """
+
+ def _delete_ignored_egg_info_dirs():
+ from pathlib import Path
+
+ from mozversioncontrol import (
+ MissingConfigureInfo,
+ MissingVCSInfo,
+ get_repository_from_env,
+ )
+
+ try:
+ with get_repository_from_env() as repo:
+ ignored_file_finder = repo.get_ignored_files_finder().find(
+ "**/*.egg-info"
+ )
+
+ unique_egg_info_dirs = {
+ Path(found[0]).parent for found in ignored_file_finder
+ }
+
+ for egg_info_dir in unique_egg_info_dirs:
+ shutil.rmtree(egg_info_dir)
+
+ except (MissingVCSInfo, MissingConfigureInfo):
+ pass
+
+ _delete_ignored_egg_info_dirs()
+
+ check_result = subprocess.run(
+ [self.python_path, "-m", "pip", "check"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+
+ if check_result.returncode:
+ if quiet:
+ # If "quiet" was specified, then the "pip install" output wasn't printed
+ # earlier, and was buffered instead. Print that buffer so that debugging
+ # the "pip check" failure is easier.
+ print(install_result.stdout)
+
+ subprocess.check_call(
+ [self.python_path, "-m", "pip", "list", "-v"], stdout=sys.stderr
+ )
+ print(check_result.stdout, file=sys.stderr)
+ raise InstallPipRequirementsException(
+ f'As part of validation after installing "{path}" into the '
+ f'"{self._site_name}" site, the site appears to contain installed '
+ "packages that are incompatible with each other."
+ )
+
+ def _pthfile_lines(self):
+ """Generate the prioritized import scope to encode in the venv's pthfile
+
+ The import priority looks like this:
+ 1. Mach's vendored/first-party modules
+ 2. Mach's site-package source (the Mach virtualenv, the system Python, or neither)
+ 3. The command's vendored/first-party modules
+ 4. The command's site-package source (either the virtualenv or the system Python,
+ if it's not already added)
+
+ Note that, when using the system Python, it may either be prioritized before or
+ after the command's vendored/first-party modules. This is a symptom of us
+ attempting to avoid conflicting with the system packages.
+
+ For example, there's at least one job in CI that operates with an ancient
+ environment with a bunch of old packages, many of whom conflict with our vendored
+ packages. However, the specific command that we're running for the job doesn't
+ need any of the system's packages, so we're safe to insulate ourselves.
+
+ Mach doesn't know the command being run when it's preparing its import scope,
+ so it has to be defensive. Therefore:
+ 1. If Mach needs a system package: system packages are higher priority.
+ 2. If Mach doesn't need a system package, but the current command does: system
+ packages are still be in the list, albeit at a lower priority.
+ """
+
+ # Prioritize Mach's vendored and first-party modules first.
+ lines = resolve_requirements(self._topsrcdir, "mach").pths_as_absolute(
+ self._topsrcdir
+ )
+ mach_site_packages_source = self._mach_site_packages_source
+ if mach_site_packages_source == SitePackagesSource.SYSTEM:
+ # When Mach is using the system environment, add it next.
+ _, system_site_paths = self._metadata.original_python.sys_path()
+ lines.extend(system_site_paths)
+ elif mach_site_packages_source == SitePackagesSource.VENV:
+ # When Mach is using its on-disk virtualenv, add its site-packages directory.
+ assert self._mach_virtualenv_root
+ lines.extend(
+ PythonVirtualenv(self._mach_virtualenv_root).site_packages_dirs()
+ )
+
+ # Add this command's vendored and first-party modules.
+ lines.extend(self._requirements.pths_as_absolute(self._topsrcdir))
+ # Finally, ensure that pip-installed packages are the lowest-priority
+ # source to import from.
+ lines.extend(
+ _deprioritize_venv_packages(self._virtualenv, self._populate_virtualenv)
+ )
+
+ # Note that an on-disk virtualenv is always created for commands, even if they
+ # are using the system as their site-packages source. This is to support use
+ # cases where a fresh Python process must be created, but it also must have
+ # access to <site>'s 1st- and 3rd-party packages.
+ return lines
+
+ def _up_to_date(self):
+ pthfile_lines = self._pthfile_lines()
+ if self._mach_site_packages_source == SitePackagesSource.SYSTEM:
+ _assert_pip_check(
+ pthfile_lines,
+ self._site_name,
+ self._requirements if not self._populate_virtualenv else None,
+ )
+
+ return _is_venv_up_to_date(
+ self._virtualenv,
+ pthfile_lines,
+ self._requirements,
+ self._metadata,
+ )
+
+
+class PythonVirtualenv:
+ """Calculates paths of interest for general python virtual environments"""
+
+ def __init__(self, prefix):
+ if _is_windows:
+ self.bin_path = os.path.join(prefix, "Scripts")
+ self.python_path = os.path.join(self.bin_path, "python.exe")
+ else:
+ self.bin_path = os.path.join(prefix, "bin")
+ self.python_path = os.path.join(self.bin_path, "python")
+ self.prefix = os.path.realpath(prefix)
+
+ @functools.lru_cache(maxsize=None)
+ def resolve_sysconfig_packages_path(self, sysconfig_path):
+ # macOS uses a different default sysconfig scheme based on whether it's using the
+ # system Python or running in a virtualenv.
+ # Manually define the scheme (following the implementation in
+ # "sysconfig._get_default_scheme()") so that we're always following the
+ # code path for a virtualenv directory structure.
+ if os.name == "posix":
+ scheme = "posix_prefix"
+ else:
+ scheme = os.name
+
+ sysconfig_paths = sysconfig.get_paths(scheme)
+ data_path = Path(sysconfig_paths["data"])
+ path = Path(sysconfig_paths[sysconfig_path])
+ relative_path = path.relative_to(data_path)
+
+ # Path to virtualenv's "site-packages" directory for provided sysconfig path
+ return os.path.normpath(os.path.normcase(Path(self.prefix) / relative_path))
+
+ def site_packages_dirs(self):
+ dirs = []
+ if sys.platform.startswith("win"):
+ dirs.append(os.path.normpath(os.path.normcase(self.prefix)))
+ purelib = self.resolve_sysconfig_packages_path("purelib")
+ platlib = self.resolve_sysconfig_packages_path("platlib")
+
+ dirs.append(purelib)
+ if platlib != purelib:
+ dirs.append(platlib)
+
+ return dirs
+
+ def pip_install_with_constraints(self, pip_args):
+ """Create a pip constraints file or existing packages
+
+ When pip installing an incompatible package, pip will follow through with
+ the install but raise a warning afterwards.
+
+ To defend our environment from breakage, we run "pip install" but add all
+ existing packages to a "constraints file". This ensures that conflicts are
+ raised as errors up-front, and the virtual environment doesn't have conflicting
+ packages installed.
+
+ Note: pip_args is expected to contain either the requested package or
+ requirements file.
+ """
+ existing_packages = self._resolve_installed_packages()
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ constraints_path = os.path.join(tempdir, "site-constraints.txt")
+ with open(constraints_path, "w") as file:
+ file.write(
+ "\n".join(
+ [
+ f"{name}=={version}"
+ for name, version in existing_packages.items()
+ ]
+ )
+ )
+
+ return self.pip_install(["--constraint", constraints_path] + pip_args)
+
+ def pip_install(self, pip_install_args, **kwargs):
+ # setuptools will use the architecture of the running Python instance when
+ # building packages. However, it's possible for the Xcode Python to be a universal
+ # binary (x86_64 and arm64) without the associated macOS SDK supporting arm64,
+ # thereby causing a build failure. To avoid this, we explicitly influence the
+ # build to only target a single architecture - our current architecture.
+ kwargs.setdefault("env", os.environ.copy()).setdefault(
+ "ARCHFLAGS", "-arch {}".format(platform.machine())
+ )
+ kwargs.setdefault("check", True)
+ kwargs.setdefault("stderr", subprocess.STDOUT)
+ kwargs.setdefault("universal_newlines", True)
+
+ # It's tempting to call pip natively via pip.main(). However,
+ # the current Python interpreter may not be the virtualenv python.
+ # This will confuse pip and cause the package to attempt to install
+ # against the executing interpreter. By creating a new process, we
+ # force the virtualenv's interpreter to be used and all is well.
+ # It /might/ be possible to cheat and set sys.executable to
+ # self.python_path. However, this seems more risk than it's worth.
+ return subprocess.run(
+ [self.python_path, "-m", "pip", "install"] + pip_install_args,
+ **kwargs,
+ )
+
+ def install_optional_packages(self, optional_requirements):
+ for requirement in optional_requirements:
+ try:
+ self.pip_install_with_constraints([str(requirement.requirement)])
+ except subprocess.CalledProcessError:
+ print(
+ f"Could not install {requirement.requirement.name}, so "
+ f"{requirement.repercussion}. Continuing."
+ )
+
+ def _resolve_installed_packages(self):
+ return _resolve_installed_packages(self.python_path)
+
+
+class RequirementsValidationResult:
+ def __init__(self):
+ self._package_discrepancies = []
+ self.has_all_packages = True
+ self.provides_any_package = False
+
+ def add_discrepancy(self, requirement, found):
+ self._package_discrepancies.append((requirement, found))
+ self.has_all_packages = False
+
+ def report(self):
+ lines = []
+ for requirement, found in self._package_discrepancies:
+ if found:
+ error = f'Installed with unexpected version "{found}"'
+ else:
+ error = "Not installed"
+ lines.append(f"{requirement}: {error}")
+ return "\n".join(lines)
+
+ @classmethod
+ def from_packages(cls, packages, requirements):
+ result = cls()
+ for pkg in requirements.pypi_requirements:
+ installed_version = packages.get(pkg.requirement.name)
+ if not installed_version or not pkg.requirement.specifier.contains(
+ installed_version
+ ):
+ result.add_discrepancy(pkg.requirement, installed_version)
+ elif installed_version:
+ result.provides_any_package = True
+
+ for pkg in requirements.pypi_optional_requirements:
+ installed_version = packages.get(pkg.requirement.name)
+ if installed_version and not pkg.requirement.specifier.contains(
+ installed_version
+ ):
+ result.add_discrepancy(pkg.requirement, installed_version)
+ elif installed_version:
+ result.provides_any_package = True
+
+ return result
+
+
+class ExternalPythonSite:
+ """Represents the Python site that is executing Mach
+
+ The external Python site could be a virtualenv (created by venv or virtualenv) or
+ the system Python itself, so we can't make any significant assumptions on its
+ structure.
+ """
+
+ def __init__(self, python_executable):
+ self._prefix = os.path.dirname(os.path.dirname(python_executable))
+ self.python_path = python_executable
+
+ @functools.lru_cache(maxsize=None)
+ def sys_path(self):
+ """Return lists of sys.path entries: one for standard library, one for the site
+
+ These two lists are calculated at the same time so that we can interpret them
+ in a single Python subprocess, as running a whole Python instance is
+ very expensive in the context of Mach initialization.
+ """
+ env = {
+ k: v
+ for k, v in os.environ.items()
+ # Don't include items injected by IDEs into the system path.
+ if k not in ("PYTHONPATH", "PYDEVD_LOAD_VALUES_ASYNC")
+ }
+ stdlib = subprocess.Popen(
+ [
+ self.python_path,
+ # Don't "import site" right away, so we can split the standard library
+ # paths from the site paths.
+ "-S",
+ "-c",
+ "import sys; from collections import OrderedDict; "
+ # Skip the first item in the sys.path, as it's the working directory
+ # of the invoked script (so, in this case, "").
+ # Use list(OrderectDict...) to de-dupe items, such as when using
+ # pyenv on Linux.
+ "print(list(OrderedDict.fromkeys(sys.path[1:])))",
+ ],
+ universal_newlines=True,
+ env=env,
+ stdout=subprocess.PIPE,
+ )
+ system = subprocess.Popen(
+ [
+ self.python_path,
+ "-c",
+ "import os; import sys; import site; "
+ "packages = site.getsitepackages(); "
+ # Only add the "user site packages" if not in a virtualenv (which is
+ # identified by the prefix == base_prefix check
+ "packages.insert(0, site.getusersitepackages()) if "
+ " sys.prefix == sys.base_prefix else None; "
+ # When a Python instance launches, it only adds each
+ # "site.getsitepackages()" entry if it exists on the file system.
+ # Replicate that behaviour to get a more accurate list of system paths.
+ "packages = [p for p in packages if os.path.exists(p)]; "
+ "print(packages)",
+ ],
+ universal_newlines=True,
+ env=env,
+ stdout=subprocess.PIPE,
+ )
+ # Run python processes in parallel - they take roughly the same time, so this
+ # cuts this functions run time in half.
+ stdlib_out, _ = stdlib.communicate()
+ system_out, _ = system.communicate()
+ assert stdlib.returncode == 0
+ assert system.returncode == 0
+ stdlib = ast.literal_eval(stdlib_out)
+ system = ast.literal_eval(system_out)
+ # On Windows, some paths are both part of the default sys.path *and* are included
+ # in the "site packages" list. Keep the "stdlib" one, and remove the dupe from
+ # the "system packages" list.
+ system = [path for path in system if path not in stdlib]
+ return stdlib, system
+
+ def sys_path_stdlib(self):
+ """Return list of default sys.path entries for the standard library"""
+ stdlib, _ = self.sys_path()
+ return stdlib
+
+
+@functools.lru_cache(maxsize=None)
+def resolve_requirements(topsrcdir, site_name):
+ manifest_path = os.path.join(topsrcdir, "python", "sites", f"{site_name}.txt")
+ if not os.path.exists(manifest_path):
+ raise Exception(
+ f'The current command is using the "{site_name}" '
+ "site. However, that site is missing its associated "
+ f'requirements definition file at "{manifest_path}".'
+ )
+
+ thunderbird_dir = os.path.join(topsrcdir, "comm")
+ is_thunderbird = os.path.exists(thunderbird_dir) and bool(
+ os.listdir(thunderbird_dir)
+ )
+ try:
+ return MachEnvRequirements.from_requirements_definition(
+ topsrcdir,
+ is_thunderbird,
+ site_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
+ manifest_path,
+ )
+ except UnexpectedFlexibleRequirementException as e:
+ raise Exception(
+ f'The "{site_name}" site does not have all pypi packages pinned '
+ f'in the format "package==version" (found "{e.raw_requirement}").\n'
+ f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} sites are "
+ "allowed to have unpinned packages."
+ )
+
+
+def _resolve_installed_packages(python_executable):
+ pip_json = subprocess.check_output(
+ [
+ python_executable,
+ "-m",
+ "pip",
+ "list",
+ "--format",
+ "json",
+ "--disable-pip-version-check",
+ ],
+ universal_newlines=True,
+ )
+
+ installed_packages = json.loads(pip_json)
+ return {package["name"]: package["version"] for package in installed_packages}
+
+
+def _ensure_python_exe(python_exe_root: Path):
+ """On some machines in CI venv does not behave consistently. Sometimes
+ only a "python3" executable is created, but we expect "python". Since
+ they are functionally identical, we can just copy "python3" to "python"
+ (and vice-versa) to solve the problem.
+ """
+ python3_exe_path = python_exe_root / "python3"
+ python_exe_path = python_exe_root / "python"
+
+ if _is_windows:
+ python3_exe_path = python3_exe_path.with_suffix(".exe")
+ python_exe_path = python_exe_path.with_suffix(".exe")
+
+ if python3_exe_path.exists() and not python_exe_path.exists():
+ shutil.copy(str(python3_exe_path), str(python_exe_path))
+
+ if python_exe_path.exists() and not python3_exe_path.exists():
+ shutil.copy(str(python_exe_path), str(python3_exe_path))
+
+ if not python_exe_path.exists() and not python3_exe_path.exists():
+ raise Exception(
+ f'Neither a "{python_exe_path.name}" or "{python3_exe_path.name}" '
+ f"were found. This means something unexpected happened during the "
+ f"virtual environment creation and we cannot proceed."
+ )
+
+
+def _ensure_pyvenv_cfg(venv_root: Path):
+ # We can work around a bug on some versions of Python 3.6 on
+ # Windows by copying the 'pyvenv.cfg' of the current venv
+ # to the new venv. This will make the new venv reference
+ # the original Python install instead of the current venv,
+ # which resolves the issue. There shouldn't be any harm in
+ # always doing this, but we'll play it safe and restrict it
+ # to Windows Python 3.6 anyway.
+ if _is_windows and sys.version_info[:2] == (3, 6):
+ this_venv = Path(sys.executable).parent.parent
+ this_venv_config = this_venv / "pyvenv.cfg"
+ if this_venv_config.exists():
+ new_venv_config = Path(venv_root) / "pyvenv.cfg"
+ shutil.copyfile(str(this_venv_config), str(new_venv_config))
+
+
+def _assert_pip_check(pthfile_lines, virtualenv_name, requirements):
+ """Check if the provided pthfile lines have a package incompatibility
+
+ If there's an incompatibility, raise an exception and allow it to bubble up since
+ it will require user intervention to resolve.
+
+ If requirements aren't provided (such as when Mach is using SYSTEM, but the command
+ site is using VENV), then skip the "pthfile satisfies requirements" step.
+ """
+ if os.environ.get(
+ f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE", None
+ ):
+ # Don't re-assert compatibility against the system python within Mach subshells.
+ return
+
+ print(
+ 'Running "pip check" to verify compatibility between the system Python and the '
+ f'"{virtualenv_name}" site.'
+ )
+
+ with tempfile.TemporaryDirectory() as check_env_path:
+ # Pip detects packages on the "sys.path" that have a ".dist-info" or
+ # a ".egg-info" directory. The majority of our Python dependencies are
+ # vendored as extracted wheels or sdists, so they are automatically picked up.
+ # This gives us sufficient confidence to do a `pip check` with both vendored
+ # packages + system packages in scope, and trust the results.
+ # Note: rather than just running the system pip with a modified "sys.path",
+ # we create a new virtualenv that has our pinned pip version, so that
+ # we get consistent results (there's been lots of pip resolver behaviour
+ # changes recently).
+ process = subprocess.run(
+ [sys.executable, "-m", "venv", "--without-pip", check_env_path],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding="UTF-8",
+ )
+
+ _ensure_pyvenv_cfg(Path(check_env_path))
+
+ if process.returncode != 0:
+ if "No module named venv" in process.stderr:
+ raise VenvModuleNotFoundException()
+ else:
+ raise subprocess.CalledProcessError(
+ process.returncode,
+ process.args,
+ output=process.stdout,
+ stderr=process.stderr,
+ )
+
+ if process.stdout:
+ print(process.stdout)
+
+ check_env = PythonVirtualenv(check_env_path)
+ _ensure_python_exe(Path(check_env.python_path).parent)
+
+ with open(
+ os.path.join(
+ os.path.join(check_env.resolve_sysconfig_packages_path("platlib")),
+ PTH_FILENAME,
+ ),
+ "w",
+ ) as f:
+ f.write("\n".join(pthfile_lines))
+
+ pip = [check_env.python_path, "-m", "pip"]
+ if requirements:
+ packages = _resolve_installed_packages(check_env.python_path)
+ validation_result = RequirementsValidationResult.from_packages(
+ packages, requirements
+ )
+ if not validation_result.has_all_packages:
+ subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
+ print(validation_result.report(), file=sys.stderr)
+ raise Exception(
+ f'The "{virtualenv_name}" site is not compatible with the installed '
+ "system Python packages."
+ )
+
+ check_result = subprocess.run(
+ pip + ["check"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+ if check_result.returncode:
+ subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
+ print(check_result.stdout, file=sys.stderr)
+ raise Exception(
+ 'According to "pip check", the current Python '
+ "environment has package-compatibility issues."
+ )
+
+ os.environ[
+ f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE"
+ ] = "1"
+
+
+def _deprioritize_venv_packages(virtualenv, populate_virtualenv):
+ # Virtualenvs implicitly add some "site packages" to the sys.path upon being
+ # activated. However, Mach generally wants to prioritize the existing sys.path
+ # (such as vendored packages) over packages installed to virtualenvs.
+ # So, this function moves the virtualenv's site-packages to the bottom of the sys.path
+ # at activation-time.
+
+ return [
+ line
+ for site_packages_dir in virtualenv.site_packages_dirs()
+ # repr(...) is needed to ensure Windows path backslashes aren't mistaken for
+ # escape sequences.
+ # Additionally, when removing the existing "site-packages" folder's entry, we have
+ # to do it in a case-insensitive way because, on Windows:
+ # * Python adds it as <venv>/lib/site-packages
+ # * While sysconfig tells us it's <venv>/Lib/site-packages
+ # * (note: on-disk, it's capitalized, so sysconfig is slightly more accurate).
+ for line in filter(
+ None,
+ (
+ "import sys; sys.path = [p for p in sys.path if "
+ f"p.lower() != {repr(site_packages_dir)}.lower()]",
+ f"import sys; sys.path.append({repr(site_packages_dir)})"
+ if populate_virtualenv
+ else None,
+ ),
+ )
+ ]
+
+
+def _create_venv_with_pthfile(
+ target_venv,
+ pthfile_lines,
+ populate_with_pip,
+ requirements,
+ metadata,
+):
+ virtualenv_root = target_venv.prefix
+ if os.path.exists(virtualenv_root):
+ shutil.rmtree(virtualenv_root)
+
+ os.makedirs(virtualenv_root)
+ metadata.write(is_finalized=False)
+
+ process = subprocess.run(
+ [sys.executable, "-m", "venv", "--without-pip", virtualenv_root],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding="UTF-8",
+ )
+
+ _ensure_pyvenv_cfg(Path(virtualenv_root))
+
+ if process.returncode != 0:
+ if "No module named venv" in process.stderr:
+ raise VenvModuleNotFoundException()
+ else:
+ raise subprocess.CalledProcessError(
+ process.returncode,
+ process.args,
+ output=process.stdout,
+ stderr=process.stderr,
+ )
+
+ if process.stdout:
+ print(process.stdout)
+
+ _ensure_python_exe(Path(target_venv.python_path).parent)
+
+ platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib")
+ pthfile_contents = "\n".join(pthfile_lines)
+ with open(os.path.join(platlib_site_packages_dir, PTH_FILENAME), "w") as f:
+ f.write(pthfile_contents)
+
+ if populate_with_pip:
+ for requirement in requirements.pypi_requirements:
+ target_venv.pip_install([str(requirement.requirement)])
+ target_venv.install_optional_packages(requirements.pypi_optional_requirements)
+
+ metadata.write(is_finalized=True)
+
+
+def _is_venv_up_to_date(
+ target_venv,
+ expected_pthfile_lines,
+ requirements,
+ expected_metadata,
+):
+ if not os.path.exists(target_venv.prefix):
+ return SiteUpToDateResult(False, f'"{target_venv.prefix}" does not exist')
+
+ # Modifications to any of the requirements manifest files mean the virtualenv should
+ # be rebuilt:
+ metadata_mtime = os.path.getmtime(
+ os.path.join(target_venv.prefix, METADATA_FILENAME)
+ )
+ for dep_file in requirements.requirements_paths:
+ if os.path.getmtime(dep_file) > metadata_mtime:
+ return SiteUpToDateResult(
+ False, f'"{dep_file}" has changed since the virtualenv was created'
+ )
+
+ try:
+ existing_metadata = MozSiteMetadata.from_path(target_venv.prefix)
+ except MozSiteMetadataOutOfDateError as e:
+ # The metadata is missing required fields, so must be out-of-date.
+ return SiteUpToDateResult(False, str(e))
+
+ if existing_metadata != expected_metadata:
+ # The metadata doesn't exist or some fields have different values.
+ return SiteUpToDateResult(
+ False,
+ f"The existing metadata on-disk ({vars(existing_metadata)}) does not match "
+ f"the expected metadata ({vars(expected_metadata)}",
+ )
+
+ platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib")
+ pthfile_path = os.path.join(platlib_site_packages_dir, PTH_FILENAME)
+ try:
+ with open(pthfile_path) as file:
+ current_pthfile_contents = file.read().strip()
+ except FileNotFoundError:
+ return SiteUpToDateResult(False, f'No pthfile found at "{pthfile_path}"')
+
+ expected_pthfile_contents = "\n".join(expected_pthfile_lines)
+ if current_pthfile_contents != expected_pthfile_contents:
+ return SiteUpToDateResult(
+ False,
+ f'The pthfile at "{pthfile_path}" does not match the expected value.\n'
+ f"# --- on-disk pthfile: ---\n"
+ f"{current_pthfile_contents}\n"
+ f"# --- expected pthfile contents ---\n"
+ f"{expected_pthfile_contents}\n"
+ f"# ---",
+ )
+
+ return SiteUpToDateResult(True)
+
+
+def activate_virtualenv(virtualenv: PythonVirtualenv):
+ os.environ["PATH"] = os.pathsep.join(
+ [virtualenv.bin_path] + os.environ.get("PATH", "").split(os.pathsep)
+ )
+ os.environ["VIRTUAL_ENV"] = virtualenv.prefix
+
+ for path in virtualenv.site_packages_dirs():
+ site.addsitedir(os.path.realpath(path))
+
+ sys.prefix = virtualenv.prefix
+
+
+def _mach_virtualenv_root(checkout_scoped_state_dir):
+ workspace = os.environ.get("WORKSPACE")
+ if os.environ.get("MOZ_AUTOMATION") and workspace:
+ # In CI, put Mach virtualenv in the $WORKSPACE dir, which should be cleaned
+ # between jobs.
+ return os.path.join(workspace, "mach_virtualenv")
+ return os.path.join(checkout_scoped_state_dir, "_virtualenvs", "mach")
diff --git a/python/mach/mach/telemetry.py b/python/mach/mach/telemetry.py
new file mode 100644
index 0000000000..233556550d
--- /dev/null
+++ b/python/mach/mach/telemetry.py
@@ -0,0 +1,305 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from textwrap import dedent
+
+import requests
+import six.moves.urllib.parse as urllib_parse
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from mozbuild.settings import TelemetrySettings
+from mozbuild.telemetry import filter_args
+from mozversioncontrol import InvalidRepoPath, get_repository_object
+from six.moves import configparser, input
+
+from mach.config import ConfigSettings
+from mach.site import MozSiteMetadata
+from mach.telemetry_interface import GleanTelemetry, NoopTelemetry
+from mach.util import get_state_dir
+
+MACH_METRICS_PATH = (Path(__file__) / ".." / ".." / "metrics.yaml").resolve()
+
+
+def create_telemetry_from_environment(settings):
+ """Creates and a Telemetry instance based on system details.
+
+ If telemetry isn't enabled, the current interpreter isn't Python 3, or Glean
+ can't be imported, then a "mock" telemetry instance is returned that doesn't
+ set or record any data. This allows consumers to optimistically set telemetry
+ data without needing to specifically handle the case where the current system
+ doesn't support it.
+ """
+
+ active_metadata = MozSiteMetadata.from_runtime()
+ is_mach_virtualenv = active_metadata and active_metadata.site_name == "mach"
+
+ if not (
+ is_applicable_telemetry_environment()
+ # Glean is not compatible with Python 2
+ and sys.version_info >= (3, 0)
+ # If not using the mach virtualenv (e.g.: bootstrap uses native python)
+ # then we can't guarantee that the glean package that we import is a
+ # compatible version. Therefore, don't use glean.
+ and is_mach_virtualenv
+ ):
+ return NoopTelemetry(False)
+
+ is_enabled = is_telemetry_enabled(settings)
+
+ try:
+ from glean import Glean
+ except ImportError:
+ return NoopTelemetry(is_enabled)
+
+ from pathlib import Path
+
+ Glean.initialize(
+ "mozilla.mach",
+ "Unknown",
+ is_enabled,
+ data_dir=Path(get_state_dir()) / "glean",
+ )
+ return GleanTelemetry()
+
+
+def report_invocation_metrics(telemetry, command):
+ metrics = telemetry.metrics(MACH_METRICS_PATH)
+ metrics.mach.command.set(command)
+ metrics.mach.duration.start()
+
+ try:
+ instance = MozbuildObject.from_environment()
+ except BuildEnvironmentNotFoundException:
+ # Mach may be invoked with the state dir as the current working
+ # directory, in which case we're not able to find the topsrcdir (so
+ # we can't create a MozbuildObject instance).
+ # Without this information, we're unable to filter argv paths, so
+ # we skip submitting them to telemetry.
+ return
+ metrics.mach.argv.set(
+ filter_args(command, sys.argv, instance.topsrcdir, instance.topobjdir)
+ )
+
+
+def is_applicable_telemetry_environment():
+ if os.environ.get("MACH_MAIN_PID") != str(os.getpid()):
+ # This is a child mach process. Since we're collecting telemetry for the parent,
+ # we don't want to collect telemetry again down here.
+ return False
+
+ if any(e in os.environ for e in ("MOZ_AUTOMATION", "TASK_ID")):
+ return False
+
+ return True
+
+
+def is_telemetry_enabled(settings):
+ if os.environ.get("DISABLE_TELEMETRY") == "1":
+ return False
+
+ return settings.mach_telemetry.is_enabled
+
+
+def arcrc_path():
+ if sys.platform.startswith("win32") or sys.platform.startswith("msys"):
+ return Path(os.environ.get("APPDATA", "")) / ".arcrc"
+ else:
+ return Path("~/.arcrc").expanduser()
+
+
+def resolve_setting_from_arcconfig(topsrcdir: Path, setting):
+ git_path = topsrcdir / ".git"
+ if git_path.is_file():
+ git_path = subprocess.check_output(
+ ["git", "rev-parse", "--git-common-dir"],
+ cwd=str(topsrcdir),
+ universal_newlines=True,
+ )
+ git_path = Path(git_path)
+
+ for arcconfig_path in [
+ topsrcdir / ".hg" / ".arcconfig",
+ git_path / ".arcconfig",
+ topsrcdir / ".arcconfig",
+ ]:
+ try:
+ with open(arcconfig_path, "r") as arcconfig_file:
+ arcconfig = json.load(arcconfig_file)
+ except (json.JSONDecodeError, FileNotFoundError):
+ continue
+
+ value = arcconfig.get(setting)
+ if value:
+ return value
+
+
+def resolve_is_employee_by_credentials(topsrcdir: Path):
+ phabricator_uri = resolve_setting_from_arcconfig(topsrcdir, "phabricator.uri")
+
+ if not phabricator_uri:
+ return None
+
+ try:
+ with open(arcrc_path(), "r") as arcrc_file:
+ arcrc = json.load(arcrc_file)
+ except (json.JSONDecodeError, FileNotFoundError):
+ return None
+
+ phabricator_token = (
+ arcrc.get("hosts", {})
+ .get(urllib_parse.urljoin(phabricator_uri, "api/"), {})
+ .get("token")
+ )
+
+ if not phabricator_token:
+ return None
+
+ bmo_uri = (
+ resolve_setting_from_arcconfig(topsrcdir, "bmo_url")
+ or "https://bugzilla.mozilla.org"
+ )
+ bmo_api_url = urllib_parse.urljoin(bmo_uri, "rest/whoami")
+ bmo_result = requests.get(
+ bmo_api_url, headers={"X-PHABRICATOR-TOKEN": phabricator_token}
+ )
+ return "mozilla-employee-confidential" in bmo_result.json().get("groups", [])
+
+
+def resolve_is_employee_by_vcs(topsrcdir: Path):
+ try:
+ vcs = get_repository_object(str(topsrcdir))
+ except InvalidRepoPath:
+ return None
+
+ email = vcs.get_user_email()
+ if not email:
+ return None
+
+ return "@mozilla.com" in email
+
+
+def resolve_is_employee(topsrcdir: Path):
+ """Detect whether or not the current user is a Mozilla employee.
+
+ Checks using Bugzilla authentication, if possible. Otherwise falls back to checking
+ if email configured in VCS is "@mozilla.com".
+
+ Returns True if the user could be identified as an employee, False if the user
+ is confirmed as not being an employee, or None if the user couldn't be
+ identified.
+ """
+ is_employee = resolve_is_employee_by_credentials(topsrcdir)
+ if is_employee is not None:
+ return is_employee
+
+ return resolve_is_employee_by_vcs(topsrcdir) or False
+
+
+def record_telemetry_settings(
+ main_settings,
+ state_dir: Path,
+ is_enabled,
+):
+ # We want to update the user's machrc file. However, the main settings object
+ # contains config from "$topsrcdir/machrc" (if it exists) which we don't want
+ # to accidentally include. So, we have to create a brand new mozbuild-specific
+ # settings, update it, then write to it.
+ settings_path = state_dir / "machrc"
+ file_settings = ConfigSettings()
+ file_settings.register_provider(TelemetrySettings)
+ try:
+ file_settings.load_file(settings_path)
+ except configparser.Error as error:
+ print(
+ f"Your mach configuration file at `{settings_path}` cannot be parsed:\n{error}"
+ )
+ return
+
+ file_settings.mach_telemetry.is_enabled = is_enabled
+ file_settings.mach_telemetry.is_set_up = True
+
+ with open(settings_path, "w") as f:
+ file_settings.write(f)
+
+ # Telemetry will want this elsewhere in the mach process, so we'll slap the
+ # new values on the main settings object.
+ main_settings.mach_telemetry.is_enabled = is_enabled
+ main_settings.mach_telemetry.is_set_up = True
+
+
+TELEMETRY_DESCRIPTION_PREAMBLE = """
+Mozilla collects data to improve the developer experience.
+To learn more about the data we intend to collect, read here:
+ https://firefox-source-docs.mozilla.org/build/buildsystem/telemetry.html
+If you have questions, please ask in #build on Matrix:
+ https://chat.mozilla.org/#/room/#build:mozilla.org
+""".strip()
+
+
+def print_telemetry_message_employee():
+ message_template = dedent(
+ """
+ %s
+
+ As a Mozilla employee, telemetry has been automatically enabled.
+ """
+ ).strip()
+ print(message_template % TELEMETRY_DESCRIPTION_PREAMBLE)
+ return True
+
+
+def prompt_telemetry_message_contributor():
+ while True:
+ prompt = (
+ dedent(
+ """
+ %s
+
+ If you'd like to opt out of data collection, select (N) at the prompt.
+ Would you like to enable build system telemetry? (Yn): """
+ )
+ % TELEMETRY_DESCRIPTION_PREAMBLE
+ ).strip()
+
+ choice = input(prompt)
+ choice = choice.strip().lower()
+ if choice == "":
+ return True
+ if choice not in ("y", "n"):
+ print("ERROR! Please enter y or n!")
+ else:
+ return choice == "y"
+
+
+def initialize_telemetry_setting(settings, topsrcdir: str, state_dir: str):
+ """Enables telemetry for employees or prompts the user."""
+ # If the user doesn't care about telemetry for this invocation, then
+ # don't make requests to Bugzilla and/or prompt for whether the
+ # user wants to opt-in.
+
+ if topsrcdir is not None:
+ topsrcdir = Path(topsrcdir)
+
+ if state_dir is not None:
+ state_dir = Path(state_dir)
+
+ if os.environ.get("DISABLE_TELEMETRY") == "1":
+ return
+
+ try:
+ is_employee = resolve_is_employee(topsrcdir)
+ except requests.exceptions.RequestException:
+ return
+
+ if is_employee:
+ is_enabled = True
+ print_telemetry_message_employee()
+ else:
+ is_enabled = prompt_telemetry_message_contributor()
+
+ record_telemetry_settings(settings, state_dir, is_enabled)
diff --git a/python/mach/mach/telemetry_interface.py b/python/mach/mach/telemetry_interface.py
new file mode 100644
index 0000000000..3ed8ce5674
--- /dev/null
+++ b/python/mach/mach/telemetry_interface.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+from pathlib import Path
+from typing import Union
+from unittest.mock import Mock
+
+from mach.site import MozSiteMetadata, SitePackagesSource
+
+
+class NoopTelemetry(object):
+ def __init__(self, failed_glean_import):
+ self._failed_glean_import = failed_glean_import
+
+ def metrics(self, metrics_path: Union[str, Path]):
+ return Mock()
+
+ def submit(self, is_bootstrap):
+ if self._failed_glean_import and not is_bootstrap:
+ active_site = MozSiteMetadata.from_runtime()
+ if active_site.mach_site_packages_source == SitePackagesSource.SYSTEM:
+ hint = (
+ "Mach is looking for glean in the system packages. This can be "
+ "resolved by installing it there, or by allowing Mach to run "
+ "without using the system Python packages."
+ )
+ elif active_site.mach_site_packages_source == SitePackagesSource.NONE:
+ hint = (
+ "This is because Mach is currently configured without a source "
+ "for native Python packages."
+ )
+ else:
+ hint = "You may need to run |mach bootstrap|."
+
+ print(
+ f"Glean could not be found, so telemetry will not be reported. {hint}",
+ file=sys.stderr,
+ )
+
+
+class GleanTelemetry(object):
+ """Records and sends Telemetry using Glean.
+
+ Metrics are defined in python/mozbuild/metrics.yaml.
+ Pings are defined in python/mozbuild/pings.yaml.
+
+ The "metrics" and "pings" properties may be replaced with no-op implementations if
+ Glean isn't available. This allows consumers to report telemetry without having
+ to guard against incompatible environments.
+
+ Also tracks whether an employee was just automatically opted into telemetry
+ during this mach invocation.
+ """
+
+ def __init__(
+ self,
+ ):
+ self._metrics_cache = {}
+
+ def metrics(self, metrics_path: Union[str, Path]):
+ if metrics_path not in self._metrics_cache:
+ from glean import load_metrics
+
+ metrics = load_metrics(metrics_path)
+ self._metrics_cache[metrics_path] = metrics
+
+ return self._metrics_cache[metrics_path]
+
+ def submit(self, _):
+ from pathlib import Path
+
+ from glean import load_pings
+
+ pings = load_pings(Path(__file__).parent.parent / "pings.yaml")
+ pings.usage.submit()
diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py
new file mode 100644
index 0000000000..a0c8d0a6ed
--- /dev/null
+++ b/python/mach/mach/terminal.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+import logging
+import sys
+
+from six.moves import range
+
+
+class LoggingHandler(logging.Handler):
+ """Custom logging handler that works with terminal window dressing.
+
+ This is alternative terminal logging handler which contains smarts for
+ emitting terminal control characters properly. Currently, it has generic
+ support for "footer" elements at the bottom of the screen. Functionality
+ can be added when needed.
+ """
+
+ def __init__(self):
+ logging.Handler.__init__(self)
+
+ self.fh = sys.stdout
+ self.footer = None
+
+ def flush(self):
+ self.acquire()
+
+ try:
+ self.fh.flush()
+ finally:
+ self.release()
+
+ def emit(self, record):
+ msg = self.format(record)
+
+ if self.footer:
+ self.footer.clear()
+
+ self.fh.write(msg)
+ self.fh.write("\n")
+
+ if self.footer:
+ self.footer.draw()
+
+ # If we don't flush, the footer may not get drawn.
+ self.flush()
+
+
+class TerminalFooter(object):
+ """Represents something drawn on the bottom of a terminal."""
+
+ def __init__(self, terminal):
+ self.t = terminal
+ self.fh = sys.stdout
+
+ def _clear_lines(self, n):
+ for i in range(n):
+ self.fh.write(self.t.move_x(0))
+ self.fh.write(self.t.clear_eol())
+ self.fh.write(self.t.move_up())
+
+ self.fh.write(self.t.move_down())
+ self.fh.write(self.t.move_x(0))
+
+ def clear(self):
+ raise Exception("clear() must be implemented.")
+
+ def draw(self):
+ raise Exception("draw() must be implemented.")
diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mach/mach/test/__init__.py
diff --git a/python/mach/mach/test/conftest.py b/python/mach/mach/test/conftest.py
new file mode 100644
index 0000000000..78129acb58
--- /dev/null
+++ b/python/mach/mach/test/conftest.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import unittest
+from collections.abc import Iterable
+from pathlib import Path
+from typing import List, Optional, Union
+
+import pytest
+from buildconfig import topsrcdir
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ # TODO io.StringIO causes failures with Python 2 (needs to be sorted out)
+ from io import StringIO
+
+from mach.main import Mach
+
+PROVIDER_DIR = Path(__file__).resolve().parent / "providers"
+
+
+@pytest.fixture(scope="class")
+def get_mach(request):
+ def _populate_context(key):
+ if key == "topdir":
+ return topsrcdir
+
+ def inner(
+ provider_files: Optional[Union[Path, List[Path]]] = None,
+ entry_point=None,
+ context_handler=None,
+ ):
+ m = Mach(str(Path.cwd()))
+ m.define_category("testing", "Mach unittest", "Testing for mach core", 10)
+ m.define_category("misc", "Mach misc", "Testing for mach core", 20)
+ m.populate_context_handler = context_handler or _populate_context
+
+ if provider_files:
+ if not isinstance(provider_files, Iterable):
+ provider_files = [provider_files]
+
+ for path in provider_files:
+ m.load_commands_from_file(PROVIDER_DIR / path)
+
+ if entry_point:
+ m.load_commands_from_entry_point(entry_point)
+
+ return m
+
+ if request.cls and issubclass(request.cls, unittest.TestCase):
+ request.cls.get_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs)
+ return inner
+
+
+@pytest.fixture(scope="class")
+def run_mach(request, get_mach):
+ def inner(argv, *args, **kwargs):
+ m = get_mach(*args, **kwargs)
+
+ stdout = StringIO()
+ stderr = StringIO()
+
+ if sys.version_info < (3, 0):
+ stdout.encoding = "UTF-8"
+ stderr.encoding = "UTF-8"
+
+ try:
+ result = m.run(argv, stdout=stdout, stderr=stderr)
+ except SystemExit:
+ result = None
+
+ return (result, stdout.getvalue(), stderr.getvalue())
+
+ if request.cls and issubclass(request.cls, unittest.TestCase):
+ request.cls._run_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs)
+ return inner
+
+
+@pytest.mark.usefixtures("get_mach", "run_mach")
+class TestBase(unittest.TestCase):
+ pass
diff --git a/python/mach/mach/test/invoke_mach_command.py b/python/mach/mach/test/invoke_mach_command.py
new file mode 100644
index 0000000000..1efa102ef5
--- /dev/null
+++ b/python/mach/mach/test/invoke_mach_command.py
@@ -0,0 +1,4 @@
+import subprocess
+import sys
+
+subprocess.check_call([sys.executable] + sys.argv[1:])
diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mach/mach/test/providers/__init__.py
diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py
new file mode 100644
index 0000000000..26cdfdf588
--- /dev/null
+++ b/python/mach/mach/test/providers/basic.py
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from mach.decorators import Command, CommandArgument
+
+
+@Command("cmd_foo", category="testing")
+def run_foo(command_context):
+ pass
+
+
+@Command("cmd_bar", category="testing")
+@CommandArgument("--baz", action="store_true", help="Run with baz")
+def run_bar(command_context, baz=None):
+ pass
diff --git a/python/mach/mach/test/providers/commands.py b/python/mach/mach/test/providers/commands.py
new file mode 100644
index 0000000000..6b8210c513
--- /dev/null
+++ b/python/mach/mach/test/providers/commands.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from functools import partial
+
+from mach.decorators import Command, CommandArgument
+
+
+def is_foo(cls):
+ """Foo must be true"""
+ return cls.foo
+
+
+def is_bar(val, cls):
+ """Bar must equal val"""
+ return cls.bar == val
+
+
+@Command("cmd_foo", category="testing")
+@CommandArgument("--arg", default=None, help="Argument help.")
+def run_foo(command_context):
+ pass
+
+
+@Command("cmd_bar", category="testing", conditions=[partial(is_bar, False)])
+def run_bar(command_context):
+ pass
+
+
+@Command("cmd_foobar", category="testing", conditions=[is_foo, partial(is_bar, True)])
+def run_foobar(command_context):
+ pass
diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py
new file mode 100644
index 0000000000..db2f3f8123
--- /dev/null
+++ b/python/mach/mach/test/providers/conditions.py
@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mach.decorators import Command
+
+
+def is_true(cls):
+ return True
+
+
+def is_false(cls):
+ return False
+
+
+@Command("cmd_condition_true", category="testing", conditions=[is_true])
+def run_condition_true(self, command_context):
+ pass
+
+
+@Command("cmd_condition_false", category="testing", conditions=[is_false])
+def run_condition_false(self, command_context):
+ pass
+
+
+@Command(
+ "cmd_condition_true_and_false", category="testing", conditions=[is_true, is_false]
+)
+def run_condition_true_and_false(self, command_context):
+ pass
+
+
+def is_ctx_foo(cls):
+ """Foo must be true"""
+ return cls._mach_context.foo
+
+
+def is_ctx_bar(cls):
+ """Bar must be true"""
+ return cls._mach_context.bar
+
+
+@Command("cmd_foo_ctx", category="testing", conditions=[is_ctx_foo])
+def run_foo_ctx(self, command_context):
+ pass
+
+
+@Command("cmd_bar_ctx", category="testing", conditions=[is_ctx_bar])
+def run_bar_ctx(self, command_context):
+ pass
+
+
+@Command("cmd_foobar_ctx", category="testing", conditions=[is_ctx_foo, is_ctx_bar])
+def run_foobar_ctx(self, command_context):
+ pass
diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py
new file mode 100644
index 0000000000..228c56f0bf
--- /dev/null
+++ b/python/mach/mach/test/providers/conditions_invalid.py
@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mach.decorators import Command
+
+
+@Command("cmd_foo", category="testing", conditions=["invalid"])
+def run_foo(command_context):
+ pass
diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py
new file mode 100644
index 0000000000..9ddc7653c0
--- /dev/null
+++ b/python/mach/mach/test/providers/throw.py
@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mach.decorators import Command, CommandArgument
+from mach.test.providers import throw2
+
+
+@Command("throw", category="testing")
+@CommandArgument("--message", "-m", default="General Error")
+def throw(command_context, message):
+ raise Exception(message)
+
+
+@Command("throw_deep", category="testing")
+@CommandArgument("--message", "-m", default="General Error")
+def throw_deep(command_context, message):
+ throw2.throw_deep(message)
diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py
new file mode 100644
index 0000000000..9ff7f2798e
--- /dev/null
+++ b/python/mach/mach/test/providers/throw2.py
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file exists to trigger the differences in mach error reporting between
+# exceptions that occur in mach command modules themselves and in the things
+# they call.
+
+
+def throw_deep(message):
+ return throw_real(message)
+
+
+def throw_real(message):
+ raise Exception(message)
diff --git a/python/mach/mach/test/python.ini b/python/mach/mach/test/python.ini
new file mode 100644
index 0000000000..de09924b67
--- /dev/null
+++ b/python/mach/mach/test/python.ini
@@ -0,0 +1,22 @@
+[DEFAULT]
+subsuite = mach
+
+[test_commands.py]
+[test_conditions.py]
+skip-if = python == 3
+[test_config.py]
+[test_decorators.py]
+[test_dispatcher.py]
+[test_entry_point.py]
+[test_error_output.py]
+skip-if = python == 3
+[test_logger.py]
+[test_mach.py]
+[test_site.py]
+[test_site_activation.py]
+[test_site_compatibility.py]
+# The Windows and Mac workers only use the internal PyPI mirror,
+# which will be missing packages required for this test.
+skip-if =
+ os == "win"
+ os == "mac"
diff --git a/python/mach/mach/test/script_site_activation.py b/python/mach/mach/test/script_site_activation.py
new file mode 100644
index 0000000000..8c23f1a19c
--- /dev/null
+++ b/python/mach/mach/test/script_site_activation.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used by "test_site_activation.py" to verify how site activations
+# affect the sys.path.
+# The sys.path is printed in three stages:
+# 1. Once at the beginning
+# 2. Once after Mach site activation
+# 3. Once after the command site activation
+# The output of this script should be an ast-parsable list with three nested lists: one
+# for each sys.path state.
+# Note that virtualenv-creation output may need to be filtered out - it can be done by
+# only ast-parsing the last line of text outputted by this script.
+
+import os
+import sys
+from unittest.mock import patch
+
+from mach.requirements import MachEnvRequirements, PthSpecifier
+from mach.site import CommandSiteManager, MachSiteManager
+
+
+def main():
+ # Should be set by calling test
+ topsrcdir = os.environ["TOPSRCDIR"]
+ command_site = os.environ["COMMAND_SITE"]
+ mach_site_requirements = os.environ["MACH_SITE_PTH_REQUIREMENTS"]
+ command_site_requirements = os.environ["COMMAND_SITE_PTH_REQUIREMENTS"]
+ work_dir = os.environ["WORK_DIR"]
+
+ def resolve_requirements(topsrcdir, site_name):
+ req = MachEnvRequirements()
+ if site_name == "mach":
+ req.pth_requirements = [
+ PthSpecifier(path) for path in mach_site_requirements.split(os.pathsep)
+ ]
+ else:
+ req.pth_requirements = [PthSpecifier(command_site_requirements)]
+ return req
+
+ with patch("mach.site.resolve_requirements", resolve_requirements):
+ initial_sys_path = sys.path.copy()
+
+ mach_site = MachSiteManager.from_environment(
+ topsrcdir,
+ lambda: work_dir,
+ )
+ mach_site.activate()
+ mach_sys_path = sys.path.copy()
+
+ command_site = CommandSiteManager.from_environment(
+ topsrcdir, lambda: work_dir, command_site, work_dir
+ )
+ command_site.activate()
+ command_sys_path = sys.path.copy()
+ print(
+ [
+ initial_sys_path,
+ mach_sys_path,
+ command_sys_path,
+ ]
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_commands.py b/python/mach/mach/test/test_commands.py
new file mode 100644
index 0000000000..38191b0898
--- /dev/null
+++ b/python/mach/mach/test/test_commands.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+from pathlib import Path
+
+import pytest
+from buildconfig import topsrcdir
+from mozunit import main
+
+import mach
+
+ALL_COMMANDS = [
+ "cmd_bar",
+ "cmd_foo",
+ "cmd_foobar",
+ "mach-commands",
+ "mach-completion",
+ "mach-debug-commands",
+]
+
+
+@pytest.fixture
+def run_completion(run_mach):
+ def inner(args=[]):
+ mach_dir = Path(mach.__file__).parent
+ providers = [
+ Path("commands.py"),
+ mach_dir / "commands" / "commandinfo.py",
+ ]
+
+ def context_handler(key):
+ if key == "topdir":
+ return topsrcdir
+
+ args = ["mach-completion"] + args
+ return run_mach(args, providers, context_handler=context_handler)
+
+ return inner
+
+
+def format(targets):
+ return "\n".join(targets) + "\n"
+
+
+def test_mach_completion(run_completion):
+ result, stdout, stderr = run_completion()
+ assert result == 0
+ assert stdout == format(ALL_COMMANDS)
+
+ result, stdout, stderr = run_completion(["cmd_f"])
+ assert result == 0
+ # While it seems like this should return only commands that have
+ # 'cmd_f' as a prefix, the completion script will handle this case
+ # properly.
+ assert stdout == format(ALL_COMMANDS)
+
+ result, stdout, stderr = run_completion(["cmd_foo"])
+ assert result == 0
+ assert stdout == format(["help", "--arg"])
+
+
+@pytest.mark.parametrize("shell", ("bash", "fish", "zsh"))
+def test_generate_mach_completion_script(run_completion, shell):
+ rv, out, err = run_completion([shell])
+ print(out)
+ print(err, file=sys.stderr)
+ assert rv == 0
+ assert err == ""
+
+ assert "cmd_foo" in out
+ assert "arg" in out
+ assert "cmd_foobar" in out
+ assert "cmd_bar" in out
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py
new file mode 100644
index 0000000000..5775790e69
--- /dev/null
+++ b/python/mach/mach/test/test_conditions.py
@@ -0,0 +1,101 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from pathlib import Path
+
+from buildconfig import topsrcdir
+from mozunit import main
+
+from mach.base import MachError
+from mach.main import Mach
+from mach.registrar import Registrar
+from mach.test.conftest import PROVIDER_DIR, TestBase
+
+
+def _make_populate_context(include_extra_attributes):
+ def _populate_context(key=None):
+ if key is None:
+ return
+
+ attributes = {
+ "topdir": topsrcdir,
+ }
+ if include_extra_attributes:
+ attributes["foo"] = True
+ attributes["bar"] = False
+
+ try:
+ return attributes[key]
+ except KeyError:
+ raise AttributeError(key)
+
+ return _populate_context
+
+
+_populate_bare_context = _make_populate_context(False)
+_populate_context = _make_populate_context(True)
+
+
+class TestConditions(TestBase):
+ """Tests for conditionally filtering commands."""
+
+ def _run(self, args, context_handler=_populate_bare_context):
+ return self._run_mach(
+ args, Path("conditions.py"), context_handler=context_handler
+ )
+
+ def test_conditions_pass(self):
+ """Test that a command which passes its conditions is runnable."""
+
+ self.assertEqual((0, "", ""), self._run(["cmd_condition_true"]))
+ self.assertEqual((0, "", ""), self._run(["cmd_foo_ctx"], _populate_context))
+
+ def test_invalid_context_message(self):
+ """Test that commands which do not pass all their conditions
+ print the proper failure message."""
+
+ def is_bar():
+ """Bar must be true"""
+
+ fail_conditions = [is_bar]
+
+ for name in ("cmd_condition_false", "cmd_condition_true_and_false"):
+ result, stdout, stderr = self._run([name])
+ self.assertEqual(1, result)
+
+ fail_msg = Registrar._condition_failed_message(name, fail_conditions)
+ self.assertEqual(fail_msg.rstrip(), stdout.rstrip())
+
+ for name in ("cmd_bar_ctx", "cmd_foobar_ctx"):
+ result, stdout, stderr = self._run([name], _populate_context)
+ self.assertEqual(1, result)
+
+ fail_msg = Registrar._condition_failed_message(name, fail_conditions)
+ self.assertEqual(fail_msg.rstrip(), stdout.rstrip())
+
+ def test_invalid_type(self):
+ """Test that a condition which is not callable raises an exception."""
+
+ m = Mach(str(Path.cwd()))
+ m.define_category("testing", "Mach unittest", "Testing for mach core", 10)
+ self.assertRaises(
+ MachError,
+ m.load_commands_from_file,
+ PROVIDER_DIR / "conditions_invalid.py",
+ )
+
+ def test_help_message(self):
+ """Test that commands that are not runnable do not show up in help."""
+
+ result, stdout, stderr = self._run(["help"], _populate_context)
+ self.assertIn("cmd_condition_true", stdout)
+ self.assertNotIn("cmd_condition_false", stdout)
+ self.assertNotIn("cmd_condition_true_and_false", stdout)
+ self.assertIn("cmd_foo_ctx", stdout)
+ self.assertNotIn("cmd_bar_ctx", stdout)
+ self.assertNotIn("cmd_foobar_ctx", stdout)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py
new file mode 100644
index 0000000000..25b75c8685
--- /dev/null
+++ b/python/mach/mach/test/test_config.py
@@ -0,0 +1,292 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import sys
+import unittest
+from pathlib import Path
+
+from mozfile.mozfile import NamedTemporaryFile
+from mozunit import main
+from six import string_types
+
+from mach.config import (
+ BooleanType,
+ ConfigException,
+ ConfigSettings,
+ IntegerType,
+ PathType,
+ PositiveIntegerType,
+ StringType,
+)
+from mach.decorators import SettingsProvider
+
+CONFIG1 = r"""
+[foo]
+
+bar = bar_value
+baz = /baz/foo.c
+"""
+
+CONFIG2 = r"""
+[foo]
+
+bar = value2
+"""
+
+
+@SettingsProvider
+class Provider1(object):
+ config_settings = [
+ ("foo.bar", StringType, "desc"),
+ ("foo.baz", PathType, "desc"),
+ ]
+
+
+@SettingsProvider
+class ProviderDuplicate(object):
+ config_settings = [
+ ("dupesect.foo", StringType, "desc"),
+ ("dupesect.foo", StringType, "desc"),
+ ]
+
+
+@SettingsProvider
+class Provider2(object):
+ config_settings = [
+ ("a.string", StringType, "desc"),
+ ("a.boolean", BooleanType, "desc"),
+ ("a.pos_int", PositiveIntegerType, "desc"),
+ ("a.int", IntegerType, "desc"),
+ ("a.path", PathType, "desc"),
+ ]
+
+
+@SettingsProvider
+class Provider3(object):
+ @classmethod
+ def config_settings(cls):
+ return [
+ ("a.string", "string", "desc"),
+ ("a.boolean", "boolean", "desc"),
+ ("a.pos_int", "pos_int", "desc"),
+ ("a.int", "int", "desc"),
+ ("a.path", "path", "desc"),
+ ]
+
+
+@SettingsProvider
+class Provider4(object):
+ config_settings = [
+ ("foo.abc", StringType, "desc", "a", {"choices": set("abc")}),
+ ("foo.xyz", StringType, "desc", "w", {"choices": set("xyz")}),
+ ]
+
+
+@SettingsProvider
+class Provider5(object):
+ config_settings = [
+ ("foo.*", "string", "desc"),
+ ("foo.bar", "string", "desc"),
+ ]
+
+
+class TestConfigSettings(unittest.TestCase):
+ def test_empty(self):
+ s = ConfigSettings()
+
+ self.assertEqual(len(s), 0)
+ self.assertNotIn("foo", s)
+
+ def test_duplicate_option(self):
+ s = ConfigSettings()
+
+ with self.assertRaises(ConfigException):
+ s.register_provider(ProviderDuplicate)
+
+ def test_simple(self):
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ self.assertEqual(len(s), 1)
+ self.assertIn("foo", s)
+
+ foo = s["foo"]
+ foo = s.foo
+
+ self.assertEqual(len(foo), 0)
+ self.assertEqual(len(foo._settings), 2)
+
+ self.assertIn("bar", foo._settings)
+ self.assertIn("baz", foo._settings)
+
+ self.assertNotIn("bar", foo)
+ foo["bar"] = "value1"
+ self.assertIn("bar", foo)
+
+ self.assertEqual(foo["bar"], "value1")
+ self.assertEqual(foo.bar, "value1")
+
+ def test_assignment_validation(self):
+ s = ConfigSettings()
+ s.register_provider(Provider2)
+
+ a = s.a
+
+ # Assigning an undeclared setting raises.
+ exc_type = AttributeError if sys.version_info < (3, 0) else KeyError
+ with self.assertRaises(exc_type):
+ a.undefined = True
+
+ with self.assertRaises(KeyError):
+ a["undefined"] = True
+
+ # Basic type validation.
+ a.string = "foo"
+ a.string = "foo"
+
+ with self.assertRaises(TypeError):
+ a.string = False
+
+ a.boolean = True
+ a.boolean = False
+
+ with self.assertRaises(TypeError):
+ a.boolean = "foo"
+
+ a.pos_int = 5
+ a.pos_int = 0
+
+ with self.assertRaises(ValueError):
+ a.pos_int = -1
+
+ with self.assertRaises(TypeError):
+ a.pos_int = "foo"
+
+ a.int = 5
+ a.int = 0
+ a.int = -5
+
+ with self.assertRaises(TypeError):
+ a.int = 1.24
+
+ with self.assertRaises(TypeError):
+ a.int = "foo"
+
+ a.path = "/home/gps"
+ a.path = "foo.c"
+ a.path = "foo/bar"
+ a.path = "./foo"
+
+ def retrieval_type_helper(self, provider):
+ s = ConfigSettings()
+ s.register_provider(provider)
+
+ a = s.a
+
+ a.string = "foo"
+ a.boolean = True
+ a.pos_int = 12
+ a.int = -4
+ a.path = "./foo/bar"
+
+ self.assertIsInstance(a.string, string_types)
+ self.assertIsInstance(a.boolean, bool)
+ self.assertIsInstance(a.pos_int, int)
+ self.assertIsInstance(a.int, int)
+ self.assertIsInstance(a.path, string_types)
+
+ def test_retrieval_type(self):
+ self.retrieval_type_helper(Provider2)
+ self.retrieval_type_helper(Provider3)
+
+ def test_choices_validation(self):
+ s = ConfigSettings()
+ s.register_provider(Provider4)
+
+ foo = s.foo
+ foo.abc
+ with self.assertRaises(ValueError):
+ foo.xyz
+
+ with self.assertRaises(ValueError):
+ foo.abc = "e"
+
+ foo.abc = "b"
+ foo.xyz = "y"
+
+ def test_wildcard_options(self):
+ s = ConfigSettings()
+ s.register_provider(Provider5)
+
+ foo = s.foo
+
+ self.assertIn("*", foo._settings)
+ self.assertNotIn("*", foo)
+
+ foo.baz = "value1"
+ foo.bar = "value2"
+
+ self.assertIn("baz", foo)
+ self.assertEqual(foo.baz, "value1")
+
+ self.assertIn("bar", foo)
+ self.assertEqual(foo.bar, "value2")
+
+ def test_file_reading_single(self):
+ temp = NamedTemporaryFile(mode="wt")
+ temp.write(CONFIG1)
+ temp.flush()
+
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ s.load_file(Path(temp.name))
+
+ self.assertEqual(s.foo.bar, "bar_value")
+
+ def test_file_reading_multiple(self):
+ """Loading multiple files has proper overwrite behavior."""
+ temp1 = NamedTemporaryFile(mode="wt")
+ temp1.write(CONFIG1)
+ temp1.flush()
+
+ temp2 = NamedTemporaryFile(mode="wt")
+ temp2.write(CONFIG2)
+ temp2.flush()
+
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ s.load_files([Path(temp1.name), Path(temp2.name)])
+
+ self.assertEqual(s.foo.bar, "value2")
+
+ def test_file_reading_missing(self):
+ """Missing files should silently be ignored."""
+
+ s = ConfigSettings()
+
+ s.load_file("/tmp/foo.ini")
+
+ def test_file_writing(self):
+ s = ConfigSettings()
+ s.register_provider(Provider2)
+
+ s.a.string = "foo"
+ s.a.boolean = False
+
+ temp = NamedTemporaryFile("wt")
+ s.write(temp)
+ temp.flush()
+
+ s2 = ConfigSettings()
+ s2.register_provider(Provider2)
+
+ s2.load_file(temp.name)
+
+ self.assertEqual(s.a.string, s2.a.string)
+ self.assertEqual(s.a.boolean, s2.a.boolean)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_decorators.py b/python/mach/mach/test/test_decorators.py
new file mode 100644
index 0000000000..f33b6e7d8f
--- /dev/null
+++ b/python/mach/mach/test/test_decorators.py
@@ -0,0 +1,133 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from pathlib import Path
+from unittest import mock
+from unittest.mock import Mock, patch
+
+import pytest
+from mozbuild.base import MachCommandBase
+from mozunit import main
+
+import mach.decorators
+import mach.registrar
+from mach.base import MachError
+from mach.decorators import Command, CommandArgument, SubCommand
+from mach.requirements import MachEnvRequirements
+from mach.site import CommandSiteManager, MozSiteMetadata, SitePackagesSource
+
+
+@pytest.fixture
+def registrar(monkeypatch):
+ test_registrar = mach.registrar.MachRegistrar()
+ test_registrar.register_category(
+ "testing", "Mach unittest", "Testing for mach decorators"
+ )
+ monkeypatch.setattr(mach.decorators, "Registrar", test_registrar)
+ return test_registrar
+
+
+def test_register_command_with_argument(registrar):
+ inner_function = Mock()
+ context = Mock()
+ context.cwd = "."
+
+ @Command("cmd_foo", category="testing")
+ @CommandArgument("--arg", default=None, help="Argument help.")
+ def run_foo(command_context, arg):
+ inner_function(arg)
+
+ registrar.dispatch("cmd_foo", context, arg="argument")
+
+ inner_function.assert_called_with("argument")
+
+
+def test_register_command_with_metrics_path(registrar):
+ context = Mock()
+ context.cwd = "."
+
+ metrics_path = "metrics/path"
+ metrics_mock = Mock()
+ context.telemetry.metrics.return_value = metrics_mock
+
+ @Command("cmd_foo", category="testing", metrics_path=metrics_path)
+ def run_foo(command_context):
+ assert command_context.metrics == metrics_mock
+
+ @SubCommand("cmd_foo", "sub_foo", metrics_path=metrics_path + "2")
+ def run_subfoo(command_context):
+ assert command_context.metrics == metrics_mock
+
+ registrar.dispatch("cmd_foo", context)
+
+ context.telemetry.metrics.assert_called_with(metrics_path)
+ assert context.handler.metrics_path == metrics_path
+
+ registrar.dispatch("cmd_foo", context, subcommand="sub_foo")
+ assert context.handler.metrics_path == metrics_path + "2"
+
+
+def test_register_command_sets_up_class_at_runtime(registrar):
+ inner_function = Mock()
+
+ context = Mock()
+ context.cwd = "."
+
+ # We test that the virtualenv is set up properly dynamically on
+ # the instance that actually runs the command.
+ @Command("cmd_foo", category="testing", virtualenv_name="env_foo")
+ def run_foo(command_context):
+ assert (
+ Path(command_context.virtualenv_manager.virtualenv_root).name == "env_foo"
+ )
+ inner_function("foo")
+
+ @Command("cmd_bar", category="testing", virtualenv_name="env_bar")
+ def run_bar(command_context):
+ assert (
+ Path(command_context.virtualenv_manager.virtualenv_root).name == "env_bar"
+ )
+ inner_function("bar")
+
+ def from_environment_patch(
+ topsrcdir: str, state_dir: str, virtualenv_name, directory: str
+ ):
+ return CommandSiteManager(
+ "",
+ "",
+ virtualenv_name,
+ virtualenv_name,
+ MozSiteMetadata(0, "mach", SitePackagesSource.VENV, "", ""),
+ True,
+ MachEnvRequirements(),
+ )
+
+ with mock.patch.object(
+ CommandSiteManager, "from_environment", from_environment_patch
+ ):
+ with patch.object(MachCommandBase, "activate_virtualenv"):
+ registrar.dispatch("cmd_foo", context)
+ inner_function.assert_called_with("foo")
+ registrar.dispatch("cmd_bar", context)
+ inner_function.assert_called_with("bar")
+
+
+def test_cannot_create_command_nonexisting_category(registrar):
+ with pytest.raises(MachError):
+
+ @Command("cmd_foo", category="bar")
+ def run_foo(command_context):
+ pass
+
+
+def test_subcommand_requires_parent_to_exist(registrar):
+ with pytest.raises(MachError):
+
+ @SubCommand("sub_foo", "foo")
+ def run_foo(command_context):
+ pass
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_dispatcher.py b/python/mach/mach/test/test_dispatcher.py
new file mode 100644
index 0000000000..85c2e9a847
--- /dev/null
+++ b/python/mach/mach/test/test_dispatcher.py
@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from io import StringIO
+from pathlib import Path
+
+import pytest
+from mozunit import main
+from six import string_types
+
+from mach.base import CommandContext
+from mach.registrar import Registrar
+
+
+@pytest.mark.usefixtures("get_mach", "run_mach")
+class TestDispatcher(unittest.TestCase):
+ """Tests dispatch related code"""
+
+ def get_parser(self, config=None):
+ mach = self.get_mach(Path("basic.py"))
+
+ for provider in Registrar.settings_providers:
+ mach.settings.register_provider(provider)
+
+ if config:
+ if isinstance(config, string_types):
+ config = StringIO(config)
+ mach.settings.load_fps([config])
+
+ context = CommandContext(cwd="", settings=mach.settings)
+ return mach.get_argument_parser(context)
+
+ def test_command_aliases(self):
+ config = """
+[alias]
+foo = cmd_foo
+bar = cmd_bar
+baz = cmd_bar --baz
+cmd_bar = cmd_bar --baz
+"""
+ parser = self.get_parser(config=config)
+
+ args = parser.parse_args(["foo"])
+ self.assertEqual(args.command, "cmd_foo")
+
+ def assert_bar_baz(argv):
+ args = parser.parse_args(argv)
+ self.assertEqual(args.command, "cmd_bar")
+ self.assertTrue(args.command_args.baz)
+
+ # The following should all result in |cmd_bar --baz|
+ assert_bar_baz(["bar", "--baz"])
+ assert_bar_baz(["baz"])
+ assert_bar_baz(["cmd_bar"])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py
new file mode 100644
index 0000000000..1129eba476
--- /dev/null
+++ b/python/mach/mach/test/test_entry_point.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import imp
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+from mozunit import main
+
+from mach.base import MachError
+from mach.test.conftest import TestBase
+
+
+class Entry:
+ """Stub replacement for pkg_resources.EntryPoint"""
+
+ def __init__(self, providers):
+ self.providers = providers
+
+ def load(self):
+ def _providers():
+ return self.providers
+
+ return _providers
+
+
+class TestEntryPoints(TestBase):
+ """Test integrating with setuptools entry points"""
+
+ provider_dir = Path(__file__).parent.resolve() / "providers"
+
+ def _run_help(self):
+ return self._run_mach(["help"], entry_point="mach.providers")
+
+ @patch("pkg_resources.iter_entry_points")
+ def test_load_entry_point_from_directory(self, mock):
+ # Ensure parent module is present otherwise we'll (likely) get
+ # an error due to unknown parent.
+ if "mach.commands" not in sys.modules:
+ mod = imp.new_module("mach.commands")
+ sys.modules["mach.commands"] = mod
+
+ mock.return_value = [Entry([self.provider_dir])]
+ # Mach error raised due to conditions_invalid.py
+ with self.assertRaises(MachError):
+ self._run_help()
+
+ @patch("pkg_resources.iter_entry_points")
+ def test_load_entry_point_from_file(self, mock):
+ mock.return_value = [Entry([self.provider_dir / "basic.py"])]
+
+ result, stdout, stderr = self._run_help()
+ self.assertIsNone(result)
+ self.assertIn("cmd_foo", stdout)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py
new file mode 100644
index 0000000000..12eab65856
--- /dev/null
+++ b/python/mach/mach/test/test_error_output.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from pathlib import Path
+
+from mozunit import main
+
+from mach.main import COMMAND_ERROR_TEMPLATE, MODULE_ERROR_TEMPLATE
+
+
+def test_command_error(run_mach):
+ result, stdout, stderr = run_mach(
+ ["throw", "--message", "Command Error"], provider_files=Path("throw.py")
+ )
+ assert result == 1
+ assert COMMAND_ERROR_TEMPLATE % "throw" in stdout
+
+
+def test_invoked_error(run_mach):
+ result, stdout, stderr = run_mach(
+ ["throw_deep", "--message", "Deep stack"], provider_files=Path("throw.py")
+ )
+ assert result == 1
+ assert MODULE_ERROR_TEMPLATE % "throw_deep" in stdout
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py
new file mode 100644
index 0000000000..643d890de8
--- /dev/null
+++ b/python/mach/mach/test/test_logger.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import time
+import unittest
+
+from mozunit import main
+
+from mach.logging import StructuredHumanFormatter
+
+
+class DummyLogger(logging.Logger):
+ def __init__(self, cb):
+ logging.Logger.__init__(self, "test")
+
+ self._cb = cb
+
+ def handle(self, record):
+ self._cb(record)
+
+
+class TestStructuredHumanFormatter(unittest.TestCase):
+ def test_non_ascii_logging(self):
+ # Ensures the formatter doesn't choke when non-ASCII characters are
+ # present in printed parameters.
+ formatter = StructuredHumanFormatter(time.time())
+
+ def on_record(record):
+ result = formatter.format(record)
+ relevant = result[9:]
+
+ self.assertEqual(relevant, "Test: s\xe9curit\xe9")
+
+ logger = DummyLogger(on_record)
+
+ value = "s\xe9curit\xe9"
+
+ logger.log(
+ logging.INFO,
+ "Test: {utf}",
+ extra={"action": "action", "params": {"utf": value}},
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_mach.py b/python/mach/mach/test/test_mach.py
new file mode 100644
index 0000000000..38379d1b49
--- /dev/null
+++ b/python/mach/mach/test/test_mach.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+from mozunit import main
+
+
+def test_set_isatty_environ(monkeypatch, get_mach):
+ # Make sure the 'MACH_STDOUT_ISATTY' variable gets set.
+ monkeypatch.delenv("MACH_STDOUT_ISATTY", raising=False)
+ monkeypatch.setattr(os, "isatty", lambda fd: True)
+
+ m = get_mach()
+ orig_run = m._run
+ env_is_set = []
+
+ def wrap_run(*args, **kwargs):
+ env_is_set.append("MACH_STDOUT_ISATTY" in os.environ)
+ return orig_run(*args, **kwargs)
+
+ monkeypatch.setattr(m, "_run", wrap_run)
+
+ ret = m.run([])
+ assert ret == 0
+ assert env_is_set[0]
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_site.py b/python/mach/mach/test/test_site.py
new file mode 100644
index 0000000000..d7c3d8c489
--- /dev/null
+++ b/python/mach/mach/test/test_site.py
@@ -0,0 +1,56 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from unittest import mock
+
+import pytest
+from buildconfig import topsrcdir
+from mozunit import main
+
+from mach.site import (
+ PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
+ SitePackagesSource,
+ resolve_requirements,
+)
+
+
+@pytest.mark.parametrize(
+ "env_native_package_source,env_use_system_python,env_moz_automation,expected",
+ [
+ ("system", False, False, SitePackagesSource.SYSTEM),
+ ("pip", False, False, SitePackagesSource.VENV),
+ ("none", False, False, SitePackagesSource.NONE),
+ (None, False, False, SitePackagesSource.VENV),
+ (None, False, True, SitePackagesSource.NONE),
+ (None, True, False, SitePackagesSource.NONE),
+ (None, True, True, SitePackagesSource.NONE),
+ ],
+)
+def test_resolve_package_source(
+ env_native_package_source, env_use_system_python, env_moz_automation, expected
+):
+ with mock.patch.dict(
+ os.environ,
+ {
+ "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": env_native_package_source or "",
+ "MACH_USE_SYSTEM_PYTHON": "1" if env_use_system_python else "",
+ "MOZ_AUTOMATION": "1" if env_moz_automation else "",
+ },
+ ):
+ assert SitePackagesSource.for_mach() == expected
+
+
+def test_all_restricted_sites_dont_have_pypi_requirements():
+ for site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS:
+ requirements = resolve_requirements(topsrcdir, site_name)
+ assert not requirements.pypi_requirements, (
+ 'Sites that must be able to operate without "pip install" must not have any '
+ f'mandatory "pypi requirements". However, the "{site_name}" site depends on: '
+ f"{requirements.pypi_requirements}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mach/mach/test/test_site_activation.py b/python/mach/mach/test/test_site_activation.py
new file mode 100644
index 0000000000..e034a27b76
--- /dev/null
+++ b/python/mach/mach/test/test_site_activation.py
@@ -0,0 +1,463 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import ast
+import functools
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from subprocess import CompletedProcess
+from typing import List
+
+import buildconfig
+import mozunit
+import pkg_resources
+import pytest
+
+from mach.site import MozSiteMetadata, PythonVirtualenv, activate_virtualenv
+
+
+class ActivationContext:
+ def __init__(
+ self,
+ topsrcdir: Path,
+ work_dir: Path,
+ original_python_path: str,
+ stdlib_paths: List[Path],
+ system_paths: List[Path],
+ required_mach_sys_paths: List[Path],
+ mach_requirement_paths: List[Path],
+ command_requirement_path: Path,
+ ):
+ self.topsrcdir = topsrcdir
+ self.work_dir = work_dir
+ self.original_python_path = original_python_path
+ self.stdlib_paths = stdlib_paths
+ self.system_paths = system_paths
+ self.required_moz_init_sys_paths = required_mach_sys_paths
+ self.mach_requirement_paths = mach_requirement_paths
+ self.command_requirement_path = command_requirement_path
+
+ def virtualenv(self, name: str) -> PythonVirtualenv:
+ base_path = self.work_dir
+
+ if name == "mach":
+ base_path = base_path / "_virtualenvs"
+ return PythonVirtualenv(str(base_path / name))
+
+
+def test_new_package_appears_in_pkg_resources():
+ try:
+ # "carrot" was chosen as the package to use because:
+ # * It has to be a package that doesn't exist in-scope at the start (so,
+ # all vendored modules included in the test virtualenv aren't usage).
+ # * It must be on our internal PyPI mirror.
+ # Of the options, "carrot" is a small install that fits these requirements.
+ pkg_resources.get_distribution("carrot")
+ assert False, "Expected to not find 'carrot' as the initial state of the test"
+ except pkg_resources.DistributionNotFound:
+ pass
+
+ with tempfile.TemporaryDirectory() as venv_dir:
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "venv",
+ venv_dir,
+ ]
+ )
+
+ venv = PythonVirtualenv(venv_dir)
+ venv.pip_install(["carrot==0.10.7"])
+
+ initial_metadata = MozSiteMetadata.from_runtime()
+ try:
+ metadata = MozSiteMetadata(None, None, None, None, venv.prefix)
+ with metadata.update_current_site(venv.python_path):
+ activate_virtualenv(venv)
+
+ assert pkg_resources.get_distribution("carrot").version == "0.10.7"
+ finally:
+ MozSiteMetadata.current = initial_metadata
+
+
+def test_sys_path_source_none_build(context):
+ original, mach, command = _run_activation_script_for_paths(context, "none", "build")
+ _assert_original_python_sys_path(context, original)
+
+ assert not os.path.exists(context.virtualenv("mach").prefix)
+ assert mach == [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ ]
+
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ context.command_requirement_path,
+ ]
+ assert command == expected_command_paths
+
+ command_venv = _sys_path_of_virtualenv(context.virtualenv("build"))
+ assert command_venv == [Path(""), *expected_command_paths]
+
+
+def test_sys_path_source_none_other(context):
+ original, mach, command = _run_activation_script_for_paths(context, "none", "other")
+ _assert_original_python_sys_path(context, original)
+
+ assert not os.path.exists(context.virtualenv("mach").prefix)
+ assert mach == [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ ]
+
+ command_virtualenv = PythonVirtualenv(str(context.work_dir / "other"))
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ context.command_requirement_path,
+ *(Path(p) for p in command_virtualenv.site_packages_dirs()),
+ ]
+ assert command == expected_command_paths
+
+ command_venv = _sys_path_of_virtualenv(context.virtualenv("other"))
+ assert command_venv == [
+ Path(""),
+ *expected_command_paths,
+ ]
+
+
+def test_sys_path_source_venv_build(context):
+ original, mach, command = _run_activation_script_for_paths(context, "pip", "build")
+ _assert_original_python_sys_path(context, original)
+
+ mach_virtualenv = context.virtualenv("mach")
+ expected_mach_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *(Path(p) for p in mach_virtualenv.site_packages_dirs()),
+ ]
+ assert mach == expected_mach_paths
+
+ command_virtualenv = context.virtualenv("build")
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *(Path(p) for p in mach_virtualenv.site_packages_dirs()),
+ context.command_requirement_path,
+ *(Path(p) for p in command_virtualenv.site_packages_dirs()),
+ ]
+ assert command == expected_command_paths
+
+ mach_venv = _sys_path_of_virtualenv(mach_virtualenv)
+ assert mach_venv == [
+ Path(""),
+ *expected_mach_paths,
+ ]
+
+ command_venv = _sys_path_of_virtualenv(command_virtualenv)
+ assert command_venv == [
+ Path(""),
+ *expected_command_paths,
+ ]
+
+
+def test_sys_path_source_venv_other(context):
+ original, mach, command = _run_activation_script_for_paths(context, "pip", "other")
+ _assert_original_python_sys_path(context, original)
+
+ mach_virtualenv = context.virtualenv("mach")
+ expected_mach_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *(Path(p) for p in mach_virtualenv.site_packages_dirs()),
+ ]
+ assert mach == expected_mach_paths
+
+ command_virtualenv = context.virtualenv("other")
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *(Path(p) for p in mach_virtualenv.site_packages_dirs()),
+ context.command_requirement_path,
+ *(Path(p) for p in command_virtualenv.site_packages_dirs()),
+ ]
+ assert command == expected_command_paths
+
+ mach_venv = _sys_path_of_virtualenv(mach_virtualenv)
+ assert mach_venv == [
+ Path(""),
+ *expected_mach_paths,
+ ]
+
+ command_venv = _sys_path_of_virtualenv(command_virtualenv)
+ assert command_venv == [
+ Path(""),
+ *expected_command_paths,
+ ]
+
+
+def test_sys_path_source_system_build(context):
+ original, mach, command = _run_activation_script_for_paths(
+ context, "system", "build"
+ )
+ _assert_original_python_sys_path(context, original)
+
+ assert not os.path.exists(context.virtualenv("mach").prefix)
+ expected_mach_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *context.system_paths,
+ ]
+ assert mach == expected_mach_paths
+
+ command_virtualenv = context.virtualenv("build")
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *context.system_paths,
+ context.command_requirement_path,
+ ]
+ assert command == expected_command_paths
+
+ command_venv = _sys_path_of_virtualenv(command_virtualenv)
+ assert command_venv == [
+ Path(""),
+ *expected_command_paths,
+ ]
+
+
+def test_sys_path_source_system_other(context):
+ result = _run_activation_script(
+ context,
+ "system",
+ "other",
+ context.original_python_path,
+ stderr=subprocess.PIPE,
+ )
+ assert result.returncode != 0
+ assert (
+ 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites '
+ "other than" in result.stderr
+ )
+
+
+def test_sys_path_source_venvsystem_build(context):
+ venv_system_python = _create_venv_system_python(
+ context.work_dir, context.original_python_path
+ )
+ venv_system_site_packages_dirs = [
+ Path(p) for p in venv_system_python.site_packages_dirs()
+ ]
+ original, mach, command = _run_activation_script_for_paths(
+ context, "system", "build", venv_system_python.python_path
+ )
+
+ assert original == [
+ Path(__file__).parent,
+ *context.required_moz_init_sys_paths,
+ *context.stdlib_paths,
+ *venv_system_site_packages_dirs,
+ ]
+
+ assert not os.path.exists(context.virtualenv("mach").prefix)
+ expected_mach_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *venv_system_site_packages_dirs,
+ ]
+ assert mach == expected_mach_paths
+
+ command_virtualenv = context.virtualenv("build")
+ expected_command_paths = [
+ *context.stdlib_paths,
+ *context.mach_requirement_paths,
+ *venv_system_site_packages_dirs,
+ context.command_requirement_path,
+ ]
+ assert command == expected_command_paths
+
+ command_venv = _sys_path_of_virtualenv(command_virtualenv)
+ assert command_venv == [
+ Path(""),
+ *expected_command_paths,
+ ]
+
+
+def test_sys_path_source_venvsystem_other(context):
+ venv_system_python = _create_venv_system_python(
+ context.work_dir, context.original_python_path
+ )
+ result = _run_activation_script(
+ context,
+ "system",
+ "other",
+ venv_system_python.python_path,
+ stderr=subprocess.PIPE,
+ )
+ assert result.returncode != 0
+ assert (
+ 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites '
+ "other than" in result.stderr
+ )
+
+
+@pytest.fixture(name="context")
+def _activation_context():
+ original_python_path, stdlib_paths, system_paths = _original_python()
+ topsrcdir = Path(buildconfig.topsrcdir)
+ required_mach_sys_paths = [
+ topsrcdir / "python" / "mach",
+ topsrcdir / "third_party" / "python" / "packaging",
+ topsrcdir / "third_party" / "python" / "pyparsing",
+ topsrcdir / "third_party" / "python" / "pip",
+ ]
+
+ with tempfile.TemporaryDirectory() as work_dir:
+ # Get "resolved" version of path to ease comparison against "site"-added sys.path
+ # entries, as "site" calculates the realpath of provided locations.
+ work_dir = Path(work_dir).resolve()
+ mach_requirement_paths = [
+ *required_mach_sys_paths,
+ work_dir / "mach_site_path",
+ ]
+ command_requirement_path = work_dir / "command_site_path"
+ (work_dir / "mach_site_path").touch()
+ command_requirement_path.touch()
+ yield ActivationContext(
+ topsrcdir,
+ work_dir,
+ original_python_path,
+ stdlib_paths,
+ system_paths,
+ required_mach_sys_paths,
+ mach_requirement_paths,
+ command_requirement_path,
+ )
+
+
+@functools.lru_cache(maxsize=None)
+def _original_python():
+ current_site = MozSiteMetadata.from_runtime()
+ stdlib_paths, system_paths = current_site.original_python.sys_path()
+ stdlib_paths = [Path(path) for path in _filter_pydev_from_paths(stdlib_paths)]
+ system_paths = [Path(path) for path in system_paths]
+ return current_site.original_python.python_path, stdlib_paths, system_paths
+
+
+def _run_activation_script(
+ context: ActivationContext,
+ source: str,
+ site_name: str,
+ invoking_python: str,
+ **kwargs
+) -> CompletedProcess:
+ return subprocess.run(
+ [
+ invoking_python,
+ str(Path(__file__).parent / "script_site_activation.py"),
+ ],
+ stdout=subprocess.PIPE,
+ universal_newlines=True,
+ env={
+ "TOPSRCDIR": str(context.topsrcdir),
+ "COMMAND_SITE": site_name,
+ "PYTHONPATH": os.pathsep.join(
+ str(p) for p in context.required_moz_init_sys_paths
+ ),
+ "MACH_SITE_PTH_REQUIREMENTS": os.pathsep.join(
+ str(p) for p in context.mach_requirement_paths
+ ),
+ "COMMAND_SITE_PTH_REQUIREMENTS": str(context.command_requirement_path),
+ "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": source,
+ "WORK_DIR": str(context.work_dir),
+ # These two variables are needed on Windows so that Python initializes
+ # properly and adds the "user site packages" to the sys.path like normal.
+ "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""),
+ "APPDATA": os.environ.get("APPDATA", ""),
+ },
+ **kwargs,
+ )
+
+
+def _run_activation_script_for_paths(
+ context: ActivationContext, source: str, site_name: str, invoking_python: str = None
+) -> List[List[Path]]:
+ """Return the states of the sys.path when activating Mach-managed sites
+
+ Three sys.path states are returned:
+ * The initial sys.path, equivalent to "path_to_python -c "import sys; print(sys.path)"
+ * The sys.path after activating the Mach site
+ * The sys.path after activating the command site
+ """
+
+ output = _run_activation_script(
+ context,
+ source,
+ site_name,
+ invoking_python or context.original_python_path,
+ check=True,
+ ).stdout
+ # Filter to the last line, which will have our nested list that we want to
+ # parse. This will avoid unrelated output, such as from virtualenv creation
+ output = output.splitlines()[-1]
+ return [
+ [Path(path) for path in _filter_pydev_from_paths(paths)]
+ for paths in ast.literal_eval(output)
+ ]
+
+
+def _assert_original_python_sys_path(context: ActivationContext, original: List[Path]):
+ # Assert that initial sys.path (prior to any activations) matches expectations.
+ assert original == [
+ Path(__file__).parent,
+ *context.required_moz_init_sys_paths,
+ *context.stdlib_paths,
+ *context.system_paths,
+ ]
+
+
+def _sys_path_of_virtualenv(virtualenv: PythonVirtualenv) -> List[Path]:
+ output = subprocess.run(
+ [virtualenv.python_path, "-c", "import sys; print(sys.path)"],
+ stdout=subprocess.PIPE,
+ universal_newlines=True,
+ env={
+ # Needed for python to initialize properly
+ "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""),
+ },
+ check=True,
+ ).stdout
+ return [Path(path) for path in _filter_pydev_from_paths(ast.literal_eval(output))]
+
+
+def _filter_pydev_from_paths(paths: List[str]) -> List[str]:
+ # Filter out injected "pydev" debugging tool if running within a JetBrains
+ # debugging context.
+ return [path for path in paths if "pydev" not in path and "JetBrains" not in path]
+
+
+def _create_venv_system_python(
+ work_dir: Path, invoking_python: str
+) -> PythonVirtualenv:
+ virtualenv = PythonVirtualenv(str(work_dir / "system_python"))
+ subprocess.run(
+ [
+ invoking_python,
+ "-m",
+ "venv",
+ virtualenv.prefix,
+ "--without-pip",
+ ],
+ check=True,
+ )
+ return virtualenv
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mach/mach/test/test_site_compatibility.py b/python/mach/mach/test/test_site_compatibility.py
new file mode 100644
index 0000000000..4c1b6d5efa
--- /dev/null
+++ b/python/mach/mach/test/test_site_compatibility.py
@@ -0,0 +1,189 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from textwrap import dedent
+
+import mozunit
+from buildconfig import topsrcdir
+
+from mach.requirements import MachEnvRequirements
+from mach.site import PythonVirtualenv
+
+
+def _resolve_command_site_names():
+ site_names = []
+ for child in (Path(topsrcdir) / "python" / "sites").iterdir():
+ if not child.is_file():
+ continue
+
+ if child.suffix != ".txt":
+ continue
+
+ if child.name == "mach.txt":
+ continue
+
+ site_names.append(child.stem)
+ return site_names
+
+
+def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env):
+ """Convert from parsed requirements object to pip-consumable format"""
+ requirements_path = Path(topsrcdir) / "python" / "sites" / f"{site_name}.txt"
+ requirements = MachEnvRequirements.from_requirements_definition(
+ topsrcdir, False, not is_mach_or_build_env, requirements_path
+ )
+
+ lines = []
+ for pypi in (
+ requirements.pypi_requirements + requirements.pypi_optional_requirements
+ ):
+ lines.append(str(pypi.requirement))
+
+ for vendored in requirements.vendored_requirements:
+ lines.append(str(cache.package_for_vendor_dir(Path(vendored.path))))
+
+ for pth in requirements.pth_requirements:
+ path = Path(pth.path)
+
+ if "third_party" not in (p.name for p in path.parents):
+ continue
+
+ for child in path.iterdir():
+ if child.name.endswith(".dist-info"):
+ raise Exception(
+ f'In {requirements_path}, the "pth:" pointing to "{path}" has a '
+ '".dist-info" file.\n'
+ 'Perhaps it should change to start with "vendored:" instead of '
+ '"pth:".'
+ )
+ if child.name.endswith(".egg-info"):
+ raise Exception(
+ f'In {requirements_path}, the "pth:" pointing to "{path}" has an '
+ '".egg-info" file.\n'
+ 'Perhaps it should change to start with "vendored:" instead of '
+ '"pth:".'
+ )
+
+ return "\n".join(lines)
+
+
+class PackageCache:
+ def __init__(self, storage_dir: Path):
+ self._cache = {}
+ self._storage_dir = storage_dir
+
+ def package_for_vendor_dir(self, vendor_path: Path):
+ if vendor_path in self._cache:
+ return self._cache[vendor_path]
+
+ if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))):
+ # This vendored package is not a wheel. It may be a source package (with
+ # a setup.py), or just some Python code that was manually copied into the
+ # tree. If it's a source package, the setup.py file may be up a few levels
+ # from the referenced Python module path.
+ package_dir = vendor_path
+ while True:
+ if (package_dir / "setup.py").exists():
+ break
+ elif package_dir.parent == package_dir:
+ raise Exception(
+ f'Package "{vendor_path}" is not a wheel and does not have a '
+ 'setup.py file. Perhaps it should be "pth:" instead of '
+ '"vendored:"?'
+ )
+ package_dir = package_dir.parent
+
+ self._cache[vendor_path] = package_dir
+ return package_dir
+
+ # Pip requires that wheels have a version number in their name, even if
+ # it ignores it. We should parse out the version and put it in here
+ # so that failure debugging is easier, but that's non-trivial work.
+ # So, this "0" satisfies pip's naming requirement while being relatively
+ # obvious that it's a placeholder.
+ output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any"
+ shutil.make_archive(str(output_path), "zip", vendor_path)
+
+ whl_path = output_path.parent / (output_path.name + ".whl")
+ (output_path.parent / (output_path.name + ".zip")).rename(whl_path)
+ self._cache[vendor_path] = whl_path
+
+ return whl_path
+
+
+def test_sites_compatible(tmpdir: str):
+ command_site_names = _resolve_command_site_names()
+ work_dir = Path(tmpdir)
+ cache = PackageCache(work_dir)
+ mach_requirements = _requirement_definition_to_pip_format("mach", cache, True)
+
+ # Create virtualenv to try to install all dependencies into.
+ virtualenv = PythonVirtualenv(str(work_dir / "env"))
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "venv",
+ "--without-pip",
+ virtualenv.prefix,
+ ]
+ )
+ platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib")
+ third_party = Path(topsrcdir) / "third_party" / "python"
+ with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile:
+ pthfile.write(
+ "\n".join(
+ [
+ str(third_party / "pip"),
+ str(third_party / "wheel"),
+ str(third_party / "setuptools"),
+ ]
+ )
+ )
+
+ for name in command_site_names:
+ print(f'Checking compatibility of "{name}" site')
+ command_requirements = _requirement_definition_to_pip_format(
+ name, cache, name == "build"
+ )
+ with open(work_dir / "requirements.txt", "w") as requirements_txt:
+ requirements_txt.write(mach_requirements)
+ requirements_txt.write("\n")
+ requirements_txt.write(command_requirements)
+
+ # Attempt to install combined set of dependencies (global Mach + current
+ # command)
+ proc = subprocess.run(
+ [
+ virtualenv.python_path,
+ "-m",
+ "pip",
+ "install",
+ "-r",
+ str(work_dir / "requirements.txt"),
+ ],
+ cwd=topsrcdir,
+ )
+ if proc.returncode != 0:
+ print(
+ dedent(
+ f"""
+ Error: The '{name}' site contains dependencies that are not
+ compatible with the 'mach' site. Check the following files for
+ any conflicting packages mentioned in the prior error message:
+
+ python/sites/mach.txt
+ python/sites/{name}.txt
+ """
+ )
+ )
+ assert False
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mach/mach/test/zero_microseconds.py b/python/mach/mach/test/zero_microseconds.py
new file mode 100644
index 0000000000..b1d523071f
--- /dev/null
+++ b/python/mach/mach/test/zero_microseconds.py
@@ -0,0 +1,12 @@
+# This code is loaded via `mach python --exec-file`, so it runs in the scope of
+# the `mach python` command.
+old = self._mach_context.post_dispatch_handler # noqa: F821
+
+
+def handler(context, handler, instance, result, start_time, end_time, depth, args):
+ global old
+ # Round off sub-second precision.
+ old(context, handler, instance, result, int(start_time), end_time, depth, args)
+
+
+self._mach_context.post_dispatch_handler = handler # noqa: F821
diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py
new file mode 100644
index 0000000000..4ed303cf3b
--- /dev/null
+++ b/python/mach/mach/util.py
@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+import os
+import sys
+from pathlib import Path, PurePosixPath
+from typing import Optional, Union
+
+
+class UserError(Exception):
+ """Represents an error caused by something the user did wrong rather than
+ an internal `mach` failure. Exceptions that are subclasses of this class
+ will not be reported as failures to Sentry.
+ """
+
+
+def setenv(key, value):
+ """Compatibility shim to ensure the proper string type is used with
+ os.environ for the version of Python being used.
+ """
+ from six import text_type
+
+ encoding = "mbcs" if sys.platform == "win32" else "utf-8"
+
+ if sys.version_info[0] == 2:
+ if isinstance(key, text_type):
+ key = key.encode(encoding)
+ if isinstance(value, text_type):
+ value = value.encode(encoding)
+ else:
+ if isinstance(key, bytes):
+ key = key.decode(encoding)
+ if isinstance(value, bytes):
+ value = value.decode(encoding)
+
+ os.environ[key] = value
+
+
+def get_state_dir(
+ specific_to_topsrcdir=False, topsrcdir: Optional[Union[str, Path]] = None
+):
+ """Obtain path to a directory to hold state.
+
+ Args:
+ specific_to_topsrcdir (bool): If True, return a state dir specific to the current
+ srcdir instead of the global state dir (default: False)
+
+ Returns:
+ A path to the state dir (str)
+ """
+ state_dir = Path(os.environ.get("MOZBUILD_STATE_PATH", Path.home() / ".mozbuild"))
+ if not specific_to_topsrcdir:
+ return str(state_dir)
+
+ if not topsrcdir:
+ # Only import MozbuildObject if topsrcdir isn't provided. This is to cover
+ # the Mach initialization stage, where "mozbuild" isn't in the import scope.
+ from mozbuild.base import MozbuildObject
+
+ topsrcdir = Path(
+ MozbuildObject.from_environment(cwd=str(Path(__file__).parent)).topsrcdir
+ )
+
+ # Ensure that the topsrcdir is a consistent string before hashing it.
+ topsrcdir = Path(topsrcdir).resolve()
+
+ # Shortening to 12 characters makes these directories a bit more manageable
+ # in a terminal and is more than good enough for this purpose.
+ srcdir_hash = hashlib.sha256(str(topsrcdir).encode("utf-8")).hexdigest()[:12]
+
+ state_dir = state_dir / "srcdirs" / f"{topsrcdir.name}-{srcdir_hash}"
+
+ if not state_dir.is_dir():
+ # We create the srcdir here rather than 'mach_initialize.py' so direct
+ # consumers of this function don't create the directory inconsistently.
+ print(f"Creating local state directory: {state_dir}")
+ state_dir.mkdir(mode=0o770, parents=True)
+ # Save the topsrcdir that this state dir corresponds to so we can clean
+ # it up in the event its srcdir was deleted.
+ with (state_dir / "topsrcdir.txt").open(mode="w") as fh:
+ fh.write(str(topsrcdir))
+
+ return str(state_dir)
+
+
+def win_to_msys_path(path: Path):
+ """Convert a windows-style path to msys-style."""
+ drive, path = os.path.splitdrive(path)
+ path = "/".join(path.split("\\"))
+ if drive:
+ if path[0] == "/":
+ path = path[1:]
+ path = f"/{drive[:-1]}/{path}"
+ return PurePosixPath(path)
+
+
+def to_optional_path(path: Optional[Path]):
+ if path:
+ return Path(path)
+ else:
+ return None
+
+
+def to_optional_str(path: Optional[Path]):
+ if path:
+ return str(path)
+ else:
+ return None
diff --git a/python/mach/metrics.yaml b/python/mach/metrics.yaml
new file mode 100644
index 0000000000..16b2aa2877
--- /dev/null
+++ b/python/mach/metrics.yaml
@@ -0,0 +1,206 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# If this file is changed, update the generated docs:
+# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs
+
+# Adding a new metric? We have docs for that!
+# https://mozilla.github.io/glean/book/user/metrics/adding-new-metrics.html
+---
+$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
+
+mach:
+ command:
+ type: string
+ description: >
+ The name of the mach command that was invoked, such as "build",
+ "doc", or "try".
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ argv:
+ type: string_list
+ description: >
+ Parameters provided to mach. Absolute paths are sanitized to be relative
+ to one of a few key base paths, such as the "$topsrcdir", "$topobjdir",
+ or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild"
+ would be replaced with "$topsrcdir/python/mozbuild".
+ If a valid replacement base path cannot be found, the path is replaced
+ with "<path omitted>".
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ success:
+ type: boolean
+ description: True if the mach invocation succeeded.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ duration:
+ type: timespan
+ description: How long it took for the command to complete.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+
+mach.system:
+ cpu_brand:
+ type: string
+ description: CPU brand string from CPUID.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ distro:
+ type: string
+ description: The name of the operating system distribution.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ distro_version:
+ type: string
+ description: The high-level OS version.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ logical_cores:
+ type: counter
+ description: Number of logical CPU cores present.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ physical_cores:
+ type: counter
+ description: Number of physical CPU cores present.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ memory:
+ type: memory_distribution
+ memory_unit: gigabyte
+ description: Amount of system memory.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ vscode_terminal:
+ type: boolean
+ description: True if the current terminal is opened via Visual Studio Code.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172#c4
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - andi@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ ssh_connection:
+ type: boolean
+ description: True if the current shell is a remote SSH connection.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172#c4
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - andi@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
+ vscode_running:
+ type: boolean
+ description: True if there is an instance of vscode running.
+ lifetime: application
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1717801
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1717801#c1
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - andi@mozilla.com
+ expires: never
+ send_in_pings:
+ - usage
diff --git a/python/mach/pings.yaml b/python/mach/pings.yaml
new file mode 100644
index 0000000000..c975437237
--- /dev/null
+++ b/python/mach/pings.yaml
@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# If this file is changed, update the generated docs:
+# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs
+---
+$schema: moz://mozilla.org/schemas/glean/pings/2-0-0
+
+usage:
+ description: >
+ Sent when the mach invocation is completed (regardless of result).
+ Contains information about the mach invocation that was made, its result,
+ and some details about the current environment and hardware.
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+ include_client_id: true
+ notification_emails:
+ - build-telemetry@mozilla.com
+ - mhentges@mozilla.com
diff --git a/python/mach/setup.cfg b/python/mach/setup.cfg
new file mode 100644
index 0000000000..3c6e79cf31
--- /dev/null
+++ b/python/mach/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
diff --git a/python/mach/setup.py b/python/mach/setup.py
new file mode 100644
index 0000000000..80426b6e00
--- /dev/null
+++ b/python/mach/setup.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+try:
+ from setuptools import setup
+except ImportError:
+ from distutils.core import setup
+
+
+VERSION = "1.0.0"
+HERE = os.path.dirname(__file__)
+README = open(os.path.join(HERE, "README.rst")).read()
+
+setup(
+ name="mach",
+ description="Generic command line command dispatching framework.",
+ long_description=README,
+ license="MPL 2.0",
+ author="Gregory Szorc",
+ author_email="gregory.szorc@gmail.com",
+ url="https://developer.mozilla.org/en-US/docs/Developer_Guide/mach",
+ packages=["mach", "mach.mixin"],
+ version=VERSION,
+ classifiers=[
+ "Environment :: Console",
+ "Development Status :: 5 - Production/Stable",
+ "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+ "Natural Language :: English",
+ "Programming Language :: Python :: 2.7",
+ "Programming Language :: Python :: 3.5",
+ ],
+ install_requires=[
+ "blessed",
+ "mozfile",
+ "mozprocess",
+ "six",
+ ],
+ tests_require=["mock"],
+)