diff options
Diffstat (limited to '')
68 files changed, 9669 insertions, 0 deletions
diff --git a/python/mach/.ruff.toml b/python/mach/.ruff.toml new file mode 100644 index 0000000000..82b1a04648 --- /dev/null +++ b/python/mach/.ruff.toml @@ -0,0 +1,4 @@ +extend = "../../pyproject.toml" + +[isort] +known-first-party = ["mach"] diff --git a/python/mach/README.rst b/python/mach/README.rst new file mode 100644 index 0000000000..7c2e00becb --- /dev/null +++ b/python/mach/README.rst @@ -0,0 +1,13 @@ +==== +mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +To learn more, read the docs in ``docs/``. diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh new file mode 100644 index 0000000000..13935cf88c --- /dev/null +++ b/python/mach/bash-completion.sh @@ -0,0 +1,18 @@ +function _mach() +{ + local cur targets + COMPREPLY=() + + # Calling `mach-completion` with -h/--help would result in the + # help text being used as the completion targets. + if [[ $COMP_LINE == *"-h"* || $COMP_LINE == *"--help"* ]]; then + return 0 + fi + + # Load the list of targets + targets=`"${COMP_WORDS[0]}" mach-completion ${COMP_LINE}` + cur="${COMP_WORDS[COMP_CWORD]}" + COMPREPLY=( $(compgen -W "$targets" -- ${cur}) ) + return 0 +} +complete -o default -F _mach mach diff --git a/python/mach/docs/commands.rst b/python/mach/docs/commands.rst new file mode 100644 index 0000000000..7547193000 --- /dev/null +++ b/python/mach/docs/commands.rst @@ -0,0 +1,129 @@ +.. _mach_commands: + +===================== +Implementing Commands +===================== + +Mach commands are defined via Python decorators. + +All the relevant decorators are defined in the *mach.decorators* module. +The important decorators are as follows: + +:py:func:`Command <mach.decorators.Command>` + A function decorator that denotes that the function should be called when + the specified command is requested. The decorator takes a command name + as its first argument and a number of additional arguments to + configure the behavior of the command. The decorated function must take a + ``command_context`` argument as its first. + ``command_context`` is a properly configured instance of a ``MozbuildObject`` + subclass, meaning it can be used for accessing things like the current config + and running processes. + +:py:func:`CommandArgument <mach.decorators.CommandArgument>` + A function decorator that defines an argument to the command. Its + arguments are essentially proxied to ArgumentParser.add_argument() + +:py:func:`SubCommand <mach.decorators.SubCommand>` + A function decorator that denotes that the function should be a + sub-command to an existing ``@Command``. The decorator takes the + parent command name as its first argument and the sub-command name + as its second argument. + + ``@CommandArgument`` can be used on ``@SubCommand`` instances just + like they can on ``@Command`` instances. + + +Here is a complete example: + +.. code-block:: python + + from mach.decorators import ( + CommandArgument, + Command, + ) + + @Command('doit', help='Do ALL OF THE THINGS.') + @CommandArgument('--force', '-f', action='store_true', + help='Force doing it.') + def doit(command_context, force=False): + # Do stuff here. + +When the module is loaded, the decorators tell mach about all handlers. +When mach runs, it takes the assembled metadata from these handlers and +hooks it up to the command line driver. Under the hood, arguments passed +to the decorators are being used to help mach parse command arguments, +formulate arguments to the methods, etc. See the documentation in the +:py:mod:`mach.base` module for more. + +The Python modules defining mach commands do not need to live inside the +main mach source tree. + +Conditionally Filtering Commands +================================ + +Sometimes it might only make sense to run a command given a certain +context. For example, running tests only makes sense if the product +they are testing has been built, and said build is available. To make +sure a command is only runnable from within a correct context, you can +define a series of conditions on the +:py:func:`Command <mach.decorators.Command>` decorator. + +A condition is simply a function that takes an instance of the +:py:func:`mozbuild.base.MachCommandBase` class as an argument, and +returns ``True`` or ``False``. If any of the conditions defined on a +command return ``False``, the command will not be runnable. The +docstring of a condition function is used in error messages, to explain +why the command cannot currently be run. + +Here is an example: + +.. code-block:: python + + from mach.decorators import ( + Command, + ) + + def build_available(cls): + """The build needs to be available.""" + return cls.build_path is not None + + @Command('run_tests', conditions=[build_available]) + def run_tests(command_context): + # Do stuff here. + +By default all commands without any conditions applied will be runnable, +but it is possible to change this behaviour by setting +``require_conditions`` to ``True``: + +.. code-block:: python + + m = mach.main.Mach() + m.require_conditions = True + +Minimizing Code in Commands +=========================== + +Mach command modules, classes, and methods work best when they are +minimal dispatchers. The reason is import bloat. Currently, the mach +core needs to import every Python file potentially containing mach +commands for every command invocation. If you have dozens of commands or +commands in modules that import a lot of Python code, these imports +could slow mach down and waste memory. + +It is thus recommended that mach modules, classes, and methods do as +little work as possible. Ideally the module should only import from +the :py:mod:`mach` package. If you need external modules, you should +import them from within the command method. + +To keep code size small, the body of a command method should be limited +to: + +1. Obtaining user input (parsing arguments, prompting, etc) +2. Calling into some other Python package +3. Formatting output + +Of course, these recommendations can be ignored if you want to risk +slower performance. + +In the future, the mach driver may cache the dispatching information or +have it intelligently loaded to facilitate lazy loading. diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst new file mode 100644 index 0000000000..8a2a99a2f5 --- /dev/null +++ b/python/mach/docs/driver.rst @@ -0,0 +1,32 @@ +.. _mach_driver: + +======= +Drivers +======= + +Entry Points +============ + +It is possible to use setuptools' entry points to load commands +directly from python packages. A mach entry point is a function which +returns a list of files or directories containing mach command +providers. e.g.: + +.. code-block:: python + + def list_providers(): + providers = [] + here = os.path.abspath(os.path.dirname(__file__)) + for p in os.listdir(here): + if p.endswith('.py'): + providers.append(os.path.join(here, p)) + return providers + +See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins +for more information on creating an entry point. To search for entry +point plugins, you can call +:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.: + +.. code-block:: python + + mach.load_commands_from_entry_point("mach.external.providers") diff --git a/python/mach/docs/faq.rst b/python/mach/docs/faq.rst new file mode 100644 index 0000000000..a640f83e87 --- /dev/null +++ b/python/mach/docs/faq.rst @@ -0,0 +1,152 @@ +.. _mach_faq: + +========================== +Frequently Asked Questions +========================== + +How do I report bugs? +--------------------- + +Bugs against the ``mach`` core can be filed in Bugzilla in the `Firefox +Build System::Mach +Core <https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Mach%20Core>`__ component. + +.. note:: + + Most ``mach`` bugs are bugs in individual commands, not bugs in the core + ``mach`` code. Bugs for individual commands should be filed against the + component that command is related to. For example, bugs in the + *build* command should be filed against *Firefox Build System :: + General*. Bugs against testing commands should be filed somewhere in + the *Testing* product. + +How do I debug a command failing with a Python exception? +--------------------------------------------------------- + +You can run a command and break into ``pdb``, the Python debugger, +when the command is invoked with: + +.. code-block:: shell + + ./mach --debug-command FAILING-COMMAND ARGS ... + +How do I debug ``mach`` itself? +------------------------------- + +If you are editing the mach code, or other Python modules you can +open the terminal and start debugging with pdb with the following: + +.. code-block:: shell + + python3 -m pdb ./mach <command> + +How do I debug ``pytest`` tests? +-------------------------------- + +First, before debugging, run ``./mach python-test`` once to ensure that +the testing virtualenv is up-to-date: + +.. code-block:: shell + + ./mach python-test path/to/test.py + +Then, using the testing virtualenv, debug the test file: + +.. code-block:: shell + + <objdir>/_virtualenvs/python-test/bin/python -m pdb path/to/test.py + +How do I profile a slow command? +-------------------------------- + +To diagnose bottlenecks, you can collect a performance profile: + +.. code-block:: shell + + ./mach --profile-command SLOW-COMMAND ARGS ... + +Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using +`snakeviz <https://jiffyclub.github.io/snakeviz/>`__: + +.. code-block:: shell + + # If you don't have snakeviz installed yet: + python3 -m pip install snakeviz + python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile + +How do I profile ``mach`` itself? +--------------------------------- + +Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile`` +directly to profile ``mach`` itself: + +.. code-block:: shell + + python3 -m cProfile -o mach.cProfile ./mach ... + python3 -m snakeviz mach.cProfile + +Is ``mach`` a build system? +--------------------------- + +No. ``mach`` is just a generic command dispatching tool that happens to have +a few commands that interact with the real build system. Historically, +``mach`` *was* born to become a better interface to the build system. +However, its potential beyond just build system interaction was quickly +realized and ``mach`` grew to fit those needs. + +How do I add features to ``mach``? +---------------------------------- +If you would like to add a new feature to ``mach`` that cannot be implemented as +a ``mach`` command, the first step is to file a bug in the +``Firefox Build System :: Mach Core`` component. + +Should I implement X as a ``mach`` command? +------------------------------------------- + +There are no hard or fast rules. Generally speaking, if you have some +piece of functionality or action that is useful to multiple people +(especially if it results in productivity wins), then you should +consider implementing a ``mach`` command for it. + +Some other cases where you should consider implementing something as a +``mach`` command: + +- When your tool is a random script in the tree. Random scripts are + hard to find and may not conform to coding conventions or best + practices. ``Mach`` provides a framework in which your tool can live that + will put it in a better position to succeed than if it were on its + own. +- When the alternative is a ``make`` target. The build team generally does + not like one-off ``make`` targets that aren't part of building (read: + compiling) the tree. This includes things related to testing and + packaging. These weigh down ``Makefiles`` and add to the burden of + maintaining the build system. Instead, you are encouraged to + implement ancillary functionality in Python. If you do implement something + in Python, hooking it up to ``mach`` is often trivial. + +How do I use 3rd-party Python packages in my ``mach`` command? +-------------------------------------------------------------- + +See :ref:`Using third-party Python packages`. + +How does ``mach`` fit into the modules system? +---------------------------------------------- + +Mozilla operates with a `modules governance +system <https://www.mozilla.org/about/governance/policies/module-ownership/>`__ where +there are different components with different owners. There is not +currently a ``mach`` module. There may or may never be one; currently ``mach`` +is owned by the build team. + +Even if a ``mach`` module were established, ``mach`` command modules would +likely never belong to it. Instead, ``mach`` command modules are owned by the +team/module that owns the system they interact with. In other words, ``mach`` +is not a power play to consolidate authority for tooling. Instead, it aims to +expose that tooling through a common, shared interface. + + +Who do I contact for help or to report issues? +---------------------------------------------- + +You can ask questions in +`#build <https://chat.mozilla.org/#/room/#build:mozilla.org>`__. diff --git a/python/mach/docs/index.rst b/python/mach/docs/index.rst new file mode 100644 index 0000000000..752fe93219 --- /dev/null +++ b/python/mach/docs/index.rst @@ -0,0 +1,89 @@ +==== +Mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +.. raw:: html + + <h2>Features</h2> + +---- + +On a high level, mach is similar to using argparse with subparsers (for +command handling). When you dig deeper, mach offers a number of +additional features: + +Distributed command definitions + With optparse/argparse, you have to define your commands on a central + parser instance. With mach, you annotate your command methods with + decorators and mach finds and dispatches to them automatically. + +Command categories + Mach commands can be grouped into categories when displayed in help. + This is currently not possible with argparse. + +Logging management + Mach provides a facility for logging (both classical text and + structured) that is available to any command handler. + +Settings files + Mach provides a facility for reading settings from an ini-like file + format. + +.. raw:: html + + <h2>Components</h2> + +---- + +Mach is conceptually composed of the following components: + +core + The mach core is the core code powering mach. This is a Python package + that contains all the business logic that makes mach work. The mach + core is common to all mach deployments. + +commands + These are what mach dispatches to. Commands are simply Python methods + registered as command names. The set of commands is unique to the + environment mach is deployed in. + +driver + The *driver* is the entry-point to mach. It is simply an executable + script that loads the mach core, tells it where commands can be found, + then asks the mach core to handle the current request. The driver is + unique to the deployed environment. But, it's usually based on an + example from this source tree. + +.. raw:: html + + <h2> Project State</h2> + +---- + +mach was originally written as a command dispatching framework to aid +Firefox development. While the code is mostly generic, there are still +some pieces that closely tie it to Mozilla/Firefox. The goal is for +these to eventually be removed and replaced with generic features so +mach is suitable for anybody to use. Until then, mach may not be the +best fit for you. + +.. toctree:: + :maxdepth: 1 + :hidden: + + usage + commands + driver + logging + settings + telemetry + windows-usage-outside-mozillabuild + faq diff --git a/python/mach/docs/logging.rst b/python/mach/docs/logging.rst new file mode 100644 index 0000000000..ff245cf032 --- /dev/null +++ b/python/mach/docs/logging.rst @@ -0,0 +1,100 @@ +.. _mach_logging: + +======= +Logging +======= + +Mach configures a built-in logging facility so commands can easily log +data. + +What sets the logging facility apart from most loggers you've seen is +that it encourages structured logging. Instead of conventional logging +where simple strings are logged, the internal logging mechanism logs all +events with the following pieces of information: + +* A string *action* +* A dict of log message fields +* A formatting string + +Essentially, instead of assembling a human-readable string at +logging-time, you create an object holding all the pieces of data that +will constitute your logged event. For each unique type of logged event, +you assign an *action* name. + +Depending on how logging is configured, your logged event could get +written a couple of different ways. + +JSON Logging +============ + +Where machines are the intended target of the logging data, a JSON +logger is configured. The JSON logger assembles an array consisting of +the following elements: + +* Decimal wall clock time in seconds since UNIX epoch +* String *action* of message +* Object with structured message data + +The JSON-serialized array is written to a configured file handle. +Consumers of this logging stream can just perform a readline() then feed +that into a JSON deserializer to reconstruct the original logged +message. They can key off the *action* element to determine how to +process individual events. There is no need to invent a parser. +Convenient, isn't it? + +Logging for Humans +================== + +Where humans are the intended consumer of a log message, the structured +log message are converted to more human-friendly form. This is done by +utilizing the *formatting* string provided at log time. The logger +simply calls the *format* method of the formatting string, passing the +dict containing the message's fields. + +When *mach* is used in a terminal that supports it, the logging facility +also supports terminal features such as colorization. This is done +automatically in the logging layer - there is no need to control this at +logging time. + +In addition, messages intended for humans typically prepends every line +with the time passed since the application started. + +Logging HOWTO +============= + +Structured logging piggybacks on top of Python's built-in logging +infrastructure provided by the *logging* package. We accomplish this by +taking advantage of *logging.Logger.log()*'s *extra* argument. To this +argument, we pass a dict with the fields *action* and *params*. These +are the string *action* and dict of message fields, respectively. The +formatting string is passed as the *msg* argument, like normal. + +If you were logging to a logger directly, you would do something like: + +.. code-block:: python + + logger.log(logging.INFO, 'My name is {name}', + extra={'action': 'my_name', 'params': {'name': 'Gregory'}}) + +The JSON logging would produce something like:: + + [1339985554.306338, "my_name", {"name": "Gregory"}] + +Human logging would produce something like:: + + 0.52 My name is Gregory + +Since there is a lot of complexity using logger.log directly, it is +recommended to go through a wrapping layer that hides part of the +complexity for you. The easiest way to do this is by utilizing the +LoggingMixin: + +.. code-block:: python + + import logging + from mach.mixin.logging import LoggingMixin + + class MyClass(LoggingMixin): + def foo(self): + self.log(logging.INFO, 'foo_start', {'bar': True}, + 'Foo performed. Bar: {bar}') diff --git a/python/mach/docs/metrics.md b/python/mach/docs/metrics.md new file mode 100644 index 0000000000..8c826f54a9 --- /dev/null +++ b/python/mach/docs/metrics.md @@ -0,0 +1,55 @@ +<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. --> + +# Metrics +This document enumerates the metrics collected by this project using the [Glean SDK](https://mozilla.github.io/glean/book/index.html). +This project may depend on other projects which also collect metrics. +This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project. + +# Pings + + - [usage](#usage) + + +## usage + +Sent when the mach invocation is completed (regardless of result). Contains information about the mach invocation that was made, its result, and some details about the current environment and hardware. + + +This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section). + +**Data reviews for this ping:** + +- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34> + +**Bugs related to this ping:** + +- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053> + +The following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| mach.argv |[string_list](https://mozilla.github.io/glean/book/user/metrics/string_list.html) |Parameters provided to mach. Absolute paths are sanitized to be relative to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" would be replaced with "$topsrcdir/python/mozbuild". If a valid replacement base path cannot be found, the path is replaced with "<path omitted>". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.command |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the mach command that was invoked, such as "build", "doc", or "try". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.duration |[timespan](https://mozilla.github.io/glean/book/user/metrics/timespan.html) |How long it took for the command to complete. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.success |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the mach invocation succeeded. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.cpu_brand |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |CPU brand string from CPUID. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.distro |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the operating system distribution. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3)||never | | +| mach.system.distro_version |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The high-level OS version. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3)||never | | +| mach.system.logical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of logical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.memory |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |Amount of system memory. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mach.system.physical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of physical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.artifact |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-artifact-builds`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.ccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--with-ccache`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.clobber |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the build was a clobber/full build. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15)||never | | +| mozbuild.compiler |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The compiler type in use (CC_TYPE), such as "clang" or "gcc". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.debug |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-debug`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.icecream |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if icecream in use. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.opt |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-optimize`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | +| mozbuild.project |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The project being built. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1654084#c2)||never | | +| mozbuild.sccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if ccache in use is sccache. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | | + + +Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection). + +<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. --> diff --git a/python/mach/docs/settings.rst b/python/mach/docs/settings.rst new file mode 100644 index 0000000000..4daba37472 --- /dev/null +++ b/python/mach/docs/settings.rst @@ -0,0 +1,138 @@ +.. _mach_settings: + +======== +Settings +======== + +Mach can read settings in from a set of configuration files. These +configuration files are either named ``machrc`` or ``.machrc`` and +are specified by the bootstrap script. In mozilla-central, these files +can live in ``~/.mozbuild`` and/or ``topsrcdir``. + +Settings can be specified anywhere, and used both by mach core or +individual commands. + + +Core Settings +============= + +These settings are implemented by mach core. + +* alias - Create a command alias. This is useful if you want to alias a command to something else, optionally including some defaults. It can either be used to create an entire new command, or provide defaults for an existing one. For example: + +.. parsed-literal:: + + [alias] + mochitest = mochitest -f browser + browser-test = mochitest -f browser + + +Defining Settings +================= + +Settings need to be explicitly defined, along with their type, +otherwise mach will throw when trying to access them. + +To define settings, use the :func:`~decorators.SettingsProvider` +decorator in an existing mach command module. E.g: + +.. code-block:: python + + from mach.decorators import SettingsProvider + from mozbuild.base import MachCommandBase + + @SettingsProvider + class ArbitraryClassName(MachCommandBase): + config_settings = [ + ('foo.bar', 'string', "A helpful description"), + ('foo.baz', 'int', "Another description", 0, {'choices': set([0,1,2])}), + ] + +``@SettingsProvider``'s must specify a variable called ``config_settings`` +that returns a list of tuples. Alternatively, it can specify a function +called ``config_settings`` that returns a list of tuples. + +Each tuple is of the form: + +.. code-block:: python + + ('<section>.<option>', '<type>', '<description>', default, extra) + +``type`` is a string and can be one of: +string, boolean, int, pos_int, path + +``description`` is a string explaining how to define the settings and +where they get used. Descriptions should ideally be multi-line paragraphs +where the first line acts as a short description. + +``default`` is optional, and provides a default value in case none was +specified by any of the configuration files. + +``extra`` is also optional and is a dict containing additional key/value +pairs to add to the setting's metadata. The following keys may be specified +in the ``extra`` dict: + + * ``choices`` - A set of allowed values for the setting. + +Wildcards +--------- + +Sometimes a section should allow arbitrarily defined options from the user, such +as the ``alias`` section mentioned above. To define a section like this, use ``*`` +as the option name. For example: + +.. parsed-literal:: + + ('foo.*', 'string', 'desc') + +This allows configuration files like this: + +.. parsed-literal:: + + [foo] + arbitrary1 = some string + arbitrary2 = some other string + + +Finding Settings +================ + +You can see which settings are available as well as their description and +expected values by running: + +.. parsed-literal:: + + ./mach settings # or + ./mach settings --list + + +Accessing Settings +================== + +Now that the settings are defined and documented, they're accessible from +individual mach commands from the mach command context. +For example: + +.. code-block:: python + + from mach.decorators import ( + Command, + SettingsProvider, + ) + from mozbuild.base import MachCommandBase + + @SettingsProvider + class ExampleSettings(object): + config_settings = [ + ('a.b', 'string', 'desc', 'default'), + ('foo.bar', 'string', 'desc',), + ('foo.baz', 'int', 'desc', 0, {'choices': set([0,1,2])}), + ] + + @Command('command', category='misc', + description='Prints a setting') + def command(command_context): + settings = command_context._mach_context.settings + print(settings.a.b) + for option in settings.foo: + print(settings.foo[option]) diff --git a/python/mach/docs/telemetry.rst b/python/mach/docs/telemetry.rst new file mode 100644 index 0000000000..2d185a970e --- /dev/null +++ b/python/mach/docs/telemetry.rst @@ -0,0 +1,37 @@ +.. _mach_telemetry: + +============== +Mach Telemetry +============== + +`Glean <https://mozilla.github.io/glean/>`_ is used to collect telemetry, and uses the metrics +defined in the ``metrics.yaml`` files in-tree. +These files are all documented in a single :ref:`generated file here<metrics>`. + +.. toctree:: + :maxdepth: 1 + + metrics + +Adding Metrics to a new Command +=============================== + +If you would like to submit telemetry metrics from your mach ``@Command``, you should take two steps: + +#. Parameterize your ``@Command`` annotation with ``metrics_path``. +#. Use the ``command_context.metrics`` handle provided by ``MachCommandBase`` + +For example:: + + METRICS_PATH = os.path.abspath(os.path.join(__file__, '..', '..', 'metrics.yaml')) + + @Command('custom-command', metrics_path=METRICS_PATH) + def custom_command(command_context): + command_context.metrics.custom.foo.set('bar') + +Updating Generated Metrics Docs +=============================== + +When a ``metrics.yaml`` is added/changed/removed, :ref:`the metrics document<metrics>` will need to be updated:: + + ./mach doc mach-telemetry diff --git a/python/mach/docs/usage.rst b/python/mach/docs/usage.rst new file mode 100644 index 0000000000..a32b35395c --- /dev/null +++ b/python/mach/docs/usage.rst @@ -0,0 +1,150 @@ +.. _mach_usage: + +========== +User Guide +========== + +Mach is the central entry point for most operations that can be performed in +mozilla-central. + + +Command Help +------------ + +To see an overview of all the available commands, run: + +.. code-block:: shell + + $ ./mach help + +For more detailed information on a specific command, run: + +.. code-block:: shell + + $ ./mach help <command> + +If a command has subcommands listed, you can see more details on the subcommand +by running: + +.. code-block:: shell + + $ ./mach help <command> <subcommand> + +Alternatively, you can pass ``-h/--help``. For example, all of the +following are valid: + +.. code-block:: shell + + $ ./mach help try + $ ./mach help try fuzzy + $ ./mach try -h + $ ./mach try fuzzy --help + + +Tab Completion +-------------- + +There are commands built-in to ``mach`` that can generate a fast tab completion +script for various shells. Supported shells are currently ``bash``, ``zsh`` and +``fish``. These generated scripts will slowly become out of date over time, so +you may want to create a cron task to periodically re-generate them. + +See below for installation instructions: + +Bash +~~~~ + +.. code-block:: shell + + $ mach mach-completion bash -f _mach + $ sudo mv _mach /etc/bash_completion.d + +Bash (homebrew) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ mach mach-completion bash -f $(brew --prefix)/etc/bash_completion.d/mach.bash-completion + +Zsh +~~~ + +.. code-block:: shell + + $ mkdir ~/.zfunc + $ mach mach-completion zsh -f ~/.zfunc/_mach + +then edit ~/.zshrc and add: + +.. code-block:: shell + + fpath+=~/.zfunc + autoload -U compinit && compinit + +You can use any directory of your choosing. + +Zsh (oh-my-zsh) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ mkdir $ZSH/plugins/mach + $ mach mach-completion zsh -f $ZSH/plugins/mach/_mach + +then edit ~/.zshrc and add 'mach' to your enabled plugins: + +.. code-block:: shell + + plugins(mach ...) + +Zsh (prezto) +~~~~~~~~~~~~ + +.. code-block:: shell + + $ mach mach-completion zsh -f ~/.zprezto/modules/completion/external/src/_mach + +Fish +~~~~ + +.. code-block:: shell + + $ ./mach mach-completion fish -f ~/.config/fish/completions/mach.fish + +Fish (homebrew) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ ./mach mach-completion fish -f (brew --prefix)/share/fish/vendor_completions.d/mach.fish + + +User Settings +------------- + +Some mach commands can read configuration from a ``machrc`` file. The default +location for this file is ``~/.mozbuild/machrc`` (you'll need to create it). +This can also be set to a different location by setting the ``MACHRC`` +environment variable. + +For a list of all the available settings, run: + +.. code-block:: shell + + $ ./mach settings + +The settings file follows the ``ini`` format, e.g: + +.. code-block:: ini + + [alias] + eslint = lint -l eslint + + [build] + telemetry = true + + [try] + default = fuzzy + + +.. _bash completion: https://searchfox.org/mozilla-central/source/python/mach/bash-completion.sh diff --git a/python/mach/docs/windows-usage-outside-mozillabuild.rst b/python/mach/docs/windows-usage-outside-mozillabuild.rst new file mode 100644 index 0000000000..6a034fd384 --- /dev/null +++ b/python/mach/docs/windows-usage-outside-mozillabuild.rst @@ -0,0 +1,124 @@ +========================================== +Using Mach on Windows Outside MozillaBuild +========================================== + +.. note:: + + These docs still require that you've followed the :ref:`Building Firefox On Windows` guide. + +`MozillaBuild <https://wiki.mozilla.org/MozillaBuild>`__ is required to build +Firefox on Windows, because it provides necessary unix-y tools such as ``sh`` and ``awk``. + +Traditionally, to interact with Mach and the Firefox Build System, Windows +developers would have to do so from within the MozillaBuild shell. This could be +disadvantageous for two main reasons: + +1. The MozillaBuild environment is unix-y and based on ``bash``, which may be unfamiliar + for developers used to the Windows Command Prompt or Powershell. +2. There have been long-standing stability issues with MozillaBuild - this is due to + the fragile interface point between the underlying "MSYS" tools and "native Windows" + binaries. + +It is now (experimentally!) possible to invoke Mach directly from other command line +environments, such as Powershell, Command Prompt, or even a developer-managed MSYS2 +environment. Windows Terminal should work as well, for those on the "cutting edge". + +.. note:: + + If you're using a Cygwin-based environment such as MSYS2, it'll probably be + best to use the Windows-native version of Python (as described below) instead of a Python + distribution provided by the environment's package manager. Otherwise you'll likely run into + compatibility issues: + + * Cygwin/MSYS Python will run into compatibility issues with Mach due to its unexpected Unix-y + conventions despite Mach assuming it's on a "Windows" platform. Additionally, there may + be performance issues. + * MinGW Python will encounter issues building native packages because they'll expect the + MSVC toolchain. + +.. warning:: + + This is only recommended for more advanced Windows developers: this work is experimental + and may run into unexpected failures! + +Following are steps for preparing Windows-native (Command Prompt/Powershell) usage of Mach: + +1. Install Python +~~~~~~~~~~~~~~~~~ + +Download Python from the `the official website <https://www.python.org/downloads/windows/>`__. + +.. note:: + + To avoid Mach compatibility issues with recent Python releases, it's recommended to install + the 2nd-most recent "major version". For example, at time of writing, the current modern Python + version is 3.10.1, so a safe version to install would be the most recent 3.9 release. + +You'll want to download the "Windows installer (64-bit)" associated with the release you've chosen. +During installation, ensure that you check the "Add Python 3.x to PATH" option, otherwise you might +`encounter issues running Mercurial <https://bz.mercurial-scm.org/show_bug.cgi?id=6635>`__. + +.. note:: + + Due to issues with Python DLL import failures with pip-installed binaries, it's not + recommended to use the Windows Store release of Python. + +2. Modify your PATH +~~~~~~~~~~~~~~~~~~~ + +The Python "user site-packages directory" needs to be added to your ``PATH`` so that packages +installed via ``pip install --user`` (such as ``hg``) can be invoked from the command-line. + +1. From the Start menu, go to the Control Panel entry for "Edit environment variables + for your account". +2. Double-click the ``Path`` row in the top list of variables. Click "New" to add a new item to + the list. +3. In a Command Prompt window, resolve the Python directory with the command + ``python -c "import site; import os; print(os.path.abspath(os.path.join(site.getusersitepackages(), '..', 'Scripts')))"``. +4. Paste the output into the new item entry in the "Edit environment variable" window. +5. Click "New" again, and add the ``bin`` folder of MozillaBuild: probably ``C:\mozilla-build\bin``. +6. Click "OK". + +3. Install Version Control System +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're using Mercurial, you'll need to install it to your Windows-native Python: + +.. code-block:: shell + + pip3 install --user mercurial windows-curses + +If you're using Git with Cinnabar, follow its `setup instructions <https://github.com/glandium/git-cinnabar#setup>`__. + +4. Set Powershell Execution Policy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're using Powershell, Windows will raise an error by default when you try to invoke +``.\mach.ps1``: + +.. code:: + + .\mach : File <topsrcdir>\mach.ps1 cannot be loaded because running scripts is disabled on this system. For + more information, see about_Execution_Policies at https:/go.microsoft.com/fwlink/?LinkID=135170. + At line:1 char:1 + +To work around this: + +1. From the Start menu, type in "Powershell", then right-click on the best match and click + "Run as administrator" +2. Run the command ``Set-ExecutionPolicy RemoteSigned`` +3. Close the Administrator Powershell window, and open a regular Powershell window +4. Go to your Firefox checkout (likely ``C:\mozilla-source\mozilla-unified``) +5. Test the new execution policy by running ``.\mach bootstrap``. If it doesn't immediately fail + with the error about "Execution Policies", then the problem is resolved. + +Success! +~~~~~~~~ + +At this point, you should be able to invoke Mach and manage your version control system outside +of MozillaBuild. + +.. tip:: + + `See here <https://crisal.io/words/2022/11/22/msys2-firefox-development.html>`__ for a detailed guide on + installing and customizing a development environment with MSYS2, zsh, and Windows Terminal. diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/__init__.py diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py new file mode 100644 index 0000000000..fac17e9b03 --- /dev/null +++ b/python/mach/mach/base.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class CommandContext(object): + """Holds run-time state so it can easily be passed to command providers.""" + + def __init__( + self, cwd: str, settings=None, log_manager=None, commands=None, **kwargs + ): + self.cwd = cwd + self.settings = settings + self.log_manager = log_manager + self.commands = commands + self.is_interactive = None # Filled in after args are parsed + self.telemetry = None # Filled in after args are parsed + self.command_attrs = {} + + for k, v in kwargs.items(): + setattr(self, k, v) + + +class MachError(Exception): + """Base class for all errors raised by mach itself.""" + + +class NoCommandError(MachError): + """No command was passed into mach.""" + + def __init__(self, namespace): + MachError.__init__(self) + self.namespace = namespace + + +class UnknownCommandError(MachError): + """Raised when we attempted to execute an unknown command.""" + + def __init__(self, command, verb, suggested_commands=None): + MachError.__init__(self) + + self.command = command + self.verb = verb + self.suggested_commands = suggested_commands or [] + + +class UnrecognizedArgumentError(MachError): + """Raised when an unknown argument is passed to mach.""" + + def __init__(self, command, arguments): + MachError.__init__(self) + + self.command = command + self.arguments = arguments + + +class FailedCommandError(Exception): + """Raised by commands to signal a handled failure to be printed by mach + + When caught by mach a FailedCommandError will print message and exit + with ''exit_code''. The optional ''reason'' is a string in cases where + other scripts may wish to handle the exception, though this is generally + intended to communicate failure to mach. + """ + + def __init__(self, message, exit_code=1, reason=""): + Exception.__init__(self, message) + self.exit_code = exit_code + self.reason = reason + + +class MissingFileError(MachError): + """Attempted to load a mach commands file that doesn't exist.""" diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/commands/__init__.py diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py new file mode 100644 index 0000000000..12c4b240ea --- /dev/null +++ b/python/mach/mach/commands/commandinfo.py @@ -0,0 +1,487 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import re +import subprocess +import sys +from itertools import chain +from pathlib import Path + +import attr +from mozbuild.util import memoize + +from mach.decorators import Command, CommandArgument, SubCommand + +COMPLETION_TEMPLATES_DIR = Path(__file__).resolve().parent / "completion_templates" + + +@attr.s +class CommandInfo(object): + name = attr.ib(type=str) + description = attr.ib(type=str) + subcommands = attr.ib(type=list) + options = attr.ib(type=dict) + subcommand = attr.ib(type=str, default=None) + + +def render_template(shell, context): + filename = "{}.template".format(shell) + with open(COMPLETION_TEMPLATES_DIR / filename) as fh: + template = fh.read() + return template % context + + +@memoize +def command_handlers(command_context): + """A dictionary of command handlers keyed by command name.""" + return command_context._mach_context.commands.command_handlers + + +@memoize +def commands(command_context): + """A sorted list of all command names.""" + return sorted(command_handlers(command_context)) + + +def _get_parser_options(parser): + options = {} + for action in parser._actions: + # ignore positional args + if not action.option_strings: + continue + + # ignore suppressed args + if action.help == argparse.SUPPRESS: + continue + + options[tuple(action.option_strings)] = action.help or "" + return options + + +@memoize +def global_options(command_context): + """Return a dict of global options. + + Of the form `{("-o", "--option"): "description"}`. + """ + for group in command_context._mach_context.global_parser._action_groups: + if group.title == "Global Arguments": + return _get_parser_options(group) + + +@memoize +def _get_handler_options(handler): + """Return a dict of options for the given handler. + + Of the form `{("-o", "--option"): "description"}`. + """ + options = {} + for option_strings, val in handler.arguments: + # ignore positional args + if option_strings[0][0] != "-": + continue + + options[tuple(option_strings)] = val.get("help", "") + + if handler._parser: + options.update(_get_parser_options(handler.parser)) + + return options + + +def _get_handler_info(handler): + try: + options = _get_handler_options(handler) + except (Exception, SystemExit): + # We don't want misbehaving commands to break tab completion, + # ignore any exceptions. + options = {} + + subcommands = [] + for sub in sorted(handler.subcommand_handlers): + subcommands.append(_get_handler_info(handler.subcommand_handlers[sub])) + + return CommandInfo( + name=handler.name, + description=handler.description or "", + options=options, + subcommands=subcommands, + subcommand=handler.subcommand, + ) + + +@memoize +def commands_info(command_context): + """Return a list of CommandInfo objects for each command.""" + commands_info = [] + # Loop over self.commands() rather than self.command_handlers().items() for + # alphabetical order. + for c in commands(command_context): + commands_info.append(_get_handler_info(command_handlers(command_context)[c])) + return commands_info + + +@Command("mach-commands", category="misc", description="List all mach commands.") +def run_commands(command_context): + print("\n".join(commands(command_context))) + + +@Command( + "mach-debug-commands", + category="misc", + description="Show info about available mach commands.", +) +@CommandArgument( + "match", + metavar="MATCH", + default=None, + nargs="?", + help="Only display commands containing given substring.", +) +def run_debug_commands(command_context, match=None): + import inspect + + for command, handler in command_handlers(command_context).items(): + if match and match not in command: + continue + + func = handler.func + + print(command) + print("=" * len(command)) + print("") + print("File: %s" % inspect.getsourcefile(func)) + print("Function: %s" % func.__name__) + print("") + + +@Command( + "mach-completion", + category="misc", + description="Prints a list of completion strings for the specified command.", +) +@CommandArgument( + "args", default=None, nargs=argparse.REMAINDER, help="Command to complete." +) +def run_completion(command_context, args): + if not args: + print("\n".join(commands(command_context))) + return + + is_help = "help" in args + command = None + for i, arg in enumerate(args): + if arg in commands(command_context): + command = arg + args = args[i + 1 :] + break + + # If no command is typed yet, just offer the commands. + if not command: + print("\n".join(commands(command_context))) + return + + handler = command_handlers(command_context)[command] + # If a subcommand was typed, update the handler. + for arg in args: + if arg in handler.subcommand_handlers: + handler = handler.subcommand_handlers[arg] + break + + targets = sorted(handler.subcommand_handlers.keys()) + if is_help: + print("\n".join(targets)) + return + + targets.append("help") + targets.extend(chain(*_get_handler_options(handler).keys())) + print("\n".join(targets)) + + +def _zsh_describe(value, description=None): + value = '"' + value.replace(":", "\\:") + if description: + description = subprocess.list2cmdline( + [re.sub(r'(["\'#&;`|*?~<>^()\[\]{}$\\\x0A\xFF])', r"\\\1", description)] + ).lstrip('"') + + if description.endswith('"') and not description.endswith(r"\""): + description = description[:-1] + + value += ":{}".format(description) + + value += '"' + + return value + + +@SubCommand( + "mach-completion", + "bash", + description="Print mach completion script for bash shell", +) +@CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", +) +def completion_bash(command_context, outfile): + commands_subcommands = [] + case_options = [] + case_subcommands = [] + for i, cmd in enumerate(commands_info(command_context)): + # Build case statement for options. + options = [] + for opt_strs, description in cmd.options.items(): + for opt in opt_strs: + options.append(_zsh_describe(opt, None).strip('"')) + + if options: + case_options.append( + "\n".join( + [ + " ({})".format(cmd.name), + ' opts="${{opts}} {}"'.format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommand options. + for sub in cmd.subcommands: + options = [] + for opt_strs, description in sub.options.items(): + for opt in opt_strs: + options.append(_zsh_describe(opt, None)) + + if options: + case_options.append( + "\n".join( + [ + ' ("{} {}")'.format(sub.name, sub.subcommand), + ' opts="${{opts}} {}"'.format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommands. + subcommands = [_zsh_describe(s.subcommand, None) for s in cmd.subcommands] + if subcommands: + commands_subcommands.append( + '[{}]=" {} "'.format( + cmd.name, " ".join([h.subcommand for h in cmd.subcommands]) + ) + ) + + case_subcommands.append( + "\n".join( + [ + " ({})".format(cmd.name), + ' subs="${{subs}} {}"'.format(" ".join(subcommands)), + " ;;", + "", + ] + ) + ) + + globalopts = [ + opt for opt_strs in global_options(command_context) for opt in opt_strs + ] + context = { + "case_options": "\n".join(case_options), + "case_subcommands": "\n".join(case_subcommands), + "commands": " ".join(commands(command_context)), + "commands_subcommands": " ".join(sorted(commands_subcommands)), + "globalopts": " ".join(sorted(globalopts)), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("bash", context), file=outfile) + + +@SubCommand( + "mach-completion", + "zsh", + description="Print mach completion script for zsh shell", +) +@CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", +) +def completion_zsh(command_context, outfile): + commands_descriptions = [] + commands_subcommands = [] + case_options = [] + case_subcommands = [] + for i, cmd in enumerate(commands_info(command_context)): + commands_descriptions.append(_zsh_describe(cmd.name, cmd.description)) + + # Build case statement for options. + options = [] + for opt_strs, description in cmd.options.items(): + for opt in opt_strs: + options.append(_zsh_describe(opt, description)) + + if options: + case_options.append( + "\n".join( + [ + " ({})".format(cmd.name), + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommand options. + for sub in cmd.subcommands: + options = [] + for opt_strs, description in sub.options.items(): + for opt in opt_strs: + options.append(_zsh_describe(opt, description)) + + if options: + case_options.append( + "\n".join( + [ + " ({} {})".format(sub.name, sub.subcommand), + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ] + ) + ) + + # Build case statement for subcommands. + subcommands = [ + _zsh_describe(s.subcommand, s.description) for s in cmd.subcommands + ] + if subcommands: + commands_subcommands.append( + '[{}]=" {} "'.format( + cmd.name, " ".join([h.subcommand for h in cmd.subcommands]) + ) + ) + + case_subcommands.append( + "\n".join( + [ + " ({})".format(cmd.name), + " subs+=({})".format(" ".join(subcommands)), + " ;;", + "", + ] + ) + ) + + globalopts = [] + for opt_strings, description in global_options(command_context).items(): + for opt in opt_strings: + globalopts.append(_zsh_describe(opt, description)) + + context = { + "case_options": "\n".join(case_options), + "case_subcommands": "\n".join(case_subcommands), + "commands": " ".join(sorted(commands_descriptions)), + "commands_subcommands": " ".join(sorted(commands_subcommands)), + "globalopts": " ".join(sorted(globalopts)), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("zsh", context), file=outfile) + + +@SubCommand( + "mach-completion", + "fish", + description="Print mach completion script for fish shell", +) +@CommandArgument( + "-f", + "--file", + dest="outfile", + default=None, + help="File path to save completion script.", +) +def completion_fish(command_context, outfile): + def _append_opt_strs(comp, opt_strs): + for opt in opt_strs: + if opt.startswith("--"): + comp += " -l {}".format(opt[2:]) + elif opt.startswith("-"): + comp += " -s {}".format(opt[1:]) + return comp + + globalopts = [] + for opt_strs, description in global_options(command_context).items(): + comp = ( + "complete -c mach -n '__fish_mach_complete_no_command' " + "-d '{}'".format(description.replace("'", "\\'")) + ) + comp = _append_opt_strs(comp, opt_strs) + globalopts.append(comp) + + cmds = [] + cmds_opts = [] + for i, cmd in enumerate(commands_info(command_context)): + cmds.append( + "complete -c mach -f -n '__fish_mach_complete_no_command' " + "-a {} -d '{}'".format(cmd.name, cmd.description.replace("'", "\\'")) + ) + + cmds_opts += ["# {}".format(cmd.name)] + + subcommands = " ".join([s.subcommand for s in cmd.subcommands]) + for opt_strs, description in cmd.options.items(): + comp = ( + "complete -c mach -A -n '__fish_mach_complete_command {} {}' " + "-d '{}'".format(cmd.name, subcommands, description.replace("'", "\\'")) + ) + comp = _append_opt_strs(comp, opt_strs) + cmds_opts.append(comp) + + for sub in cmd.subcommands: + + for opt_strs, description in sub.options.items(): + comp = ( + "complete -c mach -A -n '__fish_mach_complete_subcommand {} {}' " + "-d '{}'".format( + sub.name, sub.subcommand, description.replace("'", "\\'") + ) + ) + comp = _append_opt_strs(comp, opt_strs) + cmds_opts.append(comp) + + description = sub.description or "" + description = description.replace("'", "\\'") + comp = ( + "complete -c mach -A -n '__fish_mach_complete_command {} {}' " + "-d '{}' -a {}".format( + cmd.name, subcommands, description, sub.subcommand + ) + ) + cmds_opts.append(comp) + + if i < len(commands(command_context)) - 1: + cmds_opts.append("") + + context = { + "commands": " ".join(commands(command_context)), + "command_completions": "\n".join(cmds), + "command_option_completions": "\n".join(cmds_opts), + "global_option_completions": "\n".join(globalopts), + } + + outfile = open(outfile, "w") if outfile else sys.stdout + print(render_template("fish", context), file=outfile) diff --git a/python/mach/mach/commands/completion_templates/bash.template b/python/mach/mach/commands/completion_templates/bash.template new file mode 100644 index 0000000000..5372308702 --- /dev/null +++ b/python/mach/mach/commands/completion_templates/bash.template @@ -0,0 +1,62 @@ +_mach_complete() +{ + local com coms comsubs cur opts script sub subs + COMPREPLY=() + declare -A comsubs=( %(commands_subcommands)s ) + + _get_comp_words_by_ref -n : cur words + # for an alias, get the real script behind it + if [[ $(type -t ${words[0]}) == "alias" ]]; then + script=$(alias ${words[0]} | sed -E "s/alias ${words[0]}='(.*)'/\\1/") + else + script=${words[0]} + fi + # lookup for command and subcommand + for word in ${words[@]:1}; do + if [[ $word == -* ]]; then + continue + fi + + if [[ -z $com ]]; then + com=$word + elif [[ "${comsubs[$com]}" == *" $word "* ]]; then + sub=$word + break + fi + done + # completing for an option + if [[ ${cur} == -* ]] ; then + if [[ -n $com ]]; then + if [[ -n $sub ]]; then + optkey="$com $sub" + else + optkey="$com" + fi + case $optkey in +%(case_options)s + esac + else + # no command, complete global options + opts="%(globalopts)s" + fi + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + __ltrim_colon_completions "$cur" + return 0; + # completing for a command + elif [[ $cur == $com ]]; then + coms="%(commands)s" + COMPREPLY=($(compgen -W "${coms}" -- ${cur})) + __ltrim_colon_completions "$cur" + return 0 + else + if [[ -z $sub ]]; then + case "$com" in +%(case_subcommands)s + esac + COMPREPLY=($(compgen -W "${subs}" -- ${cur})) + __ltrim_colon_completions "$cur" + fi + return 0 + fi +} +complete -o default -F _mach_complete mach diff --git a/python/mach/mach/commands/completion_templates/fish.template b/python/mach/mach/commands/completion_templates/fish.template new file mode 100644 index 0000000000..8373ee4080 --- /dev/null +++ b/python/mach/mach/commands/completion_templates/fish.template @@ -0,0 +1,64 @@ +function __fish_mach_complete_no_command + for i in (commandline -opc) + if contains -- $i %(commands)s + return 1 + end + end + return 0 +end + +function __fish_mach_complete_command_matches + for i in (commandline -opc) + if contains -- $i %(commands)s + set com $i + break + end + end + + if not set -q com + return 1 + end + + if test "$com" != "$argv" + return 1 + end + return 0 +end + +function __fish_mach_complete_command + __fish_mach_complete_command_matches $argv[1] + if test $status -ne 0 + return 1 + end + + # If a subcommand is already entered, don't complete, we should defer to + # '__fish_mach_complete_subcommand'. + for i in (commandline -opc) + if contains -- $i $argv[2..-1] + return 1 + end + end + return 0 +end + +function __fish_mach_complete_subcommand + __fish_mach_complete_command_matches $argv[1] + if test $status -ne 0 + return 1 + end + + # Command matches, now check for subcommand + for i in (commandline -opc) + if contains -- $i $argv[2] + return 0 + end + end + return 1 +end + +# global options +%(global_option_completions)s +# commands +%(command_completions)s +# command options +%(command_option_completions)s diff --git a/python/mach/mach/commands/completion_templates/zsh.template b/python/mach/mach/commands/completion_templates/zsh.template new file mode 100644 index 0000000000..21677841ef --- /dev/null +++ b/python/mach/mach/commands/completion_templates/zsh.template @@ -0,0 +1,62 @@ +#compdef mach +_mach_complete() +{ + local com coms comsubs cur optkey opts state sub subs + cur=${words[${#words[@]}]} + typeset -A comsubs + comsubs=( %(commands_subcommands)s ) + + # lookup for command and subcommand + for word in ${words[@]:1}; do + if [[ $word == -* ]]; then + continue + fi + + if [[ -z $com ]]; then + com=$word + elif [[ ${comsubs[$com]} == *" $word "* ]]; then + sub=$word + break + fi + done + + # check for a subcommand + if [[ $cur == $com ]]; then + state="command" + coms=(%(commands)s) + elif [[ ${cur} == -* ]]; then + state="option" + if [[ -z $com ]]; then + # no command, use global options + opts=(%(globalopts)s) + fi + fi + case $state in + (command) + _describe 'command' coms + ;; + (option) + if [[ -n $sub ]]; then + optkey="$com $sub" + else + optkey="$com" + fi + case $optkey in +%(case_options)s + esac + _describe 'option' opts + ;; + *) + if [[ -z $sub ]]; then + # if we're completing a command with subcommands, add them here + case "$com" in +%(case_subcommands)s + esac + _describe 'subcommand' subs + fi + # also fallback to file completion + _arguments '*:file:_files' + esac +} +_mach_complete "$@" +compdef _mach_complete mach diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py new file mode 100644 index 0000000000..8e168a3921 --- /dev/null +++ b/python/mach/mach/commands/settings.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from textwrap import TextWrapper + +from mach.config import TYPE_CLASSES +from mach.decorators import Command, CommandArgument + + +# Interact with settings for mach. + +# Currently, we only provide functionality to view what settings are +# available. In the future, this module will be used to modify settings, help +# people create configs via a wizard, etc. + + +@Command("settings", category="devenv", description="Show available config settings.") +@CommandArgument( + "-l", + "--list", + dest="short", + action="store_true", + help="Show settings in a concise list", +) +def run_settings(command_context, short=None): + """List available settings.""" + types = {v: k for k, v in TYPE_CLASSES.items()} + wrapper = TextWrapper(initial_indent="# ", subsequent_indent="# ") + for i, section in enumerate(sorted(command_context._mach_context.settings)): + if not short: + print("%s[%s]" % ("" if i == 0 else "\n", section)) + + for option in sorted(command_context._mach_context.settings[section]._settings): + meta = command_context._mach_context.settings[section].get_meta(option) + desc = meta["description"] + + if short: + print("%s.%s -- %s" % (section, option, desc.splitlines()[0])) + continue + + if option == "*": + option = "<option>" + + if "choices" in meta: + value = "{%s}" % ", ".join(meta["choices"]) + else: + value = "<%s>" % types[meta["type_cls"]] + + print(wrapper.fill(desc)) + print(";%s=%s" % (option, value)) diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py new file mode 100644 index 0000000000..5428a9edad --- /dev/null +++ b/python/mach/mach/config.py @@ -0,0 +1,415 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +r""" +This file defines classes for representing config data/settings. + +Config data is modeled as key-value pairs. Keys are grouped together into named +sections. Individual config settings (options) have metadata associated with +them. This metadata includes type, default value, valid values, etc. + +The main interface to config data is the ConfigSettings class. 1 or more +ConfigProvider classes are associated with ConfigSettings and define what +settings are available. +""" + +import collections +import collections.abc +import sys +from functools import wraps +from pathlib import Path +from typing import List, Union + +import six +from six import string_types +from six.moves.configparser import NoSectionError, RawConfigParser + + +class ConfigException(Exception): + pass + + +class ConfigType(object): + """Abstract base class for config values.""" + + @staticmethod + def validate(value): + """Validates a Python value conforms to this type. + + Raises a TypeError or ValueError if it doesn't conform. Does not do + anything if the value is valid. + """ + + @staticmethod + def from_config(config, section, option): + """Obtain the value of this type from a RawConfigParser. + + Receives a RawConfigParser instance, a str section name, and the str + option in that section to retrieve. + + The implementation may assume the option exists in the RawConfigParser + instance. + + Implementations are not expected to validate the value. But, they + should return the appropriate Python type. + """ + + @staticmethod + def to_config(value): + return value + + +class StringType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, string_types): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +class BooleanType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, bool): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getboolean(section, option) + + @staticmethod + def to_config(value): + return "true" if value else "false" + + +class IntegerType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getint(section, option) + + +class PositiveIntegerType(IntegerType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + if value < 0: + raise ValueError() + + +class PathType(StringType): + @staticmethod + def validate(value): + if not isinstance(value, string_types): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +TYPE_CLASSES = { + "string": StringType, + "boolean": BooleanType, + "int": IntegerType, + "pos_int": PositiveIntegerType, + "path": PathType, +} + + +class DefaultValue(object): + pass + + +def reraise_attribute_error(func): + """Used to make sure __getattr__ wrappers around __getitem__ + raise AttributeError instead of KeyError. + """ + + @wraps(func) + def _(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyError: + exc_class, exc, tb = sys.exc_info() + six.reraise(AttributeError().__class__, exc, tb) + + return _ + + +class ConfigSettings(collections.abc.Mapping): + """Interface for configuration settings. + + This is the main interface to the configuration. + + A configuration is a collection of sections. Each section contains + key-value pairs. + + When an instance is created, the caller first registers ConfigProvider + instances with it. This tells the ConfigSettings what individual settings + are available and defines extra metadata associated with those settings. + This is used for validation, etc. + + Once ConfigProvider instances are registered, a config is populated. It can + be loaded from files or populated by hand. + + ConfigSettings instances are accessed like dictionaries or by using + attributes. e.g. the section "foo" is accessed through either + settings.foo or settings['foo']. + + Sections are modeled by the ConfigSection class which is defined inside + this one. They look just like dicts or classes with attributes. To access + the "bar" option in the "foo" section: + + value = settings.foo.bar + value = settings['foo']['bar'] + value = settings.foo['bar'] + + Assignment is similar: + + settings.foo.bar = value + settings['foo']['bar'] = value + settings['foo'].bar = value + + You can even delete user-assigned values: + + del settings.foo.bar + del settings['foo']['bar'] + + If there is a default, it will be returned. + + When settings are mutated, they are validated against the registered + providers. Setting unknown settings or setting values to illegal values + will result in exceptions being raised. + """ + + class ConfigSection(collections.abc.MutableMapping, object): + """Represents an individual config section.""" + + def __init__(self, config, name, settings): + object.__setattr__(self, "_config", config) + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_settings", settings) + + wildcard = any(s == "*" for s in self._settings) + object.__setattr__(self, "_wildcard", wildcard) + + @property + def options(self): + try: + return self._config.options(self._name) + except NoSectionError: + return [] + + def get_meta(self, option): + if option in self._settings: + return self._settings[option] + if self._wildcard: + return self._settings["*"] + raise KeyError("Option not registered with provider: %s" % option) + + def _validate(self, option, value): + meta = self.get_meta(option) + meta["type_cls"].validate(value) + + if "choices" in meta and value not in meta["choices"]: + raise ValueError( + "Value '%s' must be one of: %s" + % (value, ", ".join(sorted(meta["choices"]))) + ) + + # MutableMapping interface + def __len__(self): + return len(self.options) + + def __iter__(self): + return iter(self.options) + + def __contains__(self, k): + return self._config.has_option(self._name, k) + + def __getitem__(self, k): + meta = self.get_meta(k) + + if self._config.has_option(self._name, k): + v = meta["type_cls"].from_config(self._config, self._name, k) + else: + v = meta.get("default", DefaultValue) + + if v == DefaultValue: + raise KeyError("No default value registered: %s" % k) + + self._validate(k, v) + return v + + def __setitem__(self, k, v): + self._validate(k, v) + meta = self.get_meta(k) + + if not self._config.has_section(self._name): + self._config.add_section(self._name) + + self._config.set(self._name, k, meta["type_cls"].to_config(v)) + + def __delitem__(self, k): + self._config.remove_option(self._name, k) + + # Prune empty sections. + if not len(self._config.options(self._name)): + self._config.remove_section(self._name) + + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) + + @reraise_attribute_error + def __setattr__(self, k, v): + self.__setitem__(k, v) + + @reraise_attribute_error + def __delattr__(self, k): + self.__delitem__(k) + + def __init__(self): + self._config = RawConfigParser() + self._config.optionxform = str + + self._settings = {} + self._sections = {} + self._finalized = False + + def load_file(self, filename: Union[str, Path]): + self.load_files([Path(filename)]) + + def load_files(self, filenames: List[Path]): + """Load a config from files specified by their paths. + + Files are loaded in the order given. Subsequent files will overwrite + values from previous files. If a file does not exist, it will be + ignored. + """ + filtered = [f for f in filenames if f.exists()] + + fps = [open(f, "rt") for f in filtered] + self.load_fps(fps) + for fp in fps: + fp.close() + + def load_fps(self, fps): + """Load config data by reading file objects.""" + + for fp in fps: + self._config.readfp(fp) + + def write(self, fh): + """Write the config to a file object.""" + self._config.write(fh) + + @classmethod + def _format_metadata(cls, type_cls, description, default=DefaultValue, extra=None): + """Formats and returns the metadata for a setting. + + Each setting must have: + + type_cls -- a ConfigType-derived type defining the type of the setting. + + description -- str describing how to use the setting and where it + applies. + + Each setting has the following optional parameters: + + default -- The default value for the setting. If None (the default) + there is no default. + + extra -- A dict of additional key/value pairs to add to the + setting metadata. + """ + if isinstance(type_cls, string_types): + type_cls = TYPE_CLASSES[type_cls] + + meta = {"description": description, "type_cls": type_cls} + + if default != DefaultValue: + meta["default"] = default + + if extra: + meta.update(extra) + + return meta + + def register_provider(self, provider): + """Register a SettingsProvider with this settings interface.""" + + if self._finalized: + raise ConfigException("Providers cannot be registered after finalized.") + + settings = provider.config_settings + if callable(settings): + settings = settings() + + config_settings = collections.defaultdict(dict) + for setting in settings: + section, option = setting[0].split(".") + + if option in config_settings[section]: + raise ConfigException( + "Setting has already been registered: %s.%s" % (section, option) + ) + + meta = self._format_metadata(*setting[1:]) + config_settings[section][option] = meta + + for section_name, settings in config_settings.items(): + section = self._settings.get(section_name, {}) + + for k, v in settings.items(): + if k in section: + raise ConfigException( + "Setting already registered: %s.%s" % (section_name, k) + ) + + section[k] = v + + self._settings[section_name] = section + + def _finalize(self): + if self._finalized: + return + + for section, settings in self._settings.items(): + s = ConfigSettings.ConfigSection(self._config, section, settings) + self._sections[section] = s + + self._finalized = True + + # Mapping interface. + def __len__(self): + return len(self._settings) + + def __iter__(self): + self._finalize() + + return iter(self._sections.keys()) + + def __contains__(self, k): + return k in self._settings + + def __getitem__(self, k): + self._finalize() + + return self._sections[k] + + # Allow attribute access because it looks nice. + @reraise_attribute_error + def __getattr__(self, k): + return self.__getitem__(k) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py new file mode 100644 index 0000000000..fe4443e168 --- /dev/null +++ b/python/mach/mach/decorators.py @@ -0,0 +1,340 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import collections +import collections.abc +from typing import Optional + +from mozbuild.base import MachCommandBase + +from .base import MachError +from .registrar import Registrar + + +class _MachCommand(object): + """Container for mach command metadata.""" + + __slots__ = ( + # Content from decorator arguments to define the command. + "name", + "subcommand", + "category", + "description", + "conditions", + "_parser", + "arguments", + "argument_group_names", + "virtualenv_name", + "ok_if_tests_disabled", + # By default, subcommands will be sorted. If this is set to + # 'declaration', they will be left in declaration order. + "order", + # This is the function or callable that will be called when + # the command is invoked + "func", + # The path to the `metrics.yaml` file that describes data that telemetry will + # gather for this command. This path is optional. + "metrics_path", + # Dict of string to _MachCommand defining sub-commands for this + # command. + "subcommand_handlers", + # For subcommands, the global order that the subcommand's declaration + # was seen. + "decl_order", + # Whether to disable automatic logging to last_log.json for the command. + "no_auto_log", + ) + + def __init__( + self, + name=None, + subcommand=None, + category=None, + description=None, + conditions=None, + parser=None, + order=None, + virtualenv_name=None, + ok_if_tests_disabled=False, + no_auto_log=False, + ): + self.name = name + self.subcommand = subcommand + self.category = category + self.description = description + self.conditions = conditions or [] + self._parser = parser + self.arguments = [] + self.argument_group_names = [] + self.virtualenv_name = virtualenv_name + self.order = order + if ok_if_tests_disabled and category != "testing": + raise ValueError( + "ok_if_tests_disabled should only be set for " "`testing` mach commands" + ) + self.ok_if_tests_disabled = ok_if_tests_disabled + + self.func = None + self.metrics_path = None + self.subcommand_handlers = {} + self.decl_order = None + self.no_auto_log = no_auto_log + + def create_instance(self, context, virtualenv_name): + metrics = None + if self.metrics_path: + metrics = context.telemetry.metrics(self.metrics_path) + + # This ensures the resulting class is defined inside `mach` so that logging + # works as expected, and has a meaningful name + subclass = type(self.name, (MachCommandBase,), {}) + return subclass( + context, + virtualenv_name=virtualenv_name, + metrics=metrics, + no_auto_log=self.no_auto_log, + ) + + @property + def parser(self): + # Creating CLI parsers at command dispatch time can be expensive. Make + # it possible to lazy load them by using functions. + if callable(self._parser): + self._parser = self._parser() + + return self._parser + + @property + def docstring(self): + return self.func.__doc__ + + def __ior__(self, other): + if not isinstance(other, _MachCommand): + raise ValueError("can only operate on _MachCommand instances") + + for a in self.__slots__: + if not getattr(self, a): + setattr(self, a, getattr(other, a)) + + return self + + def register(self, func): + """Register the command in the Registrar with the function to be called on invocation.""" + if not self.subcommand: + if not self.conditions and Registrar.require_conditions: + return + + msg = ( + "Mach command '%s' implemented incorrectly. " + + "Conditions argument must take a list " + + "of functions. Found %s instead." + ) + + if not isinstance(self.conditions, collections.abc.Iterable): + msg = msg % (self.name, type(self.conditions)) + raise MachError(msg) + + for c in self.conditions: + if not hasattr(c, "__call__"): + msg = msg % (self.name, type(c)) + raise MachError(msg) + + self.func = func + + Registrar.register_command_handler(self) + + else: + if self.name not in Registrar.command_handlers: + raise MachError( + "Command referenced by sub-command does not exist: %s" % self.name + ) + + self.func = func + parent = Registrar.command_handlers[self.name] + + if self.subcommand in parent.subcommand_handlers: + raise MachError("sub-command already defined: %s" % self.subcommand) + + parent.subcommand_handlers[self.subcommand] = self + + +class Command(object): + """Decorator for functions or methods that provide a mach command. + + The decorator accepts arguments that define basic attributes of the + command. The following arguments are recognized: + + category -- The string category to which this command belongs. Mach's + help will group commands by category. + + description -- A brief description of what the command does. + + parser -- an optional argparse.ArgumentParser instance or callable + that returns an argparse.ArgumentParser instance to use as the + basis for the command arguments. + + For example: + + .. code-block:: python + + @Command('foo', category='misc', description='Run the foo action') + def foo(self, command_context): + pass + """ + + def __init__(self, name, metrics_path: Optional[str] = None, **kwargs): + self._mach_command = _MachCommand(name=name, **kwargs) + self._mach_command.metrics_path = metrics_path + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + func._mach_command.register(func) + + return func + + +class SubCommand(object): + """Decorator for functions or methods that provide a sub-command. + + Mach commands can have sub-commands. e.g. ``mach command foo`` or + ``mach command bar``. Each sub-command has its own parser and is + effectively its own mach command. + + The decorator accepts arguments that define basic attributes of the + sub command: + + command -- The string of the command this sub command should be + attached to. + + subcommand -- The string name of the sub command to register. + + description -- A textual description for this sub command. + """ + + global_order = 0 + + def __init__( + self, + command, + subcommand, + description=None, + parser=None, + metrics_path: Optional[str] = None, + virtualenv_name: Optional[str] = None, + ): + self._mach_command = _MachCommand( + name=command, + subcommand=subcommand, + description=description, + parser=parser, + virtualenv_name=virtualenv_name, + ) + self._mach_command.decl_order = SubCommand.global_order + SubCommand.global_order += 1 + + self._mach_command.metrics_path = metrics_path + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command |= self._mach_command + func._mach_command.register(func) + + return func + + +class CommandArgument(object): + """Decorator for additional arguments to mach subcommands. + + This decorator should be used to add arguments to mach commands. Arguments + to the decorator are proxied to ArgumentParser.add_argument(). + + For example: + + .. code-block:: python + + @Command('foo', help='Run the foo action') + @CommandArgument('-b', '--bar', action='store_true', default=False, + help='Enable bar mode.') + def foo(self, command_context): + pass + """ + + def __init__(self, *args, **kwargs): + if kwargs.get("nargs") == argparse.REMAINDER: + # These are the assertions we make in dispatcher.py about + # those types of CommandArguments. + assert len(args) == 1 + assert all( + k in ("default", "nargs", "help", "group", "metavar") for k in kwargs + ) + self._command_args = (args, kwargs) + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command.arguments.insert(0, self._command_args) + + return func + + +class CommandArgumentGroup(object): + """Decorator for additional argument groups to mach commands. + + This decorator should be used to add arguments groups to mach commands. + Arguments to the decorator are proxied to + ArgumentParser.add_argument_group(). + + For example: + + .. code-block: python + + @Command('foo', helps='Run the foo action') + @CommandArgumentGroup('group1') + @CommandArgument('-b', '--bar', group='group1', action='store_true', + default=False, help='Enable bar mode.') + def foo(self, command_context): + pass + + The name should be chosen so that it makes sense as part of the phrase + 'Command Arguments for <name>' because that's how it will be shown in the + help message. + """ + + def __init__(self, group_name): + self._group_name = group_name + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MachCommand() + + func._mach_command.argument_group_names.insert(0, self._group_name) + + return func + + +def SettingsProvider(cls): + """Class decorator to denote that this class provides Mach settings. + + When this decorator is encountered, the underlying class will automatically + be registered with the Mach registrar and will (likely) be hooked up to the + mach driver. + """ + if not hasattr(cls, "config_settings"): + raise MachError( + "@SettingsProvider must contain a config_settings attribute. It " + "may either be a list of tuples, or a callable that returns a list " + "of tuples. Each tuple must be of the form:\n" + "(<section>.<option>, <type_cls>, <description>, <default>, <choices>)\n" + "as specified by ConfigSettings._format_metadata." + ) + + Registrar.register_settings_provider(cls) + return cls diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py new file mode 100644 index 0000000000..95287eac40 --- /dev/null +++ b/python/mach/mach/dispatcher.py @@ -0,0 +1,516 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import difflib +import shlex +import sys +from operator import itemgetter + +from .base import NoCommandError, UnknownCommandError, UnrecognizedArgumentError +from .decorators import SettingsProvider + + +@SettingsProvider +class DispatchSettings: + config_settings = [ + ( + "alias.*", + "string", + """ +Create a command alias of the form `<alias>=<command> <args>`. +Aliases can also be used to set default arguments: +<command>=<command> <args> +""".strip(), + ), + ] + + +class CommandFormatter(argparse.HelpFormatter): + """Custom formatter to format just a subcommand.""" + + def add_usage(self, *args): + pass + + +class CommandAction(argparse.Action): + """An argparse action that handles mach commands. + + This class is essentially a reimplementation of argparse's sub-parsers + feature. We first tried to use sub-parsers. However, they were missing + features like grouping of commands (http://bugs.python.org/issue14037). + + The way this works involves light magic and a partial understanding of how + argparse works. + + Arguments registered with an argparse.ArgumentParser have an action + associated with them. An action is essentially a class that when called + does something with the encountered argument(s). This class is one of those + action classes. + + An instance of this class is created doing something like: + + parser.add_argument('command', action=CommandAction, registrar=r) + + Note that a mach.registrar.Registrar instance is passed in. The Registrar + holds information on all the mach commands that have been registered. + + When this argument is registered with the ArgumentParser, an instance of + this class is instantiated. One of the subtle but important things it does + is tell the argument parser that it's interested in *all* of the remaining + program arguments. So, when the ArgumentParser calls this action, we will + receive the command name plus all of its arguments. + + For more, read the docs in __call__. + """ + + def __init__( + self, + option_strings, + dest, + required=True, + default=None, + registrar=None, + context=None, + ): + # A proper API would have **kwargs here. However, since we are a little + # hacky, we intentionally omit it as a way of detecting potentially + # breaking changes with argparse's implementation. + # + # In a similar vein, default is passed in but is not needed, so we drop + # it. + argparse.Action.__init__( + self, + option_strings, + dest, + required=required, + help=argparse.SUPPRESS, + nargs=argparse.REMAINDER, + ) + + self._mach_registrar = registrar + self._context = context + + def __call__(self, parser, namespace, values, option_string=None): + """This is called when the ArgumentParser has reached our arguments. + + Since we always register ourselves with nargs=argparse.REMAINDER, + values should be a list of remaining arguments to parse. The first + argument should be the name of the command to invoke and all remaining + arguments are arguments for that command. + + The gist of the flow is that we look at the command being invoked. If + it's *help*, we handle that specially (because argparse's default help + handler isn't satisfactory). Else, we create a new, independent + ArgumentParser instance for just the invoked command (based on the + information contained in the command registrar) and feed the arguments + into that parser. We then merge the results with the main + ArgumentParser. + """ + if namespace.help: + # -h or --help is in the global arguments. + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif values: + command = values[0].lower() + args = values[1:] + if command == "help": + if args and args[0] not in ["-h", "--help"]: + # Make sure args[0] is indeed a command. + self._handle_command_help(parser, args[0], args) + else: + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif "-h" in args or "--help" in args: + # -h or --help is in the command arguments. + if "--" in args: + # -- is in command arguments + if ( + "-h" in args[: args.index("--")] + or "--help" in args[: args.index("--")] + ): + # Honor -h or --help only if it appears before -- + self._handle_command_help(parser, command, args) + sys.exit(0) + else: + self._handle_command_help(parser, command, args) + sys.exit(0) + else: + raise NoCommandError(namespace) + + # First see if the this is a user-defined alias + if command in self._context.settings.alias: + alias = self._context.settings.alias[command] + defaults = shlex.split(alias) + command = defaults.pop(0) + args = defaults + args + + if command not in self._mach_registrar.command_handlers: + # Try to find similar commands, may raise UnknownCommandError. + command = self._suggest_command(command) + + handler = self._mach_registrar.command_handlers.get(command) + + prog = command + usage = "%(prog)s [global arguments] " + command + " [command arguments]" + + subcommand = None + + # If there are sub-commands, parse the intent out immediately. + if handler.subcommand_handlers and args: + # mach <command> help <subcommand> + if set(args[: args.index("--")] if "--" in args else args).intersection( + ("help", "--help") + ): + self._handle_subcommand_help(parser, handler, args) + sys.exit(0) + # mach <command> <subcommand> ... + elif args[0] in handler.subcommand_handlers: + subcommand = args[0] + handler = handler.subcommand_handlers[subcommand] + prog = prog + " " + subcommand + usage = ( + "%(prog)s [global arguments] " + + command + + " " + + subcommand + + " [command arguments]" + ) + args.pop(0) + + # We create a new parser, populate it with the command's arguments, + # then feed all remaining arguments to it, merging the results + # with ourselves. This is essentially what argparse subparsers + # do. + + parser_args = { + "add_help": False, + "usage": usage, + } + + remainder = None + + if handler.parser: + subparser = handler.parser + subparser.context = self._context + subparser.prog = subparser.prog + " " + prog + for arg in subparser._actions[:]: + if arg.nargs == argparse.REMAINDER: + subparser._actions.remove(arg) + remainder = ( + (arg.dest,), + {"default": arg.default, "nargs": arg.nargs, "help": arg.help}, + ) + else: + subparser = argparse.ArgumentParser(**parser_args) + + for arg in handler.arguments: + # Remove our group keyword; it's not needed here. + group_name = arg[1].get("group") + if group_name: + del arg[1]["group"] + + if arg[1].get("nargs") == argparse.REMAINDER: + # parse_known_args expects all argparse.REMAINDER ('...') + # arguments to be all stuck together. Instead, we want them to + # pick any extra argument, wherever they are. + # Assume a limited CommandArgument for those arguments. + assert len(arg[0]) == 1 + assert all(k in ("default", "nargs", "help", "metavar") for k in arg[1]) + remainder = arg + else: + subparser.add_argument(*arg[0], **arg[1]) + + # We define the command information on the main parser result so as to + # not interfere with arguments passed to the command. + setattr(namespace, "mach_handler", handler) + setattr(namespace, "command", command) + setattr(namespace, "subcommand", subcommand) + + command_namespace, extra = subparser.parse_known_args(args) + setattr(namespace, "command_args", command_namespace) + if remainder: + (name,), options = remainder + # parse_known_args usefully puts all arguments after '--' in + # extra, but also puts '--' there. We don't want to pass it down + # to the command handler. Note that if multiple '--' are on the + # command line, only the first one is removed, so that subsequent + # ones are passed down. + if "--" in extra: + extra.remove("--") + + # Commands with argparse.REMAINDER arguments used to force the + # other arguments to be '+' prefixed. If a user now passes such + # an argument, if will silently end up in extra. So, check if any + # of the allowed arguments appear in a '+' prefixed form, and error + # out if that's the case. + for args, _ in handler.arguments: + for arg in args: + arg = arg.replace("-", "+", 1) + if arg in extra: + raise UnrecognizedArgumentError(command, [arg]) + + if extra: + setattr(command_namespace, name, extra) + else: + setattr(command_namespace, name, options.get("default", [])) + elif extra: + raise UnrecognizedArgumentError(command, extra) + + def _handle_main_help(self, parser, verbose): + # Since we don't need full sub-parser support for the main help output, + # we create groups in the ArgumentParser and populate each group with + # arguments corresponding to command names. This has the side-effect + # that argparse renders it nicely. + r = self._mach_registrar + disabled_commands = [] + + cats = [(k, v[2]) for k, v in r.categories.items()] + sorted_cats = sorted(cats, key=itemgetter(1), reverse=True) + for category, priority in sorted_cats: + group = None + + for command in sorted(r.commands_by_category[category]): + handler = r.command_handlers[command] + + # Instantiate a handler class to see if it should be filtered + # out for the current context or not. Condition functions can be + # applied to the command's decorator. + if handler.conditions: + instance = handler.create_instance( + self._context, handler.virtualenv_name + ) + + is_filtered = False + for c in handler.conditions: + if not c(instance): + is_filtered = True + break + if is_filtered: + description = handler.description + disabled_command = { + "command": command, + "description": description, + } + disabled_commands.append(disabled_command) + continue + + if group is None: + title, description, _priority = r.categories[category] + group = parser.add_argument_group(title, description) + + description = handler.description + group.add_argument(command, help=description, action="store_true") + + if disabled_commands and "disabled" in r.categories: + title, description, _priority = r.categories["disabled"] + group = parser.add_argument_group(title, description) + if verbose: + for c in disabled_commands: + group.add_argument( + c["command"], help=c["description"], action="store_true" + ) + + parser.print_help() + + def _populate_command_group(self, parser, handler, group): + extra_groups = {} + for group_name in handler.argument_group_names: + group_full_name = "Command Arguments for " + group_name + extra_groups[group_name] = parser.add_argument_group(group_full_name) + + for arg in handler.arguments: + # Apply our group keyword. + group_name = arg[1].get("group") + if group_name: + del arg[1]["group"] + group = extra_groups[group_name] + group.add_argument(*arg[0], **arg[1]) + + def _get_command_arguments_help(self, handler): + # This code is worth explaining. Because we are doing funky things with + # argument registration to allow the same option in both global and + # command arguments, we can't simply put all arguments on the same + # parser instance because argparse would complain. We can't register an + # argparse subparser here because it won't properly show help for + # global arguments. So, we employ a strategy similar to command + # execution where we construct a 2nd, independent ArgumentParser for + # just the command data then supplement the main help's output with + # this 2nd parser's. We use a custom formatter class to ignore some of + # the help output. + parser_args = { + "formatter_class": CommandFormatter, + "add_help": False, + } + + if handler.parser: + c_parser = handler.parser + c_parser.context = self._context + c_parser.formatter_class = NoUsageFormatter + # Accessing _action_groups is a bit shady. We are highly dependent + # on the argparse implementation not changing. We fail fast to + # detect upstream changes so we can intelligently react to them. + group = c_parser._action_groups[1] + + # By default argparse adds two groups called "positional arguments" + # and "optional arguments". We want to rename these to reflect standard + # mach terminology. + c_parser._action_groups[0].title = "Command Parameters" + c_parser._action_groups[1].title = "Command Arguments" + + if not handler.description: + handler.description = c_parser.description + c_parser.description = None + else: + c_parser = argparse.ArgumentParser(**parser_args) + group = c_parser.add_argument_group("Command Arguments") + + self._populate_command_group(c_parser, handler, group) + + return c_parser + + def _handle_command_help(self, parser, command, args): + handler = self._mach_registrar.command_handlers.get(command) + + if not handler: + raise UnknownCommandError(command, "query") + + if handler.subcommand_handlers: + self._handle_subcommand_help(parser, handler, args) + return + + c_parser = self._get_command_arguments_help(handler) + + # Set the long help of the command to the docstring (if present) or + # the command decorator description argument (if present). + if handler.docstring: + parser.description = format_docstring(handler.docstring) + elif handler.description: + parser.description = handler.description + + parser.usage = "%(prog)s [global arguments] " + command + " [command arguments]" + + # This is needed to preserve line endings in the description field, + # which may be populated from a docstring. + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.print_help() + print("") + c_parser.print_help() + + def _handle_subcommand_main_help(self, parser, handler): + parser.usage = ( + "%(prog)s [global arguments] " + + handler.name + + " subcommand [subcommand arguments]" + ) + group = parser.add_argument_group("Sub Commands") + + def by_decl_order(item): + return item[1].decl_order + + def by_name(item): + return item[1].subcommand + + subhandlers = handler.subcommand_handlers.items() + for subcommand, subhandler in sorted( + subhandlers, + key=by_decl_order if handler.order == "declaration" else by_name, + ): + group.add_argument( + subcommand, help=subhandler.description, action="store_true" + ) + + if handler.docstring: + parser.description = format_docstring(handler.docstring) + + c_parser = self._get_command_arguments_help(handler) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + + parser.print_help() + print("") + c_parser.print_help() + + def _handle_subcommand_help(self, parser, handler, args): + subcommand = set(args).intersection(list(handler.subcommand_handlers.keys())) + if not subcommand: + return self._handle_subcommand_main_help(parser, handler) + + subcommand = subcommand.pop() + subhandler = handler.subcommand_handlers[subcommand] + + # Initialize the parser if necessary + subhandler.parser + + c_parser = subhandler.parser or argparse.ArgumentParser(add_help=False) + c_parser.formatter_class = CommandFormatter + + group = c_parser.add_argument_group("Sub Command Arguments") + self._populate_command_group(c_parser, subhandler, group) + + if subhandler.docstring: + parser.description = format_docstring(subhandler.docstring) + + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.usage = ( + "%(prog)s [global arguments] " + + handler.name + + " " + + subcommand + + " [command arguments]" + ) + + parser.print_help() + print("") + c_parser.print_help() + + def _suggest_command(self, command): + names = [h.name for h in self._mach_registrar.command_handlers.values()] + # We first try to look for a valid command that is very similar to the given command. + suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8) + # If we find more than one matching command, or no command at all, + # we give command suggestions instead (with a lower matching threshold). + # All commands that start with the given command (for instance: + # 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-') + # are also included. + if len(suggested_commands) != 1: + suggested_commands = set( + difflib.get_close_matches(command, names, cutoff=0.5) + ) + suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} + raise UnknownCommandError(command, "run", suggested_commands) + sys.stderr.write( + "We're assuming the '%s' command is '%s' and we're " + "executing it for you.\n\n" % (command, suggested_commands[0]) + ) + return suggested_commands[0] + + +class NoUsageFormatter(argparse.HelpFormatter): + def _format_usage(self, *args, **kwargs): + return "" + + +def format_docstring(docstring): + """Format a raw docstring into something suitable for presentation. + + This function is based on the example function in PEP-0257. + """ + if not docstring: + return "" + lines = docstring.expandtabs().splitlines() + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + return "\n".join(trimmed) diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py new file mode 100644 index 0000000000..d39f336cc0 --- /dev/null +++ b/python/mach/mach/logging.py @@ -0,0 +1,398 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains logging functionality for mach. It essentially provides +# support for a structured logging framework built on top of Python's built-in +# logging framework. + +import codecs +import json +import logging +import os +import sys +import time + +import blessed +import six +from mozbuild.util import mozilla_build_version +from packaging.version import Version + +IS_WINDOWS = sys.platform.startswith("win") + +if IS_WINDOWS: + import msvcrt + from ctypes import byref, windll + from ctypes.wintypes import DWORD + + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + + def enable_virtual_terminal_processing(file_descriptor): + handle = msvcrt.get_osfhandle(file_descriptor) + try: + mode = DWORD() + windll.kernel32.GetConsoleMode(handle, byref(mode)) + mode.value |= ENABLE_VIRTUAL_TERMINAL_PROCESSING + windll.kernel32.SetConsoleMode(handle, mode.value) + except Exception as e: + raise e + + +def enable_blessed(): + # Only Windows has issues with enabling blessed + # and interpreting ANSI escape sequences + if not IS_WINDOWS: + return True + + if os.environ.get("NO_ANSI"): + return False + + # MozillaBuild 4.0.2 is the first Release that supports + # ANSI escape sequences, so if we're greater than that + # version, we can enable them (via Blessed). + return mozilla_build_version() >= Version("4.0.2") + + +# stdout and stderr may not necessarily be set up to write Unicode output, so +# reconfigure them if necessary. +def _wrap_stdstream(fh): + if fh in (sys.stderr, sys.stdout): + encoding = sys.getdefaultencoding() + encoding = "utf-8" if encoding in ("ascii", "charmap") else encoding + if six.PY2: + return codecs.getwriter(encoding)(fh, errors="replace") + else: + return codecs.getwriter(encoding)(fh.buffer, errors="replace") + else: + return fh + + +def format_seconds(total): + """Format number of seconds to MM:SS.DD form.""" + + minutes, seconds = divmod(total, 60) + + return "%2d:%05.2f" % (minutes, seconds) + + +class ConvertToStructuredFilter(logging.Filter): + """Filter that converts unstructured records into structured ones.""" + + def filter(self, record): + if hasattr(record, "action") and hasattr(record, "params"): + return True + + record.action = "unstructured" + record.params = {"msg": record.getMessage()} + record.msg = "{msg}" + + return True + + +class StructuredJSONFormatter(logging.Formatter): + """Log formatter that writes a structured JSON entry.""" + + def format(self, record): + action = getattr(record, "action", "UNKNOWN") + params = getattr(record, "params", {}) + + return json.dumps([record.created, action, params]) + + +class StructuredHumanFormatter(logging.Formatter): + """Log formatter that writes structured messages for humans. + + It is important that this formatter never be added to a logger that + produces unstructured/classic log messages. If it is, the call to format() + could fail because the string could contain things (like JSON) that look + like formatting character sequences. + + Because of this limitation, format() will fail with a KeyError if an + unstructured record is passed or if the structured message is malformed. + """ + + def __init__(self, start_time, write_interval=False, write_times=True): + self.start_time = start_time + self.write_interval = write_interval + self.write_times = write_times + self.last_time = None + + def format(self, record): + formatted_msg = record.msg.format(**getattr(record, "params", {})) + + elapsed_time = ( + format_seconds(self._time(record)) + " " if self.write_times else "" + ) + + rv = elapsed_time + formatted_msg + formatted_stack_trace_result = formatted_stack_trace(record, self) + + if formatted_stack_trace_result != "": + stack_trace = "\n" + elapsed_time + formatted_stack_trace_result + rv += stack_trace.replace("\n", f"\n{elapsed_time}") + + return rv + + def _time(self, record): + t = record.created - self.start_time + + if self.write_interval and self.last_time is not None: + t = record.created - self.last_time + + self.last_time = record.created + + return t + + +class StructuredTerminalFormatter(StructuredHumanFormatter): + """Log formatter for structured messages writing to a terminal.""" + + def set_terminal(self, terminal): + self.terminal = terminal + self._sgr0 = terminal.normal if terminal else "" + + def format(self, record): + formatted_msg = record.msg.format(**getattr(record, "params", {})) + elapsed_time = ( + self.terminal.blue(format_seconds(self._time(record))) + " " + if self.write_times + else "" + ) + + rv = elapsed_time + self._colorize(formatted_msg) + self._sgr0 + formatted_stack_trace_result = formatted_stack_trace(record, self) + + if formatted_stack_trace_result != "": + stack_trace = "\n" + elapsed_time + formatted_stack_trace_result + rv += stack_trace.replace("\n", f"\n{elapsed_time}") + + # Some processes (notably Clang) don't reset terminal attributes after + # printing newlines. This can lead to terminal attributes getting in a + # wonky state. Work around this by sending the sgr0 sequence after every + # line to reset all attributes. For programs that rely on the next line + # inheriting the same attributes, this will prevent that from happening. + # But that's better than "corrupting" the terminal. + return rv + self._sgr0 + + def _colorize(self, s): + if not self.terminal: + return s + + result = s + + reftest = s.startswith("REFTEST ") + if reftest: + s = s[8:] + + if s.startswith("TEST-PASS"): + result = self.terminal.green(s[0:9]) + s[9:] + elif s.startswith("TEST-UNEXPECTED"): + result = self.terminal.red(s[0:20]) + s[20:] + elif s.startswith("TEST-START"): + result = self.terminal.yellow(s[0:10]) + s[10:] + elif s.startswith("TEST-INFO"): + result = self.terminal.yellow(s[0:9]) + s[9:] + + if reftest: + result = "REFTEST " + result + + return result + + +def formatted_stack_trace(record, formatter): + """ + Formatting behavior here intended to mimic a portion of the + standard library's logging.Formatter::format function + """ + rv = "" + + if record.exc_info: + # Cache the traceback text to avoid converting it multiple times + # (it's constant anyway) + if not record.exc_text: + record.exc_text = formatter.formatException(record.exc_info) + if record.exc_text: + rv = record.exc_text + if record.stack_info: + if rv[-1:] != "\n": + rv = rv + "\n" + rv = rv + formatter.formatStack(record.stack_info) + + return rv + + +class LoggingManager(object): + """Holds and controls global logging state. + + An application should instantiate one of these and configure it as needed. + + This class provides a mechanism to configure the output of logging data + both from mach and from the overall logging system (e.g. from other + modules). + """ + + def __init__(self): + self.start_time = time.time() + + self.json_handlers = [] + self.terminal_handler = None + self.terminal_formatter = None + + self.root_logger = logging.getLogger() + self.root_logger.setLevel(logging.DEBUG) + + # Installing NullHandler on the root logger ensures that *all* log + # messages have at least one handler. This prevents Python from + # complaining about "no handlers could be found for logger XXX." + self.root_logger.addHandler(logging.NullHandler()) + + mach_logger = logging.getLogger("mach") + mach_logger.setLevel(logging.DEBUG) + + self.structured_filter = ConvertToStructuredFilter() + + self.structured_loggers = [mach_logger] + + self._terminal = None + + def create_terminal(self): + if enable_blessed(): + # Sometimes blessed fails to set up the terminal, in that case, silently fail. + try: + terminal = blessed.Terminal(stream=_wrap_stdstream(sys.stdout)) + + if terminal.is_a_tty: + self._terminal = terminal + except Exception: + pass + + @property + def terminal(self): + return self._terminal + + def add_json_handler(self, fh): + """Enable JSON logging on the specified file object.""" + + # Configure the consumer of structured messages. + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(StructuredJSONFormatter()) + handler.setLevel(logging.DEBUG) + + # And hook it up. + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.json_handlers.append(handler) + + def add_terminal_logging( + self, fh=sys.stdout, level=logging.INFO, write_interval=False, write_times=True + ): + """Enable logging to the terminal.""" + self.create_terminal() + + if IS_WINDOWS: + try: + # fileno() can raise in some cases, like unit tests. + # so we can try to enable this but if we fail it's fine + enable_virtual_terminal_processing(sys.stdout.fileno()) + enable_virtual_terminal_processing(sys.stderr.fileno()) + except Exception: + pass + + fh = _wrap_stdstream(fh) + formatter = StructuredHumanFormatter( + self.start_time, write_interval=write_interval, write_times=write_times + ) + + if self.terminal: + formatter = StructuredTerminalFormatter( + self.start_time, write_interval=write_interval, write_times=write_times + ) + formatter.set_terminal(self.terminal) + + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(formatter) + handler.setLevel(level) + + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + self.terminal_formatter = formatter + + def replace_terminal_handler(self, handler): + """Replace the installed terminal handler. + + Returns the old handler or None if none was configured. + If the new handler is None, removes any existing handler and disables + logging to the terminal. + """ + old = self.terminal_handler + + if old: + for logger in self.structured_loggers: + logger.removeHandler(old) + + if handler: + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + + return old + + def enable_unstructured(self): + """Enable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.addFilter(self.structured_filter) + self.root_logger.addHandler(self.terminal_handler) + + def disable_unstructured(self): + """Disable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.removeFilter(self.structured_filter) + self.root_logger.removeHandler(self.terminal_handler) + + def register_structured_logger(self, logger, terminal=True, json=True): + """Register a structured logger. + + This needs to be called for all structured loggers that don't chain up + to the mach logger in order for their output to be captured. + """ + self.structured_loggers.append(logger) + + if terminal and self.terminal_handler: + logger.addHandler(self.terminal_handler) + + if json: + for handler in self.json_handlers: + logger.addHandler(handler) + + def enable_all_structured_loggers(self, terminal=True, json=True): + """Enable logging of all structured messages from all loggers. + + ``terminal`` and ``json`` determine which log handlers to operate + on. By default, all known handlers are operated on. + """ + + # Glean makes logs that we're not interested in, so we squelch them. + logging.getLogger("glean").setLevel(logging.CRITICAL) + + # Remove current handlers from all loggers so we don't double + # register handlers. + for logger in self.root_logger.manager.loggerDict.values(): + # Some entries might be logging.PlaceHolder. + if not isinstance(logger, logging.Logger): + continue + + if terminal: + logger.removeHandler(self.terminal_handler) + + if json: + for handler in self.json_handlers: + logger.removeHandler(handler) + + # Wipe out existing registered structured loggers since they + # all propagate to root logger. + self.structured_loggers = [] + self.register_structured_logger(self.root_logger, terminal=terminal, json=json) diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py new file mode 100644 index 0000000000..9ab880341d --- /dev/null +++ b/python/mach/mach/main.py @@ -0,0 +1,735 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides functionality for the command-line build tool +# (mach). It is packaged as a module because everything is a library. + +import argparse +import codecs +import errno +import imp +import logging +import os +import sys +import traceback +import uuid +from collections.abc import Iterable +from pathlib import Path +from typing import Dict, List, Union + +from .base import ( + CommandContext, + FailedCommandError, + MachError, + MissingFileError, + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) +from .config import ConfigSettings +from .dispatcher import CommandAction +from .logging import LoggingManager +from .registrar import Registrar +from .sentry import NoopErrorReporter, register_sentry +from .telemetry import create_telemetry_from_environment, report_invocation_metrics +from .util import UserError, setenv + +SUGGEST_MACH_BUSTED_TEMPLATE = r""" +You can invoke ``./mach busted`` to check if this issue is already on file. If it +isn't, please use ``./mach busted file %s`` to report it. If ``./mach busted`` is +misbehaving, you can also inspect the dependencies of bug 1543241. +""".lstrip() + +MACH_ERROR_TEMPLATE = ( + r""" +The error occurred in mach itself. This is likely a bug in mach itself or a +fundamental problem with a loaded module. + +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +ERROR_FOOTER = r""" +If filing a bug, please include the full output of mach, including this error +message. + +The details of the failure are as follows: +""".lstrip() + +USER_ERROR = r""" +This is a user error and does not appear to be a bug in mach. +""".lstrip() + +COMMAND_ERROR_TEMPLATE = ( + r""" +The error occurred in the implementation of the invoked mach command. + +This should never occur and is likely a bug in the implementation of that +command. +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +MODULE_ERROR_TEMPLATE = ( + r""" +The error occurred in code that was called by the mach command. This is either +a bug in the called code itself or in the way that mach is calling it. +""".lstrip() + + SUGGEST_MACH_BUSTED_TEMPLATE +) + +NO_COMMAND_ERROR = r""" +It looks like you tried to run mach without a command. + +Run ``mach help`` to show a list of commands. +""".lstrip() + +UNKNOWN_COMMAND_ERROR = r""" +It looks like you are trying to %s an unknown mach command: %s +%s +Run ``mach help`` to show a list of commands. +""".lstrip() + +SUGGESTED_COMMANDS_MESSAGE = r""" +Did you want to %s any of these commands instead: %s? +""" + +UNRECOGNIZED_ARGUMENT_ERROR = r""" +It looks like you passed an unrecognized argument into mach. + +The %s command does not accept the arguments: %s +""".lstrip() + +INVALID_ENTRY_POINT = r""" +Entry points should return a list of command providers or directories +containing command providers. The following entry point is invalid: + + %s + +You are seeing this because there is an error in an external module attempting +to implement a mach command. Please fix the error, or uninstall the module from +your system. +""".lstrip() + + +class ArgumentParser(argparse.ArgumentParser): + """Custom implementation argument parser to make things look pretty.""" + + def error(self, message): + """Custom error reporter to give more helpful text on bad commands.""" + if not message.startswith("argument command: invalid choice"): + argparse.ArgumentParser.error(self, message) + assert False + + print("Invalid command specified. The list of commands is below.\n") + self.print_help() + sys.exit(1) + + def format_help(self): + text = argparse.ArgumentParser.format_help(self) + + # Strip out the silly command list that would preceed the pretty list. + # + # Commands: + # {foo,bar} + # foo Do foo. + # bar Do bar. + search = "Commands:\n {" + start = text.find(search) + + if start != -1: + end = text.find("}\n", start) + assert end != -1 + + real_start = start + len("Commands:\n") + real_end = end + len("}\n") + + text = text[0:real_start] + text[real_end:] + + return text + + +class ContextWrapper(object): + def __init__(self, context, handler): + object.__setattr__(self, "_context", context) + object.__setattr__(self, "_handler", handler) + + def __getattribute__(self, key): + try: + return getattr(object.__getattribute__(self, "_context"), key) + except AttributeError as e: + try: + ret = object.__getattribute__(self, "_handler")(key) + except (AttributeError, TypeError): + # TypeError is in case the handler comes from old code not + # taking a key argument. + raise e + setattr(self, key, ret) + return ret + + def __setattr__(self, key, value): + setattr(object.__getattribute__(self, "_context"), key, value) + + +class MachCommandReference: + """A reference to a mach command. + + Holds the metadata for a mach command. + """ + + module: Path + + def __init__(self, module: Union[str, Path]): + self.module = Path(module) + + +class Mach(object): + """Main mach driver type. + + This type is responsible for holding global mach state and dispatching + a command from arguments. + + The following attributes may be assigned to the instance to influence + behavior: + + populate_context_handler -- If defined, it must be a callable. The + callable signature is the following: + populate_context_handler(key=None) + It acts as a fallback getter for the mach.base.CommandContext + instance. + This allows to augment the context instance with arbitrary data + for use in command handlers. + + require_conditions -- If True, commands that do not have any condition + functions applied will be skipped. Defaults to False. + + settings_paths -- A list of files or directories in which to search + for settings files to load. + + """ + + USAGE = """%(prog)s [global arguments] command [command arguments] + +mach (German for "do") is the main interface to the Mozilla build system and +common developer tasks. + +You tell mach the command you want to perform and it does it for you. + +Some common commands are: + + %(prog)s build Build/compile the source tree. + %(prog)s help Show full help, including the list of all commands. + +To see more help for a specific command, run: + + %(prog)s help <command> +""" + + def __init__(self, cwd: str): + assert Path(cwd).is_dir() + + self.cwd = cwd + self.log_manager = LoggingManager() + self.logger = logging.getLogger(__name__) + self.settings = ConfigSettings() + self.settings_paths = [] + + if "MACHRC" in os.environ: + self.settings_paths.append(os.environ["MACHRC"]) + + self.log_manager.register_structured_logger(self.logger) + self.populate_context_handler = None + + def load_commands_from_directory(self, path: Path): + """Scan for mach commands from modules in a directory. + + This takes a path to a directory, loads the .py files in it, and + registers and found mach command providers with this mach instance. + """ + for f in sorted(path.iterdir()): + if not f.suffix == ".py" or f.name == "__init__.py": + continue + + full_path = path / f + module_name = f"mach.commands.{str(f)[0:-3]}" + + self.load_commands_from_file(full_path, module_name=module_name) + + def load_commands_from_file(self, path: Union[str, Path], module_name=None): + """Scan for mach commands from a file. + + This takes a path to a file and loads it as a Python module under the + module name specified. If no name is specified, a random one will be + chosen. + """ + if module_name is None: + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if "mach.commands" not in sys.modules: + mod = imp.new_module("mach.commands") + sys.modules["mach.commands"] = mod + + module_name = f"mach.commands.{uuid.uuid4().hex}" + + try: + imp.load_source(module_name, str(path)) + except IOError as e: + if e.errno != errno.ENOENT: + raise + + raise MissingFileError(f"{path} does not exist") + + def load_commands_from_spec( + self, spec: Dict[str, MachCommandReference], topsrcdir: str, missing_ok=False + ): + """Load mach commands based on the given spec. + + Takes a dictionary mapping command names to their metadata. + """ + modules = set(spec[command].module for command in spec) + + for path in modules: + try: + self.load_commands_from_file(topsrcdir / path) + except MissingFileError: + if not missing_ok: + raise + + def load_commands_from_entry_point(self, group="mach.providers"): + """Scan installed packages for mach command provider entry points. An + entry point is a function that returns a list of paths to files or + directories containing command providers. + + This takes an optional group argument which specifies the entry point + group to use. If not specified, it defaults to 'mach.providers'. + """ + try: + import pkg_resources + except ImportError: + print( + "Could not find setuptools, ignoring command entry points", + file=sys.stderr, + ) + return + + for entry in pkg_resources.iter_entry_points(group=group, name=None): + paths = entry.load()() + if not isinstance(paths, Iterable): + print(INVALID_ENTRY_POINT % entry) + sys.exit(1) + + for path in paths: + path = Path(path) + if path.is_file(): + self.load_commands_from_file(path) + elif path.is_dir(): + self.load_commands_from_directory(path) + else: + print(f"command provider '{path}' does not exist") + + def define_category(self, name, title, description, priority=50): + """Provide a description for a named command category.""" + + Registrar.register_category(name, title, description, priority) + + @property + def require_conditions(self): + return Registrar.require_conditions + + @require_conditions.setter + def require_conditions(self, value): + Registrar.require_conditions = value + + def run(self, argv, stdin=None, stdout=None, stderr=None): + """Runs mach with arguments provided from the command line. + + Returns the integer exit code that should be used. 0 means success. All + other values indicate failure. + """ + sentry = NoopErrorReporter() + + # If no encoding is defined, we default to UTF-8 because without this + # Python 2.7 will assume the default encoding of ASCII. This will blow + # up with UnicodeEncodeError as soon as it encounters a non-ASCII + # character in a unicode instance. We simply install a wrapper around + # the streams and restore once we have finished. + stdin = sys.stdin if stdin is None else stdin + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + + orig_stdin = sys.stdin + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + sys.stdin = stdin + sys.stdout = stdout + sys.stderr = stderr + + orig_env = dict(os.environ) + + try: + # Load settings as early as possible so things in dispatcher.py + # can use them. + for provider in Registrar.settings_providers: + self.settings.register_provider(provider) + + setting_paths_to_pass = [Path(path) for path in self.settings_paths] + self.load_settings(setting_paths_to_pass) + + if sys.version_info < (3, 0): + if stdin.encoding is None: + sys.stdin = codecs.getreader("utf-8")(stdin) + + if stdout.encoding is None: + sys.stdout = codecs.getwriter("utf-8")(stdout) + + if stderr.encoding is None: + sys.stderr = codecs.getwriter("utf-8")(stderr) + + # Allow invoked processes (which may not have a handle on the + # original stdout file descriptor) to know if the original stdout + # is a TTY. This provides a mechanism to allow said processes to + # enable emitting code codes, for example. + if os.isatty(orig_stdout.fileno()): + setenv("MACH_STDOUT_ISATTY", "1") + + return self._run(argv) + except KeyboardInterrupt: + print("mach interrupted by signal or user action. Stopping.") + return 1 + + except Exception: + # _run swallows exceptions in invoked handlers and converts them to + # a proper exit code. So, the only scenario where we should get an + # exception here is if _run itself raises. If _run raises, that's a + # bug in mach (or a loaded command module being silly) and thus + # should be reported differently. + self._print_error_header(argv, sys.stdout) + print(MACH_ERROR_TEMPLATE % "general") + + exc_type, exc_value, exc_tb = sys.exc_info() + stack = traceback.extract_tb(exc_tb) + + sentry_event_id = sentry.report_exception(exc_value) + self._print_exception( + sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id + ) + + return 1 + + finally: + os.environ.clear() + os.environ.update(orig_env) + + sys.stdin = orig_stdin + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def _run(self, argv): + if self.populate_context_handler: + topsrcdir = Path(self.populate_context_handler("topdir")) + sentry = register_sentry(argv, self.settings, topsrcdir) + else: + sentry = NoopErrorReporter() + + context = CommandContext( + cwd=self.cwd, + settings=self.settings, + log_manager=self.log_manager, + commands=Registrar, + ) + + if self.populate_context_handler: + context = ContextWrapper(context, self.populate_context_handler) + + parser = self.get_argument_parser(context) + context.global_parser = parser + + if not len(argv): + # We don't register the usage until here because if it is globally + # registered, argparse always prints it. This is not desired when + # running with --help. + parser.usage = Mach.USAGE + parser.print_usage() + return 0 + + try: + args = parser.parse_args(argv) + except NoCommandError: + print(NO_COMMAND_ERROR) + return 1 + except UnknownCommandError as e: + suggestion_message = ( + SUGGESTED_COMMANDS_MESSAGE % (e.verb, ", ".join(e.suggested_commands)) + if e.suggested_commands + else "" + ) + print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) + return 1 + except UnrecognizedArgumentError as e: + print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, " ".join(e.arguments))) + return 1 + + if not hasattr(args, "mach_handler"): + raise MachError("ArgumentParser result missing mach handler info.") + + context.is_interactive = ( + args.is_interactive + and sys.__stdout__.isatty() + and sys.__stderr__.isatty() + and not os.environ.get("MOZ_AUTOMATION", None) + ) + context.telemetry = create_telemetry_from_environment(self.settings) + + handler = getattr(args, "mach_handler") + report_invocation_metrics(context.telemetry, handler.name) + + # Add JSON logging to a file if requested. + if args.logfile: + self.log_manager.add_json_handler(args.logfile) + + # Up the logging level if requested. + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + + self.log_manager.register_structured_logger(logging.getLogger("mach")) + + write_times = True + if ( + args.log_no_times + or "MACH_NO_WRITE_TIMES" in os.environ + or "MOZ_AUTOMATION" in os.environ + ): + write_times = False + + # Always enable terminal logging. The log manager figures out if we are + # actually in a TTY or are a pipe and does the right thing. + self.log_manager.add_terminal_logging( + level=log_level, write_interval=args.log_interval, write_times=write_times + ) + + if args.settings_file: + # Argument parsing has already happened, so settings that apply + # to command line handling (e.g alias, defaults) will be ignored. + self.load_settings([Path(args.settings_file)]) + + try: + return Registrar._run_command_handler( + handler, + context, + debug_command=args.debug_command, + profile_command=args.profile_command, + **vars(args.command_args), + ) + except KeyboardInterrupt as ki: + raise ki + except FailedCommandError as e: + print(e) + return e.exit_code + except UserError: + # We explicitly don't report UserErrors to Sentry. + exc_type, exc_value, exc_tb = sys.exc_info() + # The first two frames are us and are never used. + stack = traceback.extract_tb(exc_tb)[2:] + self._print_error_header(argv, sys.stdout) + print(USER_ERROR) + self._print_exception(sys.stdout, exc_type, exc_value, stack) + return 1 + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + sentry_event_id = sentry.report_exception(exc_value) + + # The first two frames are us and are never used. + stack = traceback.extract_tb(exc_tb)[2:] + + # If we have nothing on the stack, the exception was raised as part + # of calling the @Command method itself. This likely means a + # mismatch between @CommandArgument and arguments to the method. + # e.g. there exists a @CommandArgument without the corresponding + # argument on the method. We handle that here until the module + # loader grows the ability to validate better. + if not len(stack): + print(COMMAND_ERROR_TEMPLATE % handler.name) + self._print_exception( + sys.stdout, + exc_type, + exc_value, + traceback.extract_tb(exc_tb), + sentry_event_id=sentry_event_id, + ) + return 1 + + # Split the frames into those from the module containing the + # command and everything else. + command_frames = [] + other_frames = [] + + initial_file = stack[0][0] + + for frame in stack: + if frame[0] == initial_file: + command_frames.append(frame) + else: + other_frames.append(frame) + + # If the exception was in the module providing the command, it's + # likely the bug is in the mach command module, not something else. + # If there are other frames, the bug is likely not the mach + # command's fault. + self._print_error_header(argv, sys.stdout) + + if len(other_frames): + print(MODULE_ERROR_TEMPLATE % handler.name) + else: + print(COMMAND_ERROR_TEMPLATE % handler.name) + + self._print_exception( + sys.stdout, exc_type, exc_value, stack, sentry_event_id=sentry_event_id + ) + + return 1 + + def log(self, level, action, params, format_str): + """Helper method to record a structured log event.""" + self.logger.log(level, format_str, extra={"action": action, "params": params}) + + def _print_error_header(self, argv, fh): + fh.write("Error running mach:\n\n") + fh.write(" ") + fh.write(repr(argv)) + fh.write("\n\n") + + def _print_exception(self, fh, exc_type, exc_value, stack, sentry_event_id=None): + fh.write(ERROR_FOOTER) + fh.write("\n") + + for l in traceback.format_exception_only(exc_type, exc_value): + fh.write(l) + + fh.write("\n") + for l in traceback.format_list(stack): + fh.write(l) + + if not sentry_event_id: + return + + fh.write("\nSentry event ID: {}\n".format(sentry_event_id)) + + def load_settings(self, paths: List[Path]): + """Load the specified settings files. + + If a directory is specified, the following basenames will be + searched for in this order: + + machrc, .machrc + """ + valid_names = ("machrc", ".machrc") + + def find_in_dir(base: Path): + if base.is_file(): + return base + + for name in valid_names: + path = base / name + if path.is_file(): + return path + + files = map(find_in_dir, paths) + files = filter(bool, files) + + self.settings.load_files(list(files)) + + def get_argument_parser(self, context): + """Returns an argument parser for the command-line interface.""" + + parser = ArgumentParser( + add_help=False, + usage="%(prog)s [global arguments] " "command [command arguments]", + ) + + # WARNING!!! If you add a global argument here, also add it to the + # global argument handling in the top-level `mach` script. + # Order is important here as it dictates the order the auto-generated + # help messages are printed. + global_group = parser.add_argument_group("Global Arguments") + + global_group.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="Print verbose output.", + ) + global_group.add_argument( + "-l", + "--log-file", + dest="logfile", + metavar="FILENAME", + type=argparse.FileType("a"), + help="Filename to write log data to.", + ) + global_group.add_argument( + "--log-interval", + dest="log_interval", + action="store_true", + default=False, + help="Prefix log line with interval from last message rather " + "than relative time. Note that this is NOT execution time " + "if there are parallel operations.", + ) + global_group.add_argument( + "--no-interactive", + dest="is_interactive", + action="store_false", + help="Automatically selects the default option on any " + "interactive prompts. If the output is not a terminal, " + "then --no-interactive is assumed.", + ) + suppress_log_by_default = False + if "INSIDE_EMACS" in os.environ: + suppress_log_by_default = True + global_group.add_argument( + "--log-no-times", + dest="log_no_times", + action="store_true", + default=suppress_log_by_default, + help="Do not prefix log lines with times. By default, " + "mach will prefix each output line with the time since " + "command start.", + ) + global_group.add_argument( + "-h", + "--help", + dest="help", + action="store_true", + default=False, + help="Show this help message.", + ) + global_group.add_argument( + "--debug-command", + action="store_true", + help="Start a Python debugger when command is dispatched.", + ) + global_group.add_argument( + "--profile-command", + action="store_true", + help="Capture a Python profile of the mach process as command is dispatched.", + ) + global_group.add_argument( + "--settings", + dest="settings_file", + metavar="FILENAME", + default=None, + help="Path to settings file.", + ) + + # We need to be last because CommandAction swallows all remaining + # arguments and argparse parses arguments in the order they were added. + parser.add_argument( + "command", action=CommandAction, registrar=Registrar, context=context + ) + + return parser diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/mixin/__init__.py diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py new file mode 100644 index 0000000000..4ba6955a2d --- /dev/null +++ b/python/mach/mach/mixin/logging.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + + +class LoggingMixin(object): + """Provides functionality to control logging.""" + + def populate_logger(self, name=None): + """Ensure this class instance has a logger associated with it. + + Users of this mixin that call log() will need to ensure self._logger is + a logging.Logger instance before they call log(). This function ensures + self._logger is defined by populating it if it isn't. + """ + if hasattr(self, "_logger"): + return + + if name is None: + name = ".".join([self.__module__, self.__class__.__name__]) + + self._logger = logging.getLogger(name) + + def log(self, level, action, params, format_str): + """Log a structured log event. + + A structured log event consists of a logging level, a string action, a + dictionary of attributes, and a formatting string. + + The logging level is one of the logging.* constants, such as + logging.INFO. + + The action string is essentially the enumeration of the event. Each + different type of logged event should have a different action. + + The params dict is the metadata constituting the logged event. + + The formatting string is used to convert the structured message back to + human-readable format. Conversion back to human-readable form is + performed by calling format() on this string, feeding into it the dict + of attributes constituting the event. + + Example Usage: + + .. code-block:: python + + self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, + 'User login: {username}') + """ + self._logger.log(level, format_str, extra={"action": action, "params": params}) diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py new file mode 100644 index 0000000000..d5fd733a17 --- /dev/null +++ b/python/mach/mach/mixin/process.py @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides mixins to perform process execution. + +import logging +import os +import signal +import subprocess +import sys +from pathlib import Path +from typing import Optional + +from mozprocess.processhandler import ProcessHandlerMixin + +from .logging import LoggingMixin + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if "SHELL" in os.environ: + _current_shell = os.environ["SHELL"] +elif "MOZILLABUILD" in os.environ: + mozillabuild = os.environ["MOZILLABUILD"] + if (Path(mozillabuild) / "msys2").exists(): + _current_shell = mozillabuild + "/msys2/usr/bin/sh.exe" + else: + _current_shell = mozillabuild + "/msys/bin/sh.exe" +elif "COMSPEC" in os.environ: + _current_shell = os.environ["COMSPEC"] +elif sys.platform != "win32": + # Fall back to a standard shell. + _current_shell = "/bin/sh" +else: + raise Exception("Could not detect environment shell!") + +_in_msys = False + +if ( + os.environ.get("MSYSTEM", None) in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ +): + _in_msys = True + + if not _current_shell.lower().endswith(".exe"): + _current_shell += ".exe" + + +class LineHandlingEarlyReturn(Exception): + pass + + +class ProcessExecutionMixin(LoggingMixin): + """Mix-in that provides process execution functionality.""" + + def run_process( + self, + args=None, + cwd: Optional[str] = None, + append_env=None, + explicit_env=None, + log_name=None, + log_level=logging.INFO, + line_handler=None, + require_unix_environment=False, + ensure_exit_code=0, + ignore_children=False, + pass_thru=False, + python_unbuffered=True, + ): + """Runs a single process to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + + ignore_children is proxied to mozprocess's ignore_children. + + ensure_exit_code is used to ensure the exit code of a process matches + what is expected. If it is an integer, we raise an Exception if the + exit code does not match this value. If it is True, we ensure the exit + code is 0. If it is False, we don't perform any exit code validation. + + pass_thru is a special execution mode where the child process inherits + this process's standard file handles (stdin, stdout, stderr) as well as + additional file descriptors. It should be used for interactive processes + where buffering from mozprocess could be an issue. pass_thru does not + use mozprocess. Therefore, arguments like log_name, line_handler, + and ignore_children have no effect. + + When python_unbuffered is set, the PYTHONUNBUFFERED environment variable + will be set in the child process. This is normally advantageous (see bug + 1627873) but is detrimental in certain circumstances (specifically, we + have seen issues when using pass_thru mode to open a Python subshell, as + in bug 1628838). This variable should be set to False to avoid bustage + in those circumstances. + """ + args = self._normalize_command(args, require_unix_environment) + + self.log(logging.INFO, "new_process", {"args": " ".join(args)}, "{args}") + + def handleLine(line): + # Converts str to unicode on Python 2 and bytes to str on Python 3. + if isinstance(line, bytes): + line = line.decode(sys.stdout.encoding or "utf-8", "replace") + + if line_handler: + try: + line_handler(line) + except LineHandlingEarlyReturn: + return + + if line.startswith("BUILDTASK") or not log_name: + return + + self.log(log_level, log_name, {"line": line.rstrip()}, "{line}") + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(append_env) + + if python_unbuffered: + use_env["PYTHONUNBUFFERED"] = "1" + + self.log(logging.DEBUG, "process", {"env": str(use_env)}, "Environment: {env}") + + if pass_thru: + proc = subprocess.Popen(args, cwd=cwd, env=use_env, close_fds=False) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as + # a result of Ctrl+C, proc.wait() will return a status code, and, + # we get out of the loop. If it doesn't, like e.g. gdb, we continue + # waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + else: + p = ProcessHandlerMixin( + args, + cwd=cwd, + env=use_env, + processOutputLine=[handleLine], + universal_newlines=True, + ignore_children=ignore_children, + ) + p.run() + p.processOutput() + status = None + sig = None + while status is None: + try: + if sig is None: + status = p.wait() + else: + status = p.kill(sig=sig) + except KeyboardInterrupt: + if sig is None: + sig = signal.SIGINT + elif sig == signal.SIGINT: + # If we've already tried SIGINT, escalate. + sig = signal.SIGKILL + + if ensure_exit_code is False: + return status + + if ensure_exit_code is True: + ensure_exit_code = 0 + + if status != ensure_exit_code: + raise Exception( + "Process executed with non-0 exit code %d: %s" % (status, args) + ) + + return status + + def _normalize_command(self, args, require_unix_environment): + """Adjust command arguments to run in the necessary environment. + + This exists mainly to facilitate execution of programs requiring a *NIX + shell when running on Windows. The caller specifies whether a shell + environment is required. If it is and we are running on Windows but + aren't running in the UNIX-like msys environment, then we rewrite the + command to execute via a shell. + """ + assert isinstance(args, list) and len(args) + + if not require_unix_environment or not _in_msys: + return args + + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace("\\", "/") + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:]) + return [_current_shell, "-c", cline] diff --git a/python/mach/mach/python_lockfile.py b/python/mach/mach/python_lockfile.py new file mode 100644 index 0000000000..78f201d4ed --- /dev/null +++ b/python/mach/mach/python_lockfile.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import shutil +import subprocess +import sys +from pathlib import Path + +import toml +from packaging.requirements import Requirement + + +class PoetryLockfiles: + def __init__( + self, + poetry_lockfile: Path, + pip_lockfile: Path, + ): + self.poetry_lockfile = poetry_lockfile + self.pip_lockfile = pip_lockfile + + +class PoetryHandle: + def __init__(self, work_dir: Path): + self._work_dir = work_dir + self._dependencies = {} + + def add_requirement(self, requirement: Requirement): + self._dependencies[requirement.name] = str(requirement.specifier) + + def add_requirements_in_file(self, requirements_in: Path): + with open(requirements_in) as requirements_in: + for line in requirements_in.readlines(): + if line.startswith("#"): + continue + + req = Requirement(line) + self.add_requirement(req) + + def reuse_existing_lockfile(self, lockfile_path: Path): + """Make minimal number of changes to the lockfile to satisfy new requirements""" + shutil.copy(str(lockfile_path), str(self._work_dir / "poetry.lock")) + + def generate_lockfiles(self, do_update): + """Generate pip-style lockfiles that satisfy provided requirements + + One lockfile will be made for all mandatory requirements, and then an extra, + compatible lockfile will be created for each optional requirement. + + Args: + do_update: if True, then implicitly upgrade the versions of transitive + dependencies + """ + + poetry_config = { + "name": "poetry-test", + "description": "", + "version": "0", + "authors": [], + "dependencies": {"python": "^3.7"}, + } + poetry_config["dependencies"].update(self._dependencies) + + pyproject = {"tool": {"poetry": poetry_config}} + with open(self._work_dir / "pyproject.toml", "w") as pyproject_file: + toml.dump(pyproject, pyproject_file) + + self._run_poetry(["lock"] + (["--no-update"] if not do_update else [])) + self._run_poetry(["export", "-o", "requirements.txt"]) + + return PoetryLockfiles( + self._work_dir / "poetry.lock", + self._work_dir / "requirements.txt", + ) + + def _run_poetry(self, args): + subprocess.check_call( + [sys.executable, "-m", "poetry"] + args, cwd=self._work_dir + ) diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py new file mode 100644 index 0000000000..75481596f4 --- /dev/null +++ b/python/mach/mach/registrar.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import time +from cProfile import Profile +from pathlib import Path + +import six + +from .base import MachError + +INVALID_COMMAND_CONTEXT = r""" +It looks like you tried to run a mach command from an invalid context. The %s +command failed to meet the following conditions: %s + +Run |mach help| to show a list of all commands available to the current context. +""".lstrip() + + +class MachRegistrar(object): + """Container for mach command and config providers.""" + + def __init__(self): + self.command_handlers = {} + self.commands_by_category = {} + self.settings_providers = set() + self.categories = {} + self.require_conditions = False + self.command_depth = 0 + + def register_command_handler(self, handler): + name = handler.name + + if not handler.category: + raise MachError( + "Cannot register a mach command without a " "category: %s" % name + ) + + if handler.category not in self.categories: + raise MachError( + "Cannot register a command to an undefined " + "category: %s -> %s" % (name, handler.category) + ) + + self.command_handlers[name] = handler + self.commands_by_category[handler.category].add(name) + + def register_settings_provider(self, cls): + self.settings_providers.add(cls) + + def register_category(self, name, title, description, priority=50): + self.categories[name] = (title, description, priority) + self.commands_by_category[name] = set() + + @classmethod + def _condition_failed_message(cls, name, conditions): + msg = ["\n"] + for c in conditions: + part = [" %s" % getattr(c, "__name__", c)] + if c.__doc__ is not None: + part.append(c.__doc__) + msg.append(" - ".join(part)) + return INVALID_COMMAND_CONTEXT % (name, "\n".join(msg)) + + @classmethod + def _instance(_, handler, context, **kwargs): + if context is None: + raise ValueError("Expected a non-None context.") + + prerun = getattr(context, "pre_dispatch_handler", None) + if prerun: + prerun(context, handler, args=kwargs) + + context.handler = handler + return handler.create_instance(context, handler.virtualenv_name) + + @classmethod + def _fail_conditions(_, handler, instance): + fail_conditions = [] + if handler.conditions: + for c in handler.conditions: + if not c(instance): + fail_conditions.append(c) + + return fail_conditions + + def _run_command_handler( + self, handler, context, debug_command=False, profile_command=False, **kwargs + ): + instance = MachRegistrar._instance(handler, context, **kwargs) + fail_conditions = MachRegistrar._fail_conditions(handler, instance) + if fail_conditions: + print( + MachRegistrar._condition_failed_message(handler.name, fail_conditions) + ) + return 1 + + self.command_depth += 1 + fn = handler.func + if handler.virtualenv_name: + instance.activate_virtualenv() + + profile = None + if profile_command: + profile = Profile() + profile.enable() + + start_time = time.monotonic() + + if debug_command: + import pdb + + result = pdb.runcall(fn, instance, **kwargs) + else: + result = fn(instance, **kwargs) + + end_time = time.monotonic() + + if profile_command: + profile.disable() + profile_file = ( + Path(context.topdir) / f"mach_profile_{handler.name}.cProfile" + ) + profile.dump_stats(profile_file) + print( + f'Mach command profile created at "{profile_file}". To visualize, use ' + f"snakeviz:" + ) + print("python3 -m pip install snakeviz") + print(f"python3 -m snakeviz {profile_file.name}") + + result = result or 0 + assert isinstance(result, six.integer_types) + + if not debug_command: + postrun = getattr(context, "post_dispatch_handler", None) + if postrun: + postrun( + context, + handler, + instance, + not result, + start_time, + end_time, + self.command_depth, + args=kwargs, + ) + self.command_depth -= 1 + + return result + + def dispatch(self, name, context, argv=None, subcommand=None, **kwargs): + """Dispatch/run a command. + + Commands can use this to call other commands. + """ + handler = self.command_handlers[name] + + if subcommand: + handler = handler.subcommand_handlers[subcommand] + + if handler.parser: + parser = handler.parser + + # save and restore existing defaults so **kwargs don't persist across + # subsequent invocations of Registrar.dispatch() + old_defaults = parser._defaults.copy() + parser.set_defaults(**kwargs) + kwargs, unknown = parser.parse_known_args(argv or []) + kwargs = vars(kwargs) + parser._defaults = old_defaults + + if unknown: + if subcommand: + name = "{} {}".format(name, subcommand) + parser.error( + "unrecognized arguments for {}: {}".format( + name, ", ".join(["'{}'".format(arg) for arg in unknown]) + ) + ) + + return self._run_command_handler(handler, context, **kwargs) + + +Registrar = MachRegistrar() diff --git a/python/mach/mach/requirements.py b/python/mach/mach/requirements.py new file mode 100644 index 0000000000..d5141e23f6 --- /dev/null +++ b/python/mach/mach/requirements.py @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +from pathlib import Path + +from packaging.requirements import Requirement + +THUNDERBIRD_PYPI_ERROR = """ +Thunderbird requirements definitions cannot include PyPI packages. +""".strip() + + +class PthSpecifier: + def __init__(self, path: str): + self.path = path + + +class PypiSpecifier: + def __init__(self, requirement): + self.requirement = requirement + + +class PypiOptionalSpecifier(PypiSpecifier): + def __init__(self, repercussion, requirement): + super().__init__(requirement) + self.repercussion = repercussion + + +class MachEnvRequirements: + """Requirements associated with a "site dependency manifest", as + defined in "python/sites/". + + Represents the dependencies of a site. The source files consist + of colon-delimited fields. The first field + specifies the action. The remaining fields are arguments to that + action. The following actions are supported: + + pth -- Adds the path given as argument to "mach.pth" under + the virtualenv's site packages directory. + + pypi -- Fetch the package, plus dependencies, from PyPI. + + pypi-optional -- Attempt to install the package and dependencies from PyPI. + Continue using the site, even if the package could not be installed. + + packages.txt -- Denotes that the specified path is a child manifest. It + will be read and processed as if its contents were concatenated + into the manifest being read. + + thunderbird-packages.txt -- Denotes a Thunderbird child manifest. + Thunderbird child manifests are only activated when working on Thunderbird, + and they can cannot have "pypi" or "pypi-optional" entries. + """ + + def __init__(self): + self.requirements_paths = [] + self.pth_requirements = [] + self.pypi_requirements = [] + self.pypi_optional_requirements = [] + self.vendored_requirements = [] + + def pths_as_absolute(self, topsrcdir: str): + return [ + os.path.normcase(Path(topsrcdir) / pth.path) + for pth in (self.pth_requirements + self.vendored_requirements) + ] + + @classmethod + def from_requirements_definition( + cls, + topsrcdir: str, + is_thunderbird, + only_strict_requirements, + requirements_definition, + ): + requirements = cls() + _parse_mach_env_requirements( + requirements, + Path(requirements_definition), + Path(topsrcdir), + is_thunderbird, + only_strict_requirements, + ) + return requirements + + +def _parse_mach_env_requirements( + requirements_output, + root_requirements_path: Path, + topsrcdir: Path, + is_thunderbird, + only_strict_requirements, +): + def _parse_requirements_line( + current_requirements_path: Path, line, line_number, is_thunderbird_packages_txt + ): + line = line.strip() + if not line or line.startswith("#"): + return + + action, params = line.rstrip().split(":", maxsplit=1) + if action == "pth": + path = topsrcdir / params + if not path.exists(): + # In sparse checkouts, not all paths will be populated. + return + + requirements_output.pth_requirements.append(PthSpecifier(params)) + elif action == "vendored": + requirements_output.vendored_requirements.append(PthSpecifier(params)) + elif action == "packages.txt": + _parse_requirements_definition_file( + topsrcdir / params, + is_thunderbird_packages_txt, + ) + elif action == "pypi": + if is_thunderbird_packages_txt: + raise Exception(THUNDERBIRD_PYPI_ERROR) + + requirements_output.pypi_requirements.append( + PypiSpecifier( + _parse_package_specifier(params, only_strict_requirements) + ) + ) + elif action == "pypi-optional": + if is_thunderbird_packages_txt: + raise Exception(THUNDERBIRD_PYPI_ERROR) + + if len(params.split(":", maxsplit=1)) != 2: + raise Exception( + "Expected pypi-optional package to have a repercussion " + 'description in the format "package:fallback explanation", ' + 'found "{}"'.format(params) + ) + raw_requirement, repercussion = params.split(":") + requirements_output.pypi_optional_requirements.append( + PypiOptionalSpecifier( + repercussion, + _parse_package_specifier(raw_requirement, only_strict_requirements), + ) + ) + elif action == "thunderbird-packages.txt": + if is_thunderbird: + _parse_requirements_definition_file( + topsrcdir / params, is_thunderbird_packages_txt=True + ) + else: + raise Exception("Unknown requirements definition action: %s" % action) + + def _parse_requirements_definition_file( + requirements_path: Path, is_thunderbird_packages_txt + ): + """Parse requirements file into list of requirements""" + if not requirements_path.is_file(): + raise Exception(f'Missing requirements file at "{requirements_path}"') + + requirements_output.requirements_paths.append(str(requirements_path)) + + with open(requirements_path, "r") as requirements_file: + lines = [line for line in requirements_file] + + for number, line in enumerate(lines, start=1): + _parse_requirements_line( + requirements_path, line, number, is_thunderbird_packages_txt + ) + + _parse_requirements_definition_file(root_requirements_path, False) + + +class UnexpectedFlexibleRequirementException(Exception): + def __init__(self, raw_requirement): + self.raw_requirement = raw_requirement + + +def _parse_package_specifier(raw_requirement, only_strict_requirements): + requirement = Requirement(raw_requirement) + + if only_strict_requirements and [ + s for s in requirement.specifier if s.operator != "==" + ]: + raise UnexpectedFlexibleRequirementException(raw_requirement) + return requirement diff --git a/python/mach/mach/sentry.py b/python/mach/mach/sentry.py new file mode 100644 index 0000000000..5008f8a40c --- /dev/null +++ b/python/mach/mach/sentry.py @@ -0,0 +1,222 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import abc +import re +from pathlib import Path +from threading import Thread + +import sentry_sdk +from mozversioncontrol import ( + InvalidRepoPath, + MissingUpstreamRepo, + MissingVCSTool, + get_repository_object, +) +from six import string_types + +from mach.telemetry import is_telemetry_enabled +from mach.util import get_state_dir + +# https://sentry.io/organizations/mozilla/projects/mach/ +_SENTRY_DSN = ( + "https://5cfe351fb3a24e8d82c751252b48722b@o1069899.ingest.sentry.io/6250014" +) + + +class ErrorReporter(object): + @abc.abstractmethod + def report_exception(self, exception): + """Report the exception to remote error-tracking software.""" + + +class SentryErrorReporter(ErrorReporter): + """Reports errors using Sentry.""" + + def report_exception(self, exception): + return sentry_sdk.capture_exception(exception) + + +class NoopErrorReporter(ErrorReporter): + """Drops errors instead of reporting them. + + This is useful in cases where error-reporting is specifically disabled, such as + when telemetry hasn't been allowed. + """ + + def report_exception(self, exception): + return None + + +def register_sentry(argv, settings, topsrcdir: Path): + if not is_telemetry_enabled(settings): + return NoopErrorReporter() + + global _is_unmodified_mach_core_thread + _is_unmodified_mach_core_thread = Thread( + target=_is_unmodified_mach_core, + args=[topsrcdir], + daemon=True, + ) + _is_unmodified_mach_core_thread.start() + + sentry_sdk.init( + _SENTRY_DSN, before_send=lambda event, _: _process_event(event, topsrcdir) + ) + sentry_sdk.add_breadcrumb(message="./mach {}".format(" ".join(argv))) + return SentryErrorReporter() + + +def _process_event(sentry_event, topsrcdir: Path): + # Returning nothing causes the event to be dropped: + # https://docs.sentry.io/platforms/python/configuration/filtering/#using-beforesend + repo = _get_repository_object(topsrcdir) + if repo is None: + # We don't know the repo state, so we don't know if mach files are + # unmodified. + return + + base_ref = repo.base_ref_as_hg() + if not base_ref: + # If we don't know which revision this exception is attached to, then it's + # not worth sending + return + + _is_unmodified_mach_core_thread.join() + if not _is_unmodified_mach_core_result: + return + + for map_fn in (_settle_mach_module_id, _patch_absolute_paths, _delete_server_name): + sentry_event = map_fn(sentry_event, topsrcdir) + + sentry_event["release"] = "hg-rev-{}".format(base_ref) + return sentry_event + + +def _settle_mach_module_id(sentry_event, _): + # Sentry groups issues according to the stack frames and their associated + # "module" properties. However, one of the modules is being reported + # like "mach.commands.26a828ef5164403eaff4305ab4cb0fab" (with a generated id). + # This function replaces that generated id with the static string "<generated>" + # so that grouping behaves as expected + + stacktrace_frames = sentry_event["exception"]["values"][0]["stacktrace"]["frames"] + for frame in stacktrace_frames: + module = frame.get("module") + if not module: + continue + + module = re.sub( + "mach\\.commands\\.[a-f0-9]{32}", "mach.commands.<generated>", module + ) + frame["module"] = module + return sentry_event + + +def _patch_absolute_paths(sentry_event, topsrcdir: Path): + # As discussed here (https://bugzilla.mozilla.org/show_bug.cgi?id=1636251#c28), + # we remove usernames from file names with a best-effort basis. The most likely + # place for usernames to manifest in Sentry information is within absolute paths, + # such as: "/home/mitch/dev/firefox/mach" + # We replace the state_dir, obj_dir, src_dir with "<...>" placeholders. + # Note that we also do a blanket find-and-replace of the user's name with "<user>", + # which may have ill effects if the user's name is, by happenstance, a substring + # of some other value within the Sentry event. + def recursive_patch(value, needle, replacement): + if isinstance(value, list): + return [recursive_patch(v, needle, replacement) for v in value] + elif isinstance(value, dict): + for key in list(value.keys()): + next_value = value.pop(key) + key = needle.sub(replacement, key) + value[key] = recursive_patch(next_value, needle, replacement) + return value + elif isinstance(value, string_types): + return needle.sub(replacement, value) + else: + return value + + for (target_path, replacement) in ( + (get_state_dir(), "<statedir>"), + (str(topsrcdir), "<topsrcdir>"), + (str(Path.home()), "~"), + ): + # Sentry converts "vars" to their "representations". When paths are in local + # variables on Windows, "C:\Users\MozillaUser\Desktop" becomes + # "'C:\\Users\\MozillaUser\\Desktop'". To still catch this case, we "repr" + # the home directory and scrub the beginning and end quotes, then + # find-and-replace on that. + repr_path = repr(target_path)[1:-1] + + for target in (target_path, repr_path): + # Paths in the Sentry event aren't consistent: + # * On *nix, they're mostly forward slashes. + # * On *nix, not all absolute paths start with a leading forward slash. + # * On Windows, they're mostly backslashes. + # * On Windows, `.extra."sys.argv"` uses forward slashes. + # * The Python variables in-scope captured by the Sentry report may be + # inconsistent, even for a single path. For example, on + # Windows, Mach calculates the state_dir as "C:\Users\<user>/.mozbuild". + + # Handle the case where not all absolute paths start with a leading + # forward slash: make the initial slash optional in the search string. + if target.startswith("/"): + target = "/?" + target[1:] + + # Handle all possible slash variants: our search string should match + # both forward slashes and backslashes. This is done by dynamically + # replacing each "/" and "\" with the regex "[\/\\]" (match both). + slash_regex = re.compile(r"[\/\\]") + # The regex module parses string backslash escapes before compiling the + # regex, so we need to add more backslashes: + # "[\\/\\\\]" => [\/\\] => match "/" and "\" + target = slash_regex.sub(r"[\\/\\\\]", target) + + # Compile the regex and patch the event. + needle_regex = re.compile(target, re.IGNORECASE) + sentry_event = recursive_patch(sentry_event, needle_regex, replacement) + return sentry_event + + +def _delete_server_name(sentry_event, _): + sentry_event.pop("server_name") + return sentry_event + + +def _get_repository_object(topsrcdir: Path): + try: + return get_repository_object(str(topsrcdir)) + except (InvalidRepoPath, MissingVCSTool): + return None + + +def _is_unmodified_mach_core(topsrcdir: Path): + """True if mach is unmodified compared to the public tree. + + To avoid submitting Sentry events for errors caused by user's + local changes, we attempt to detect if mach (or code affecting mach) + has been modified in the user's local state: + * In a revision off of a "ancestor to central" revision, or: + * In the working, uncommitted state. + + If "$topsrcdir/mach" and "*.py" haven't been touched, then we can be + pretty confident that the Mach behaviour that caused the exception + also exists in the public tree. + """ + global _is_unmodified_mach_core_result + + repo = _get_repository_object(topsrcdir) + try: + files = set(repo.get_outgoing_files()) | set(repo.get_changed_files()) + _is_unmodified_mach_core_result = not any( + [file for file in files if file == "mach" or file.endswith(".py")] + ) + except MissingUpstreamRepo: + # If we don't know the upstream state, we don't know if the mach files + # have been unmodified. + _is_unmodified_mach_core_result = False + + +_is_unmodified_mach_core_result = None +_is_unmodified_mach_core_thread = None diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py new file mode 100644 index 0000000000..58c1eac3fa --- /dev/null +++ b/python/mach/mach/site.py @@ -0,0 +1,1405 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains code for managing the Python import scope for Mach. This +# generally involves populating a Python virtualenv. + +import ast +import enum +import functools +import json +import os +import platform +import shutil +import site +import subprocess +import sys +import sysconfig +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Callable, Optional + +from mach.requirements import ( + MachEnvRequirements, + UnexpectedFlexibleRequirementException, +) + +PTH_FILENAME = "mach.pth" +METADATA_FILENAME = "moz_virtualenv_metadata.json" +# The following virtualenvs *may* be used in a context where they aren't allowed to +# install pip packages over the network. In such a case, they must access unvendored +# python packages via the system environment. +PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS = ("mach", "build", "common") + +_is_windows = sys.platform == "cygwin" or (sys.platform == "win32" and os.sep == "\\") + + +class VenvModuleNotFoundException(Exception): + def __init__(self): + msg = ( + 'Mach was unable to find the "venv" module, which is needed ' + "to create virtual environments in Python. You may need to " + "install it manually using the package manager for your system." + ) + super(Exception, self).__init__(msg) + + +class VirtualenvOutOfDateException(Exception): + pass + + +class MozSiteMetadataOutOfDateError(Exception): + pass + + +class InstallPipRequirementsException(Exception): + pass + + +class SiteUpToDateResult: + def __init__(self, is_up_to_date, reason=None): + self.is_up_to_date = is_up_to_date + self.reason = reason + + +class SitePackagesSource(enum.Enum): + NONE = "none" + SYSTEM = "system" + VENV = "pip" + + @classmethod + def for_mach(cls): + source = os.environ.get("MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE", "").lower() + if source == "system": + source = SitePackagesSource.SYSTEM + elif source == "none": + source = SitePackagesSource.NONE + elif source == "pip": + source = SitePackagesSource.VENV + elif source: + raise Exception( + "Unexpected MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE value, expected one " + 'of "system", "pip", "none", or to not be set' + ) + + mach_use_system_python = bool(os.environ.get("MACH_USE_SYSTEM_PYTHON")) + if source: + if mach_use_system_python: + raise Exception( + "The MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE environment variable is " + "set, so the MACH_USE_SYSTEM_PYTHON variable is redundant and " + "should be unset." + ) + return source + + # Only print this warning once for the Mach site, so we don't spam it every + # time a site handle is created. + if mach_use_system_python: + print( + 'The "MACH_USE_SYSTEM_PYTHON" environment variable is deprecated, ' + "please unset it or replace it with either " + '"MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE=system" or ' + '"MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE=none"' + ) + + return ( + SitePackagesSource.NONE + if (mach_use_system_python or os.environ.get("MOZ_AUTOMATION")) + else SitePackagesSource.VENV + ) + + +class MozSiteMetadata: + """Details about a Moz-managed python site + + When a Moz-managed site is active, its associated metadata is available + at "MozSiteMetadata.current". + + Sites that have associated virtualenvs (so, those that aren't strictly leaning on + the external python packages) will have their metadata written to + <prefix>/moz_virtualenv_metadata.json. + """ + + # Used to track which which virtualenv has been activated in-process. + current: Optional["MozSiteMetadata"] = None + + def __init__( + self, + hex_version: int, + site_name: str, + mach_site_packages_source: SitePackagesSource, + original_python: "ExternalPythonSite", + prefix: str, + ): + """ + Args: + hex_version: The python version number from sys.hexversion + site_name: The name of the site this metadata is associated with + site_packages_source: Where this site imports its + pip-installed dependencies from + mach_site_packages_source: Where the Mach site imports + its pip-installed dependencies from + original_python: The external Python site that was + used to invoke Mach. Usually the system Python, such as /usr/bin/python3 + prefix: The same value as "sys.prefix" is when running within the + associated Python site. The same thing as the "virtualenv root". + """ + + self.hex_version = hex_version + self.site_name = site_name + self.mach_site_packages_source = mach_site_packages_source + # original_python is needed for commands that tweak the system, such + # as "./mach install-moz-phab". + self.original_python = original_python + self.prefix = prefix + + def write(self, is_finalized): + raw = { + "hex_version": self.hex_version, + "virtualenv_name": self.site_name, + "mach_site_packages_source": self.mach_site_packages_source.name, + "original_python_executable": self.original_python.python_path, + "is_finalized": is_finalized, + } + with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file: + json.dump(raw, file) + + def __eq__(self, other): + return ( + type(self) == type(other) + and self.hex_version == other.hex_version + and self.site_name == other.site_name + and self.mach_site_packages_source == other.mach_site_packages_source + # On Windows, execution environment can lead to different cases. Normalize. + and Path(self.original_python.python_path) + == Path(other.original_python.python_path) + ) + + @classmethod + def from_runtime(cls): + if cls.current: + return cls.current + + return cls.from_path(sys.prefix) + + @classmethod + def from_path(cls, prefix): + metadata_path = os.path.join(prefix, METADATA_FILENAME) + out_of_date_exception = MozSiteMetadataOutOfDateError( + f'The virtualenv at "{prefix}" is out-of-date.' + ) + try: + with open(metadata_path, "r") as file: + raw = json.load(file) + + if not raw.get("is_finalized", False): + raise out_of_date_exception + + return cls( + raw["hex_version"], + raw["virtualenv_name"], + SitePackagesSource[raw["mach_site_packages_source"]], + ExternalPythonSite(raw["original_python_executable"]), + metadata_path, + ) + except FileNotFoundError: + return None + except KeyError: + raise out_of_date_exception + + @contextmanager + def update_current_site(self, executable): + """Updates necessary global state when a site is activated + + Due to needing to fetch some state before the actual activation happens, this + is represented as a context manager and should be used as follows: + + with metadata.update_current_site(executable): + # Perform the actual implementation of changing the site, whether that is + # by exec-ing "activate_this.py" in a virtualenv, modifying the sys.path + # directly, or some other means + ... + """ + + try: + import pkg_resources + except ModuleNotFoundError: + pkg_resources = None + + yield + MozSiteMetadata.current = self + + sys.executable = executable + + if pkg_resources: + # Rebuild the working_set based on the new sys.path. + pkg_resources._initialize_master_working_set() + + +class MachSiteManager: + """Represents the activate-able "import scope" Mach needs + + Whether running independently, using the system packages, or automatically managing + dependencies with "pip install", this class provides an easy handle to verify + that the "site" is up-to-date (whether than means that system packages don't + collide with vendored packages, or that the on-disk virtualenv needs rebuilding). + + Note that, this is a *virtual* site: an on-disk Python virtualenv + is only created if there will be "pip installs" into the Mach site. + """ + + def __init__( + self, + topsrcdir: str, + virtualenv_root: Optional[str], + requirements: MachEnvRequirements, + original_python: "ExternalPythonSite", + site_packages_source: SitePackagesSource, + ): + """ + Args: + topsrcdir: The path to the Firefox repo + virtualenv_root: The path to the the associated Mach virtualenv, + if any + requirements: The requirements associated with the Mach site, parsed from + the file at python/sites/mach.txt + original_python: The external Python site that was used to invoke Mach. + If Mach invocations are nested, then "original_python" refers to + Python site that was used to start Mach first. + Usually the system Python, such as /usr/bin/python3. + site_packages_source: Where the Mach site will import its pip-installed + dependencies from + """ + self._topsrcdir = topsrcdir + self._site_packages_source = site_packages_source + self._requirements = requirements + self._virtualenv_root = virtualenv_root + self._metadata = MozSiteMetadata( + sys.hexversion, + "mach", + site_packages_source, + original_python, + self._virtualenv_root, + ) + + @classmethod + def from_environment(cls, topsrcdir: str, get_state_dir: Callable[[], str]): + """ + Args: + topsrcdir: The path to the Firefox repo + get_state_dir: A function that resolves the path to the checkout-scoped + state_dir, generally ~/.mozbuild/srcdirs/<checkout-based-dir>/ + """ + + requirements = resolve_requirements(topsrcdir, "mach") + # Mach needs to operate in environments in which no pip packages are installed + # yet, and the system isn't guaranteed to have the packages we need. For example, + # "./mach bootstrap" can't have any dependencies. + # So, all external dependencies of Mach's must be optional. + assert ( + not requirements.pypi_requirements + ), "Mach pip package requirements must be optional." + + # external_python is the Python interpreter that invoked Mach for this process. + external_python = ExternalPythonSite(sys.executable) + + # original_python is the first Python interpreter that invoked the top-level + # Mach process. This is different from "external_python" when there's nested + # Mach invocations. + active_metadata = MozSiteMetadata.from_runtime() + if active_metadata: + original_python = active_metadata.original_python + else: + original_python = external_python + + source = SitePackagesSource.for_mach() + virtualenv_root = ( + _mach_virtualenv_root(get_state_dir()) + if source == SitePackagesSource.VENV + else None + ) + return cls( + topsrcdir, + virtualenv_root, + requirements, + original_python, + source, + ) + + def _up_to_date(self): + if self._site_packages_source == SitePackagesSource.NONE: + return SiteUpToDateResult(True) + elif self._site_packages_source == SitePackagesSource.SYSTEM: + _assert_pip_check(self._sys_path(), "mach", self._requirements) + return SiteUpToDateResult(True) + elif self._site_packages_source == SitePackagesSource.VENV: + environment = self._virtualenv() + return _is_venv_up_to_date( + environment, + self._pthfile_lines(environment), + self._requirements, + self._metadata, + ) + + def ensure(self, *, force=False): + result = self._up_to_date() + if force or not result.is_up_to_date: + if Path(sys.prefix) == Path(self._metadata.prefix): + # If the Mach virtualenv is already activated, then the changes caused + # by rebuilding the virtualenv won't take effect until the next time + # Mach is used, which can lead to confusing one-off errors. + # Instead, request that the user resolve the out-of-date situation, + # *then* come back and run the intended command. + raise VirtualenvOutOfDateException(result.reason) + self._build() + + def attempt_populate_optional_packages(self): + if self._site_packages_source != SitePackagesSource.VENV: + pass + + self._virtualenv().install_optional_packages( + self._requirements.pypi_optional_requirements + ) + + def activate(self): + assert not MozSiteMetadata.current + + self.ensure() + with self._metadata.update_current_site( + self._virtualenv().python_path + if self._site_packages_source == SitePackagesSource.VENV + else sys.executable, + ): + # Reset the sys.path to insulate ourselves from the environment. + # This should be safe to do, since activation of the Mach site happens so + # early in the Mach lifecycle that no packages should have been imported + # from external sources yet. + sys.path = self._sys_path() + if self._site_packages_source == SitePackagesSource.VENV: + # Activate the Mach virtualenv in the current Python context. This + # automatically adds the virtualenv's "site-packages" to our scope, in + # addition to our first-party/vendored modules since they're specified + # in the "mach.pth" file. + activate_virtualenv(self._virtualenv()) + + def _build(self): + if self._site_packages_source != SitePackagesSource.VENV: + # The Mach virtualenv doesn't have a physical virtualenv on-disk if it won't + # be "pip install"-ing. So, there's no build work to do. + return + + environment = self._virtualenv() + _create_venv_with_pthfile( + environment, + self._pthfile_lines(environment), + True, + self._requirements, + self._metadata, + ) + + def _sys_path(self): + if self._site_packages_source == SitePackagesSource.SYSTEM: + stdlib_paths, system_site_paths = self._metadata.original_python.sys_path() + return [ + *stdlib_paths, + *self._requirements.pths_as_absolute(self._topsrcdir), + *system_site_paths, + ] + elif self._site_packages_source == SitePackagesSource.NONE: + stdlib_paths = self._metadata.original_python.sys_path_stdlib() + return [ + *stdlib_paths, + *self._requirements.pths_as_absolute(self._topsrcdir), + ] + elif self._site_packages_source == SitePackagesSource.VENV: + stdlib_paths = self._metadata.original_python.sys_path_stdlib() + return [ + *stdlib_paths, + # self._requirements will be added as part of the virtualenv activation. + ] + + def _pthfile_lines(self, environment): + return [ + # Prioritize vendored and first-party modules first. + *self._requirements.pths_as_absolute(self._topsrcdir), + # Then, include the virtualenv's site-packages. + *_deprioritize_venv_packages( + environment, self._site_packages_source == SitePackagesSource.VENV + ), + ] + + def _virtualenv(self): + assert self._site_packages_source == SitePackagesSource.VENV + return PythonVirtualenv(self._metadata.prefix) + + +class CommandSiteManager: + """Activate sites and ad-hoc-install pip packages + + Provides tools to ensure that a command's scope will have expected, compatible + packages. Manages prioritization of the import scope, and ensures consistency + regardless of how a virtualenv is used (whether via in-process activation, or when + used standalone to invoke a script). + + A few notes: + + * The command environment always inherits Mach's import scope. This is + because "unloading" packages in Python is error-prone, so in-process activations + will always carry Mach's dependencies along with it. Accordingly, compatibility + between each command environment and the Mach environment must be maintained + + * Unlike the Mach environment, command environments *always* have an associated + physical virtualenv on-disk. This is because some commands invoke child Python + processes, and that child process should have the same import scope. + + """ + + def __init__( + self, + topsrcdir: str, + mach_virtualenv_root: Optional[str], + virtualenv_root: str, + site_name: str, + active_metadata: MozSiteMetadata, + populate_virtualenv: bool, + requirements: MachEnvRequirements, + ): + """ + Args: + topsrcdir: The path to the Firefox repo + mach_virtualenv_root: The path to the Mach virtualenv, if any + virtualenv_root: The path to the virtualenv associated with this site + site_name: The name of this site, such as "build" + active_metadata: The currently-active moz-managed site + populate_virtualenv: True if packages should be installed to the on-disk + virtualenv with "pip". False if the virtualenv should only include + sys.path modifications, and all 3rd-party packages should be imported from + Mach's site packages source. + requirements: The requirements associated with this site, parsed from + the file at python/sites/<site_name>.txt + """ + self._topsrcdir = topsrcdir + self._mach_virtualenv_root = mach_virtualenv_root + self.virtualenv_root = virtualenv_root + self._site_name = site_name + self._virtualenv = PythonVirtualenv(self.virtualenv_root) + self.python_path = self._virtualenv.python_path + self.bin_path = self._virtualenv.bin_path + self._populate_virtualenv = populate_virtualenv + self._mach_site_packages_source = active_metadata.mach_site_packages_source + self._requirements = requirements + self._metadata = MozSiteMetadata( + sys.hexversion, + site_name, + active_metadata.mach_site_packages_source, + active_metadata.original_python, + virtualenv_root, + ) + + @classmethod + def from_environment( + cls, + topsrcdir: str, + get_state_dir: Callable[[], Optional[str]], + site_name: str, + command_virtualenvs_dir: str, + ): + """ + Args: + topsrcdir: The path to the Firefox repo + get_state_dir: A function that resolves the path to the checkout-scoped + state_dir, generally ~/.mozbuild/srcdirs/<checkout-based-dir>/ + site_name: The name of this site, such as "build" + command_virtualenvs_dir: The location under which this site's virtualenv + should be created + """ + active_metadata = MozSiteMetadata.from_runtime() + assert ( + active_metadata + ), "A Mach-managed site must be active before doing work with command sites" + + mach_site_packages_source = active_metadata.mach_site_packages_source + pip_restricted_site = site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS + if ( + not pip_restricted_site + and mach_site_packages_source == SitePackagesSource.SYSTEM + ): + # Sites that aren't pip-network-install-restricted are likely going to be + # incompatible with the system. Besides, this use case shouldn't exist, since + # using the system packages is supposed to only be needed to lower risk of + # important processes like building Firefox. + raise Exception( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any ' + f"sites other than {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS}. The " + f'current attempted site is "{site_name}".' + ) + + mach_virtualenv_root = ( + _mach_virtualenv_root(get_state_dir()) + if mach_site_packages_source == SitePackagesSource.VENV + else None + ) + populate_virtualenv = ( + mach_site_packages_source == SitePackagesSource.VENV + or not pip_restricted_site + ) + return cls( + topsrcdir, + mach_virtualenv_root, + os.path.join(command_virtualenvs_dir, site_name), + site_name, + active_metadata, + populate_virtualenv, + resolve_requirements(topsrcdir, site_name), + ) + + def ensure(self): + """Ensure that this virtualenv is built, up-to-date, and ready for use + If using a virtualenv Python binary directly, it's useful to call this function + first to ensure that the virtualenv doesn't have obsolete references or packages. + """ + result = self._up_to_date() + if not result.is_up_to_date: + print(f"Site not up-to-date reason: {result.reason}") + active_site = MozSiteMetadata.from_runtime() + if active_site.site_name == self._site_name: + print(result.reason, file=sys.stderr) + raise Exception( + f'The "{self._site_name}" site is out-of-date, even though it has ' + f"already been activated. Was it modified while this Mach process " + f"was running?" + ) + + _create_venv_with_pthfile( + self._virtualenv, + self._pthfile_lines(), + self._populate_virtualenv, + self._requirements, + self._metadata, + ) + + def activate(self): + """Activate this site in the current Python context. + + If you run a random Python script and wish to "activate" the + site, you can simply instantiate an instance of this class + and call .activate() to make the virtualenv active. + """ + + active_site = MozSiteMetadata.from_runtime() + site_is_already_active = active_site.site_name == self._site_name + if ( + active_site.site_name not in ("mach", "common") + and not site_is_already_active + ): + raise Exception( + f'Activating from one command site ("{active_site.site_name}") to ' + f'another ("{self._site_name}") is not allowed, because they may ' + "be incompatible." + ) + + self.ensure() + + if site_is_already_active: + return + + with self._metadata.update_current_site(self._virtualenv.python_path): + activate_virtualenv(self._virtualenv) + + def install_pip_package(self, package): + """Install a package via pip. + + The supplied package is specified using a pip requirement specifier. + e.g. 'foo' or 'foo==1.0'. + + If the package is already installed, this is a no-op. + """ + if Path(sys.prefix) == Path(self.virtualenv_root): + # If we're already running in this interpreter, we can optimize in + # the case that the package requirement is already satisfied. + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + if req.satisfied_by is not None: + return + + self._virtualenv.pip_install_with_constraints([package]) + + def install_pip_requirements(self, path, require_hashes=True, quiet=False): + """Install a pip requirements.txt file. + + The supplied path is a text file containing pip requirement + specifiers. + + If require_hashes is True, each specifier must contain the + expected hash of the downloaded package. See: + https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode + """ + + if not os.path.isabs(path): + path = os.path.join(self._topsrcdir, path) + + args = ["--requirement", path] + + if require_hashes: + args.append("--require-hashes") + + install_result = self._virtualenv.pip_install( + args, + check=not quiet, + stdout=subprocess.PIPE if quiet else None, + ) + if install_result.returncode: + print(install_result.stdout) + raise InstallPipRequirementsException( + f'Failed to install "{path}" into the "{self._site_name}" site.' + ) + + check_result = subprocess.run( + [self.python_path, "-m", "pip", "check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + if not check_result.returncode: + return + + """ + Some commands may use the "setup.py" script of first-party modules. This causes + a "*.egg-info" dir to be created for that module (which pip can then detect as + a package). Since we add all first-party module directories to the .pthfile for + the "mach" venv, these first-party modules are then detected by all venvs after + they are created. The problem is that these .egg-info directories can become + stale (since if the first-party module is updated it's not guaranteed that the + command that runs the "setup.py" was ran afterwards). This can cause + incompatibilities with the pip check (since the dependencies can change between + different versions). + + These .egg-info dirs are in our VCS ignore lists (eg: ".hgignore") because they + are necessary to run some commands, so we don't want to always purge them, and we + also don't want to accidentally commit them. Given this, we can leverage our VCS + to find all the current first-party .egg-info dirs. + + If we're in the case where 'pip check' fails, then we can try purging the + first-party .egg-info dirs, then run the 'pip check' again afterwards. If it's + still failing, then we know the .egg-info dirs weren't the problem. If that's + the case we can just raise the error encountered, which is the same as before. + """ + + def _delete_ignored_egg_info_dirs(): + from pathlib import Path + + from mozversioncontrol import ( + MissingConfigureInfo, + MissingVCSInfo, + get_repository_from_env, + ) + + try: + with get_repository_from_env() as repo: + ignored_file_finder = repo.get_ignored_files_finder().find( + "**/*.egg-info" + ) + + unique_egg_info_dirs = { + Path(found[0]).parent for found in ignored_file_finder + } + + for egg_info_dir in unique_egg_info_dirs: + shutil.rmtree(egg_info_dir) + + except (MissingVCSInfo, MissingConfigureInfo): + pass + + _delete_ignored_egg_info_dirs() + + check_result = subprocess.run( + [self.python_path, "-m", "pip", "check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + if check_result.returncode: + if quiet: + # If "quiet" was specified, then the "pip install" output wasn't printed + # earlier, and was buffered instead. Print that buffer so that debugging + # the "pip check" failure is easier. + print(install_result.stdout) + + subprocess.check_call( + [self.python_path, "-m", "pip", "list", "-v"], stdout=sys.stderr + ) + print(check_result.stdout, file=sys.stderr) + raise InstallPipRequirementsException( + f'As part of validation after installing "{path}" into the ' + f'"{self._site_name}" site, the site appears to contain installed ' + "packages that are incompatible with each other." + ) + + def _pthfile_lines(self): + """Generate the prioritized import scope to encode in the venv's pthfile + + The import priority looks like this: + 1. Mach's vendored/first-party modules + 2. Mach's site-package source (the Mach virtualenv, the system Python, or neither) + 3. The command's vendored/first-party modules + 4. The command's site-package source (either the virtualenv or the system Python, + if it's not already added) + + Note that, when using the system Python, it may either be prioritized before or + after the command's vendored/first-party modules. This is a symptom of us + attempting to avoid conflicting with the system packages. + + For example, there's at least one job in CI that operates with an ancient + environment with a bunch of old packages, many of whom conflict with our vendored + packages. However, the specific command that we're running for the job doesn't + need any of the system's packages, so we're safe to insulate ourselves. + + Mach doesn't know the command being run when it's preparing its import scope, + so it has to be defensive. Therefore: + 1. If Mach needs a system package: system packages are higher priority. + 2. If Mach doesn't need a system package, but the current command does: system + packages are still be in the list, albeit at a lower priority. + """ + + # Prioritize Mach's vendored and first-party modules first. + lines = resolve_requirements(self._topsrcdir, "mach").pths_as_absolute( + self._topsrcdir + ) + mach_site_packages_source = self._mach_site_packages_source + if mach_site_packages_source == SitePackagesSource.SYSTEM: + # When Mach is using the system environment, add it next. + _, system_site_paths = self._metadata.original_python.sys_path() + lines.extend(system_site_paths) + elif mach_site_packages_source == SitePackagesSource.VENV: + # When Mach is using its on-disk virtualenv, add its site-packages directory. + assert self._mach_virtualenv_root + lines.extend( + PythonVirtualenv(self._mach_virtualenv_root).site_packages_dirs() + ) + + # Add this command's vendored and first-party modules. + lines.extend(self._requirements.pths_as_absolute(self._topsrcdir)) + # Finally, ensure that pip-installed packages are the lowest-priority + # source to import from. + lines.extend( + _deprioritize_venv_packages(self._virtualenv, self._populate_virtualenv) + ) + + # Note that an on-disk virtualenv is always created for commands, even if they + # are using the system as their site-packages source. This is to support use + # cases where a fresh Python process must be created, but it also must have + # access to <site>'s 1st- and 3rd-party packages. + return lines + + def _up_to_date(self): + pthfile_lines = self._pthfile_lines() + if self._mach_site_packages_source == SitePackagesSource.SYSTEM: + _assert_pip_check( + pthfile_lines, + self._site_name, + self._requirements if not self._populate_virtualenv else None, + ) + + return _is_venv_up_to_date( + self._virtualenv, + pthfile_lines, + self._requirements, + self._metadata, + ) + + +class PythonVirtualenv: + """Calculates paths of interest for general python virtual environments""" + + def __init__(self, prefix): + if _is_windows: + self.bin_path = os.path.join(prefix, "Scripts") + self.python_path = os.path.join(self.bin_path, "python.exe") + else: + self.bin_path = os.path.join(prefix, "bin") + self.python_path = os.path.join(self.bin_path, "python") + self.prefix = os.path.realpath(prefix) + + @functools.lru_cache(maxsize=None) + def resolve_sysconfig_packages_path(self, sysconfig_path): + # macOS uses a different default sysconfig scheme based on whether it's using the + # system Python or running in a virtualenv. + # Manually define the scheme (following the implementation in + # "sysconfig._get_default_scheme()") so that we're always following the + # code path for a virtualenv directory structure. + if os.name == "posix": + scheme = "posix_prefix" + else: + scheme = os.name + + sysconfig_paths = sysconfig.get_paths(scheme) + data_path = Path(sysconfig_paths["data"]) + path = Path(sysconfig_paths[sysconfig_path]) + relative_path = path.relative_to(data_path) + + # Path to virtualenv's "site-packages" directory for provided sysconfig path + return os.path.normpath(os.path.normcase(Path(self.prefix) / relative_path)) + + def site_packages_dirs(self): + dirs = [] + if sys.platform.startswith("win"): + dirs.append(os.path.normpath(os.path.normcase(self.prefix))) + purelib = self.resolve_sysconfig_packages_path("purelib") + platlib = self.resolve_sysconfig_packages_path("platlib") + + dirs.append(purelib) + if platlib != purelib: + dirs.append(platlib) + + return dirs + + def pip_install_with_constraints(self, pip_args): + """Create a pip constraints file or existing packages + + When pip installing an incompatible package, pip will follow through with + the install but raise a warning afterwards. + + To defend our environment from breakage, we run "pip install" but add all + existing packages to a "constraints file". This ensures that conflicts are + raised as errors up-front, and the virtual environment doesn't have conflicting + packages installed. + + Note: pip_args is expected to contain either the requested package or + requirements file. + """ + existing_packages = self._resolve_installed_packages() + + with tempfile.TemporaryDirectory() as tempdir: + constraints_path = os.path.join(tempdir, "site-constraints.txt") + with open(constraints_path, "w") as file: + file.write( + "\n".join( + [ + f"{name}=={version}" + for name, version in existing_packages.items() + ] + ) + ) + + return self.pip_install(["--constraint", constraints_path] + pip_args) + + def pip_install(self, pip_install_args, **kwargs): + # setuptools will use the architecture of the running Python instance when + # building packages. However, it's possible for the Xcode Python to be a universal + # binary (x86_64 and arm64) without the associated macOS SDK supporting arm64, + # thereby causing a build failure. To avoid this, we explicitly influence the + # build to only target a single architecture - our current architecture. + kwargs.setdefault("env", os.environ.copy()).setdefault( + "ARCHFLAGS", "-arch {}".format(platform.machine()) + ) + kwargs.setdefault("check", True) + kwargs.setdefault("stderr", subprocess.STDOUT) + kwargs.setdefault("universal_newlines", True) + + # It's tempting to call pip natively via pip.main(). However, + # the current Python interpreter may not be the virtualenv python. + # This will confuse pip and cause the package to attempt to install + # against the executing interpreter. By creating a new process, we + # force the virtualenv's interpreter to be used and all is well. + # It /might/ be possible to cheat and set sys.executable to + # self.python_path. However, this seems more risk than it's worth. + return subprocess.run( + [self.python_path, "-m", "pip", "install"] + pip_install_args, + **kwargs, + ) + + def install_optional_packages(self, optional_requirements): + for requirement in optional_requirements: + try: + self.pip_install_with_constraints([str(requirement.requirement)]) + except subprocess.CalledProcessError: + print( + f"Could not install {requirement.requirement.name}, so " + f"{requirement.repercussion}. Continuing." + ) + + def _resolve_installed_packages(self): + return _resolve_installed_packages(self.python_path) + + +class RequirementsValidationResult: + def __init__(self): + self._package_discrepancies = [] + self.has_all_packages = True + self.provides_any_package = False + + def add_discrepancy(self, requirement, found): + self._package_discrepancies.append((requirement, found)) + self.has_all_packages = False + + def report(self): + lines = [] + for requirement, found in self._package_discrepancies: + if found: + error = f'Installed with unexpected version "{found}"' + else: + error = "Not installed" + lines.append(f"{requirement}: {error}") + return "\n".join(lines) + + @classmethod + def from_packages(cls, packages, requirements): + result = cls() + for pkg in requirements.pypi_requirements: + installed_version = packages.get(pkg.requirement.name) + if not installed_version or not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + elif installed_version: + result.provides_any_package = True + + for pkg in requirements.pypi_optional_requirements: + installed_version = packages.get(pkg.requirement.name) + if installed_version and not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + elif installed_version: + result.provides_any_package = True + + return result + + +class ExternalPythonSite: + """Represents the Python site that is executing Mach + + The external Python site could be a virtualenv (created by venv or virtualenv) or + the system Python itself, so we can't make any significant assumptions on its + structure. + """ + + def __init__(self, python_executable): + self._prefix = os.path.dirname(os.path.dirname(python_executable)) + self.python_path = python_executable + + @functools.lru_cache(maxsize=None) + def sys_path(self): + """Return lists of sys.path entries: one for standard library, one for the site + + These two lists are calculated at the same time so that we can interpret them + in a single Python subprocess, as running a whole Python instance is + very expensive in the context of Mach initialization. + """ + env = { + k: v + for k, v in os.environ.items() + # Don't include items injected by IDEs into the system path. + if k not in ("PYTHONPATH", "PYDEVD_LOAD_VALUES_ASYNC") + } + stdlib = subprocess.Popen( + [ + self.python_path, + # Don't "import site" right away, so we can split the standard library + # paths from the site paths. + "-S", + "-c", + "import sys; from collections import OrderedDict; " + # Skip the first item in the sys.path, as it's the working directory + # of the invoked script (so, in this case, ""). + # Use list(OrderectDict...) to de-dupe items, such as when using + # pyenv on Linux. + "print(list(OrderedDict.fromkeys(sys.path[1:])))", + ], + universal_newlines=True, + env=env, + stdout=subprocess.PIPE, + ) + system = subprocess.Popen( + [ + self.python_path, + "-c", + "import os; import sys; import site; " + "packages = site.getsitepackages(); " + # Only add the "user site packages" if not in a virtualenv (which is + # identified by the prefix == base_prefix check + "packages.insert(0, site.getusersitepackages()) if " + " sys.prefix == sys.base_prefix else None; " + # When a Python instance launches, it only adds each + # "site.getsitepackages()" entry if it exists on the file system. + # Replicate that behaviour to get a more accurate list of system paths. + "packages = [p for p in packages if os.path.exists(p)]; " + "print(packages)", + ], + universal_newlines=True, + env=env, + stdout=subprocess.PIPE, + ) + # Run python processes in parallel - they take roughly the same time, so this + # cuts this functions run time in half. + stdlib_out, _ = stdlib.communicate() + system_out, _ = system.communicate() + assert stdlib.returncode == 0 + assert system.returncode == 0 + stdlib = ast.literal_eval(stdlib_out) + system = ast.literal_eval(system_out) + # On Windows, some paths are both part of the default sys.path *and* are included + # in the "site packages" list. Keep the "stdlib" one, and remove the dupe from + # the "system packages" list. + system = [path for path in system if path not in stdlib] + return stdlib, system + + def sys_path_stdlib(self): + """Return list of default sys.path entries for the standard library""" + stdlib, _ = self.sys_path() + return stdlib + + +@functools.lru_cache(maxsize=None) +def resolve_requirements(topsrcdir, site_name): + manifest_path = os.path.join(topsrcdir, "python", "sites", f"{site_name}.txt") + if not os.path.exists(manifest_path): + raise Exception( + f'The current command is using the "{site_name}" ' + "site. However, that site is missing its associated " + f'requirements definition file at "{manifest_path}".' + ) + + thunderbird_dir = os.path.join(topsrcdir, "comm") + is_thunderbird = os.path.exists(thunderbird_dir) and bool( + os.listdir(thunderbird_dir) + ) + try: + return MachEnvRequirements.from_requirements_definition( + topsrcdir, + is_thunderbird, + site_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS, + manifest_path, + ) + except UnexpectedFlexibleRequirementException as e: + raise Exception( + f'The "{site_name}" site does not have all pypi packages pinned ' + f'in the format "package==version" (found "{e.raw_requirement}").\n' + f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} sites are " + "allowed to have unpinned packages." + ) + + +def _resolve_installed_packages(python_executable): + pip_json = subprocess.check_output( + [ + python_executable, + "-m", + "pip", + "list", + "--format", + "json", + "--disable-pip-version-check", + ], + universal_newlines=True, + ) + + installed_packages = json.loads(pip_json) + return {package["name"]: package["version"] for package in installed_packages} + + +def _ensure_python_exe(python_exe_root: Path): + """On some machines in CI venv does not behave consistently. Sometimes + only a "python3" executable is created, but we expect "python". Since + they are functionally identical, we can just copy "python3" to "python" + (and vice-versa) to solve the problem. + """ + python3_exe_path = python_exe_root / "python3" + python_exe_path = python_exe_root / "python" + + if _is_windows: + python3_exe_path = python3_exe_path.with_suffix(".exe") + python_exe_path = python_exe_path.with_suffix(".exe") + + if python3_exe_path.exists() and not python_exe_path.exists(): + shutil.copy(str(python3_exe_path), str(python_exe_path)) + + if python_exe_path.exists() and not python3_exe_path.exists(): + shutil.copy(str(python_exe_path), str(python3_exe_path)) + + if not python_exe_path.exists() and not python3_exe_path.exists(): + raise Exception( + f'Neither a "{python_exe_path.name}" or "{python3_exe_path.name}" ' + f"were found. This means something unexpected happened during the " + f"virtual environment creation and we cannot proceed." + ) + + +def _ensure_pyvenv_cfg(venv_root: Path): + # We can work around a bug on some versions of Python 3.6 on + # Windows by copying the 'pyvenv.cfg' of the current venv + # to the new venv. This will make the new venv reference + # the original Python install instead of the current venv, + # which resolves the issue. There shouldn't be any harm in + # always doing this, but we'll play it safe and restrict it + # to Windows Python 3.6 anyway. + if _is_windows and sys.version_info[:2] == (3, 6): + this_venv = Path(sys.executable).parent.parent + this_venv_config = this_venv / "pyvenv.cfg" + if this_venv_config.exists(): + new_venv_config = Path(venv_root) / "pyvenv.cfg" + shutil.copyfile(str(this_venv_config), str(new_venv_config)) + + +def _assert_pip_check(pthfile_lines, virtualenv_name, requirements): + """Check if the provided pthfile lines have a package incompatibility + + If there's an incompatibility, raise an exception and allow it to bubble up since + it will require user intervention to resolve. + + If requirements aren't provided (such as when Mach is using SYSTEM, but the command + site is using VENV), then skip the "pthfile satisfies requirements" step. + """ + if os.environ.get( + f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE", None + ): + # Don't re-assert compatibility against the system python within Mach subshells. + return + + print( + 'Running "pip check" to verify compatibility between the system Python and the ' + f'"{virtualenv_name}" site.' + ) + + with tempfile.TemporaryDirectory() as check_env_path: + # Pip detects packages on the "sys.path" that have a ".dist-info" or + # a ".egg-info" directory. The majority of our Python dependencies are + # vendored as extracted wheels or sdists, so they are automatically picked up. + # This gives us sufficient confidence to do a `pip check` with both vendored + # packages + system packages in scope, and trust the results. + # Note: rather than just running the system pip with a modified "sys.path", + # we create a new virtualenv that has our pinned pip version, so that + # we get consistent results (there's been lots of pip resolver behaviour + # changes recently). + process = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", check_env_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + ) + + _ensure_pyvenv_cfg(Path(check_env_path)) + + if process.returncode != 0: + if "No module named venv" in process.stderr: + raise VenvModuleNotFoundException() + else: + raise subprocess.CalledProcessError( + process.returncode, + process.args, + output=process.stdout, + stderr=process.stderr, + ) + + if process.stdout: + print(process.stdout) + + check_env = PythonVirtualenv(check_env_path) + _ensure_python_exe(Path(check_env.python_path).parent) + + with open( + os.path.join( + os.path.join(check_env.resolve_sysconfig_packages_path("platlib")), + PTH_FILENAME, + ), + "w", + ) as f: + f.write("\n".join(pthfile_lines)) + + pip = [check_env.python_path, "-m", "pip"] + if requirements: + packages = _resolve_installed_packages(check_env.python_path) + validation_result = RequirementsValidationResult.from_packages( + packages, requirements + ) + if not validation_result.has_all_packages: + subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) + print(validation_result.report(), file=sys.stderr) + raise Exception( + f'The "{virtualenv_name}" site is not compatible with the installed ' + "system Python packages." + ) + + check_result = subprocess.run( + pip + ["check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if check_result.returncode: + subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) + print(check_result.stdout, file=sys.stderr) + raise Exception( + 'According to "pip check", the current Python ' + "environment has package-compatibility issues." + ) + + os.environ[ + f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE" + ] = "1" + + +def _deprioritize_venv_packages(virtualenv, populate_virtualenv): + # Virtualenvs implicitly add some "site packages" to the sys.path upon being + # activated. However, Mach generally wants to prioritize the existing sys.path + # (such as vendored packages) over packages installed to virtualenvs. + # So, this function moves the virtualenv's site-packages to the bottom of the sys.path + # at activation-time. + + return [ + line + for site_packages_dir in virtualenv.site_packages_dirs() + # repr(...) is needed to ensure Windows path backslashes aren't mistaken for + # escape sequences. + # Additionally, when removing the existing "site-packages" folder's entry, we have + # to do it in a case-insensitive way because, on Windows: + # * Python adds it as <venv>/lib/site-packages + # * While sysconfig tells us it's <venv>/Lib/site-packages + # * (note: on-disk, it's capitalized, so sysconfig is slightly more accurate). + for line in filter( + None, + ( + "import sys; sys.path = [p for p in sys.path if " + f"p.lower() != {repr(site_packages_dir)}.lower()]", + f"import sys; sys.path.append({repr(site_packages_dir)})" + if populate_virtualenv + else None, + ), + ) + ] + + +def _create_venv_with_pthfile( + target_venv, + pthfile_lines, + populate_with_pip, + requirements, + metadata, +): + virtualenv_root = target_venv.prefix + if os.path.exists(virtualenv_root): + shutil.rmtree(virtualenv_root) + + os.makedirs(virtualenv_root) + metadata.write(is_finalized=False) + + process = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", virtualenv_root], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + ) + + _ensure_pyvenv_cfg(Path(virtualenv_root)) + + if process.returncode != 0: + if "No module named venv" in process.stderr: + raise VenvModuleNotFoundException() + else: + raise subprocess.CalledProcessError( + process.returncode, + process.args, + output=process.stdout, + stderr=process.stderr, + ) + + if process.stdout: + print(process.stdout) + + _ensure_python_exe(Path(target_venv.python_path).parent) + + platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib") + pthfile_contents = "\n".join(pthfile_lines) + with open(os.path.join(platlib_site_packages_dir, PTH_FILENAME), "w") as f: + f.write(pthfile_contents) + + if populate_with_pip: + for requirement in requirements.pypi_requirements: + target_venv.pip_install([str(requirement.requirement)]) + target_venv.install_optional_packages(requirements.pypi_optional_requirements) + + metadata.write(is_finalized=True) + + +def _is_venv_up_to_date( + target_venv, + expected_pthfile_lines, + requirements, + expected_metadata, +): + if not os.path.exists(target_venv.prefix): + return SiteUpToDateResult(False, f'"{target_venv.prefix}" does not exist') + + # Modifications to any of the requirements manifest files mean the virtualenv should + # be rebuilt: + metadata_mtime = os.path.getmtime( + os.path.join(target_venv.prefix, METADATA_FILENAME) + ) + for dep_file in requirements.requirements_paths: + if os.path.getmtime(dep_file) > metadata_mtime: + return SiteUpToDateResult( + False, f'"{dep_file}" has changed since the virtualenv was created' + ) + + try: + existing_metadata = MozSiteMetadata.from_path(target_venv.prefix) + except MozSiteMetadataOutOfDateError as e: + # The metadata is missing required fields, so must be out-of-date. + return SiteUpToDateResult(False, str(e)) + + if existing_metadata != expected_metadata: + # The metadata doesn't exist or some fields have different values. + return SiteUpToDateResult( + False, + f"The existing metadata on-disk ({vars(existing_metadata)}) does not match " + f"the expected metadata ({vars(expected_metadata)}", + ) + + platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib") + pthfile_path = os.path.join(platlib_site_packages_dir, PTH_FILENAME) + try: + with open(pthfile_path) as file: + current_pthfile_contents = file.read().strip() + except FileNotFoundError: + return SiteUpToDateResult(False, f'No pthfile found at "{pthfile_path}"') + + expected_pthfile_contents = "\n".join(expected_pthfile_lines) + if current_pthfile_contents != expected_pthfile_contents: + return SiteUpToDateResult( + False, + f'The pthfile at "{pthfile_path}" does not match the expected value.\n' + f"# --- on-disk pthfile: ---\n" + f"{current_pthfile_contents}\n" + f"# --- expected pthfile contents ---\n" + f"{expected_pthfile_contents}\n" + f"# ---", + ) + + return SiteUpToDateResult(True) + + +def activate_virtualenv(virtualenv: PythonVirtualenv): + os.environ["PATH"] = os.pathsep.join( + [virtualenv.bin_path] + os.environ.get("PATH", "").split(os.pathsep) + ) + os.environ["VIRTUAL_ENV"] = virtualenv.prefix + + for path in virtualenv.site_packages_dirs(): + site.addsitedir(os.path.realpath(path)) + + sys.prefix = virtualenv.prefix + + +def _mach_virtualenv_root(checkout_scoped_state_dir): + workspace = os.environ.get("WORKSPACE") + if os.environ.get("MOZ_AUTOMATION") and workspace: + # In CI, put Mach virtualenv in the $WORKSPACE dir, which should be cleaned + # between jobs. + return os.path.join(workspace, "mach_virtualenv") + return os.path.join(checkout_scoped_state_dir, "_virtualenvs", "mach") diff --git a/python/mach/mach/telemetry.py b/python/mach/mach/telemetry.py new file mode 100644 index 0000000000..233556550d --- /dev/null +++ b/python/mach/mach/telemetry.py @@ -0,0 +1,305 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import requests +import six.moves.urllib.parse as urllib_parse +from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject +from mozbuild.settings import TelemetrySettings +from mozbuild.telemetry import filter_args +from mozversioncontrol import InvalidRepoPath, get_repository_object +from six.moves import configparser, input + +from mach.config import ConfigSettings +from mach.site import MozSiteMetadata +from mach.telemetry_interface import GleanTelemetry, NoopTelemetry +from mach.util import get_state_dir + +MACH_METRICS_PATH = (Path(__file__) / ".." / ".." / "metrics.yaml").resolve() + + +def create_telemetry_from_environment(settings): + """Creates and a Telemetry instance based on system details. + + If telemetry isn't enabled, the current interpreter isn't Python 3, or Glean + can't be imported, then a "mock" telemetry instance is returned that doesn't + set or record any data. This allows consumers to optimistically set telemetry + data without needing to specifically handle the case where the current system + doesn't support it. + """ + + active_metadata = MozSiteMetadata.from_runtime() + is_mach_virtualenv = active_metadata and active_metadata.site_name == "mach" + + if not ( + is_applicable_telemetry_environment() + # Glean is not compatible with Python 2 + and sys.version_info >= (3, 0) + # If not using the mach virtualenv (e.g.: bootstrap uses native python) + # then we can't guarantee that the glean package that we import is a + # compatible version. Therefore, don't use glean. + and is_mach_virtualenv + ): + return NoopTelemetry(False) + + is_enabled = is_telemetry_enabled(settings) + + try: + from glean import Glean + except ImportError: + return NoopTelemetry(is_enabled) + + from pathlib import Path + + Glean.initialize( + "mozilla.mach", + "Unknown", + is_enabled, + data_dir=Path(get_state_dir()) / "glean", + ) + return GleanTelemetry() + + +def report_invocation_metrics(telemetry, command): + metrics = telemetry.metrics(MACH_METRICS_PATH) + metrics.mach.command.set(command) + metrics.mach.duration.start() + + try: + instance = MozbuildObject.from_environment() + except BuildEnvironmentNotFoundException: + # Mach may be invoked with the state dir as the current working + # directory, in which case we're not able to find the topsrcdir (so + # we can't create a MozbuildObject instance). + # Without this information, we're unable to filter argv paths, so + # we skip submitting them to telemetry. + return + metrics.mach.argv.set( + filter_args(command, sys.argv, instance.topsrcdir, instance.topobjdir) + ) + + +def is_applicable_telemetry_environment(): + if os.environ.get("MACH_MAIN_PID") != str(os.getpid()): + # This is a child mach process. Since we're collecting telemetry for the parent, + # we don't want to collect telemetry again down here. + return False + + if any(e in os.environ for e in ("MOZ_AUTOMATION", "TASK_ID")): + return False + + return True + + +def is_telemetry_enabled(settings): + if os.environ.get("DISABLE_TELEMETRY") == "1": + return False + + return settings.mach_telemetry.is_enabled + + +def arcrc_path(): + if sys.platform.startswith("win32") or sys.platform.startswith("msys"): + return Path(os.environ.get("APPDATA", "")) / ".arcrc" + else: + return Path("~/.arcrc").expanduser() + + +def resolve_setting_from_arcconfig(topsrcdir: Path, setting): + git_path = topsrcdir / ".git" + if git_path.is_file(): + git_path = subprocess.check_output( + ["git", "rev-parse", "--git-common-dir"], + cwd=str(topsrcdir), + universal_newlines=True, + ) + git_path = Path(git_path) + + for arcconfig_path in [ + topsrcdir / ".hg" / ".arcconfig", + git_path / ".arcconfig", + topsrcdir / ".arcconfig", + ]: + try: + with open(arcconfig_path, "r") as arcconfig_file: + arcconfig = json.load(arcconfig_file) + except (json.JSONDecodeError, FileNotFoundError): + continue + + value = arcconfig.get(setting) + if value: + return value + + +def resolve_is_employee_by_credentials(topsrcdir: Path): + phabricator_uri = resolve_setting_from_arcconfig(topsrcdir, "phabricator.uri") + + if not phabricator_uri: + return None + + try: + with open(arcrc_path(), "r") as arcrc_file: + arcrc = json.load(arcrc_file) + except (json.JSONDecodeError, FileNotFoundError): + return None + + phabricator_token = ( + arcrc.get("hosts", {}) + .get(urllib_parse.urljoin(phabricator_uri, "api/"), {}) + .get("token") + ) + + if not phabricator_token: + return None + + bmo_uri = ( + resolve_setting_from_arcconfig(topsrcdir, "bmo_url") + or "https://bugzilla.mozilla.org" + ) + bmo_api_url = urllib_parse.urljoin(bmo_uri, "rest/whoami") + bmo_result = requests.get( + bmo_api_url, headers={"X-PHABRICATOR-TOKEN": phabricator_token} + ) + return "mozilla-employee-confidential" in bmo_result.json().get("groups", []) + + +def resolve_is_employee_by_vcs(topsrcdir: Path): + try: + vcs = get_repository_object(str(topsrcdir)) + except InvalidRepoPath: + return None + + email = vcs.get_user_email() + if not email: + return None + + return "@mozilla.com" in email + + +def resolve_is_employee(topsrcdir: Path): + """Detect whether or not the current user is a Mozilla employee. + + Checks using Bugzilla authentication, if possible. Otherwise falls back to checking + if email configured in VCS is "@mozilla.com". + + Returns True if the user could be identified as an employee, False if the user + is confirmed as not being an employee, or None if the user couldn't be + identified. + """ + is_employee = resolve_is_employee_by_credentials(topsrcdir) + if is_employee is not None: + return is_employee + + return resolve_is_employee_by_vcs(topsrcdir) or False + + +def record_telemetry_settings( + main_settings, + state_dir: Path, + is_enabled, +): + # We want to update the user's machrc file. However, the main settings object + # contains config from "$topsrcdir/machrc" (if it exists) which we don't want + # to accidentally include. So, we have to create a brand new mozbuild-specific + # settings, update it, then write to it. + settings_path = state_dir / "machrc" + file_settings = ConfigSettings() + file_settings.register_provider(TelemetrySettings) + try: + file_settings.load_file(settings_path) + except configparser.Error as error: + print( + f"Your mach configuration file at `{settings_path}` cannot be parsed:\n{error}" + ) + return + + file_settings.mach_telemetry.is_enabled = is_enabled + file_settings.mach_telemetry.is_set_up = True + + with open(settings_path, "w") as f: + file_settings.write(f) + + # Telemetry will want this elsewhere in the mach process, so we'll slap the + # new values on the main settings object. + main_settings.mach_telemetry.is_enabled = is_enabled + main_settings.mach_telemetry.is_set_up = True + + +TELEMETRY_DESCRIPTION_PREAMBLE = """ +Mozilla collects data to improve the developer experience. +To learn more about the data we intend to collect, read here: + https://firefox-source-docs.mozilla.org/build/buildsystem/telemetry.html +If you have questions, please ask in #build on Matrix: + https://chat.mozilla.org/#/room/#build:mozilla.org +""".strip() + + +def print_telemetry_message_employee(): + message_template = dedent( + """ + %s + + As a Mozilla employee, telemetry has been automatically enabled. + """ + ).strip() + print(message_template % TELEMETRY_DESCRIPTION_PREAMBLE) + return True + + +def prompt_telemetry_message_contributor(): + while True: + prompt = ( + dedent( + """ + %s + + If you'd like to opt out of data collection, select (N) at the prompt. + Would you like to enable build system telemetry? (Yn): """ + ) + % TELEMETRY_DESCRIPTION_PREAMBLE + ).strip() + + choice = input(prompt) + choice = choice.strip().lower() + if choice == "": + return True + if choice not in ("y", "n"): + print("ERROR! Please enter y or n!") + else: + return choice == "y" + + +def initialize_telemetry_setting(settings, topsrcdir: str, state_dir: str): + """Enables telemetry for employees or prompts the user.""" + # If the user doesn't care about telemetry for this invocation, then + # don't make requests to Bugzilla and/or prompt for whether the + # user wants to opt-in. + + if topsrcdir is not None: + topsrcdir = Path(topsrcdir) + + if state_dir is not None: + state_dir = Path(state_dir) + + if os.environ.get("DISABLE_TELEMETRY") == "1": + return + + try: + is_employee = resolve_is_employee(topsrcdir) + except requests.exceptions.RequestException: + return + + if is_employee: + is_enabled = True + print_telemetry_message_employee() + else: + is_enabled = prompt_telemetry_message_contributor() + + record_telemetry_settings(settings, state_dir, is_enabled) diff --git a/python/mach/mach/telemetry_interface.py b/python/mach/mach/telemetry_interface.py new file mode 100644 index 0000000000..3ed8ce5674 --- /dev/null +++ b/python/mach/mach/telemetry_interface.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +from pathlib import Path +from typing import Union +from unittest.mock import Mock + +from mach.site import MozSiteMetadata, SitePackagesSource + + +class NoopTelemetry(object): + def __init__(self, failed_glean_import): + self._failed_glean_import = failed_glean_import + + def metrics(self, metrics_path: Union[str, Path]): + return Mock() + + def submit(self, is_bootstrap): + if self._failed_glean_import and not is_bootstrap: + active_site = MozSiteMetadata.from_runtime() + if active_site.mach_site_packages_source == SitePackagesSource.SYSTEM: + hint = ( + "Mach is looking for glean in the system packages. This can be " + "resolved by installing it there, or by allowing Mach to run " + "without using the system Python packages." + ) + elif active_site.mach_site_packages_source == SitePackagesSource.NONE: + hint = ( + "This is because Mach is currently configured without a source " + "for native Python packages." + ) + else: + hint = "You may need to run |mach bootstrap|." + + print( + f"Glean could not be found, so telemetry will not be reported. {hint}", + file=sys.stderr, + ) + + +class GleanTelemetry(object): + """Records and sends Telemetry using Glean. + + Metrics are defined in python/mozbuild/metrics.yaml. + Pings are defined in python/mozbuild/pings.yaml. + + The "metrics" and "pings" properties may be replaced with no-op implementations if + Glean isn't available. This allows consumers to report telemetry without having + to guard against incompatible environments. + + Also tracks whether an employee was just automatically opted into telemetry + during this mach invocation. + """ + + def __init__( + self, + ): + self._metrics_cache = {} + + def metrics(self, metrics_path: Union[str, Path]): + if metrics_path not in self._metrics_cache: + from glean import load_metrics + + metrics = load_metrics(metrics_path) + self._metrics_cache[metrics_path] = metrics + + return self._metrics_cache[metrics_path] + + def submit(self, _): + from pathlib import Path + + from glean import load_pings + + pings = load_pings(Path(__file__).parent.parent / "pings.yaml") + pings.usage.submit() diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py new file mode 100644 index 0000000000..a0c8d0a6ed --- /dev/null +++ b/python/mach/mach/terminal.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This file contains code for interacting with terminals. + +All the terminal interaction code is consolidated so the complexity can be in +one place, away from code that is commonly looked at. +""" + +import logging +import sys + +from six.moves import range + + +class LoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This is alternative terminal logging handler which contains smarts for + emitting terminal control characters properly. Currently, it has generic + support for "footer" elements at the bottom of the screen. Functionality + can be added when needed. + """ + + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write("\n") + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.flush() + + +class TerminalFooter(object): + """Represents something drawn on the bottom of a terminal.""" + + def __init__(self, terminal): + self.t = terminal + self.fh = sys.stdout + + def _clear_lines(self, n): + for i in range(n): + self.fh.write(self.t.move_x(0)) + self.fh.write(self.t.clear_eol()) + self.fh.write(self.t.move_up()) + + self.fh.write(self.t.move_down()) + self.fh.write(self.t.move_x(0)) + + def clear(self): + raise Exception("clear() must be implemented.") + + def draw(self): + raise Exception("draw() must be implemented.") diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/__init__.py diff --git a/python/mach/mach/test/conftest.py b/python/mach/mach/test/conftest.py new file mode 100644 index 0000000000..78129acb58 --- /dev/null +++ b/python/mach/mach/test/conftest.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import unittest +from collections.abc import Iterable +from pathlib import Path +from typing import List, Optional, Union + +import pytest +from buildconfig import topsrcdir + +try: + from StringIO import StringIO +except ImportError: + # TODO io.StringIO causes failures with Python 2 (needs to be sorted out) + from io import StringIO + +from mach.main import Mach + +PROVIDER_DIR = Path(__file__).resolve().parent / "providers" + + +@pytest.fixture(scope="class") +def get_mach(request): + def _populate_context(key): + if key == "topdir": + return topsrcdir + + def inner( + provider_files: Optional[Union[Path, List[Path]]] = None, + entry_point=None, + context_handler=None, + ): + m = Mach(str(Path.cwd())) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + m.define_category("misc", "Mach misc", "Testing for mach core", 20) + m.populate_context_handler = context_handler or _populate_context + + if provider_files: + if not isinstance(provider_files, Iterable): + provider_files = [provider_files] + + for path in provider_files: + m.load_commands_from_file(PROVIDER_DIR / path) + + if entry_point: + m.load_commands_from_entry_point(entry_point) + + return m + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls.get_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.fixture(scope="class") +def run_mach(request, get_mach): + def inner(argv, *args, **kwargs): + m = get_mach(*args, **kwargs) + + stdout = StringIO() + stderr = StringIO() + + if sys.version_info < (3, 0): + stdout.encoding = "UTF-8" + stderr.encoding = "UTF-8" + + try: + result = m.run(argv, stdout=stdout, stderr=stderr) + except SystemExit: + result = None + + return (result, stdout.getvalue(), stderr.getvalue()) + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls._run_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestBase(unittest.TestCase): + pass diff --git a/python/mach/mach/test/invoke_mach_command.py b/python/mach/mach/test/invoke_mach_command.py new file mode 100644 index 0000000000..1efa102ef5 --- /dev/null +++ b/python/mach/mach/test/invoke_mach_command.py @@ -0,0 +1,4 @@ +import subprocess +import sys + +subprocess.check_call([sys.executable] + sys.argv[1:]) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/providers/__init__.py diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py new file mode 100644 index 0000000000..26cdfdf588 --- /dev/null +++ b/python/mach/mach/test/providers/basic.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mach.decorators import Command, CommandArgument + + +@Command("cmd_foo", category="testing") +def run_foo(command_context): + pass + + +@Command("cmd_bar", category="testing") +@CommandArgument("--baz", action="store_true", help="Run with baz") +def run_bar(command_context, baz=None): + pass diff --git a/python/mach/mach/test/providers/commands.py b/python/mach/mach/test/providers/commands.py new file mode 100644 index 0000000000..6b8210c513 --- /dev/null +++ b/python/mach/mach/test/providers/commands.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from functools import partial + +from mach.decorators import Command, CommandArgument + + +def is_foo(cls): + """Foo must be true""" + return cls.foo + + +def is_bar(val, cls): + """Bar must equal val""" + return cls.bar == val + + +@Command("cmd_foo", category="testing") +@CommandArgument("--arg", default=None, help="Argument help.") +def run_foo(command_context): + pass + + +@Command("cmd_bar", category="testing", conditions=[partial(is_bar, False)]) +def run_bar(command_context): + pass + + +@Command("cmd_foobar", category="testing", conditions=[is_foo, partial(is_bar, True)]) +def run_foobar(command_context): + pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py new file mode 100644 index 0000000000..db2f3f8123 --- /dev/null +++ b/python/mach/mach/test/providers/conditions.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command + + +def is_true(cls): + return True + + +def is_false(cls): + return False + + +@Command("cmd_condition_true", category="testing", conditions=[is_true]) +def run_condition_true(self, command_context): + pass + + +@Command("cmd_condition_false", category="testing", conditions=[is_false]) +def run_condition_false(self, command_context): + pass + + +@Command( + "cmd_condition_true_and_false", category="testing", conditions=[is_true, is_false] +) +def run_condition_true_and_false(self, command_context): + pass + + +def is_ctx_foo(cls): + """Foo must be true""" + return cls._mach_context.foo + + +def is_ctx_bar(cls): + """Bar must be true""" + return cls._mach_context.bar + + +@Command("cmd_foo_ctx", category="testing", conditions=[is_ctx_foo]) +def run_foo_ctx(self, command_context): + pass + + +@Command("cmd_bar_ctx", category="testing", conditions=[is_ctx_bar]) +def run_bar_ctx(self, command_context): + pass + + +@Command("cmd_foobar_ctx", category="testing", conditions=[is_ctx_foo, is_ctx_bar]) +def run_foobar_ctx(self, command_context): + pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py new file mode 100644 index 0000000000..228c56f0bf --- /dev/null +++ b/python/mach/mach/test/providers/conditions_invalid.py @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command + + +@Command("cmd_foo", category="testing", conditions=["invalid"]) +def run_foo(command_context): + pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py new file mode 100644 index 0000000000..9ddc7653c0 --- /dev/null +++ b/python/mach/mach/test/providers/throw.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command, CommandArgument +from mach.test.providers import throw2 + + +@Command("throw", category="testing") +@CommandArgument("--message", "-m", default="General Error") +def throw(command_context, message): + raise Exception(message) + + +@Command("throw_deep", category="testing") +@CommandArgument("--message", "-m", default="General Error") +def throw_deep(command_context, message): + throw2.throw_deep(message) diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py new file mode 100644 index 0000000000..9ff7f2798e --- /dev/null +++ b/python/mach/mach/test/providers/throw2.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file exists to trigger the differences in mach error reporting between +# exceptions that occur in mach command modules themselves and in the things +# they call. + + +def throw_deep(message): + return throw_real(message) + + +def throw_real(message): + raise Exception(message) diff --git a/python/mach/mach/test/python.ini b/python/mach/mach/test/python.ini new file mode 100644 index 0000000000..de09924b67 --- /dev/null +++ b/python/mach/mach/test/python.ini @@ -0,0 +1,22 @@ +[DEFAULT] +subsuite = mach + +[test_commands.py] +[test_conditions.py] +skip-if = python == 3 +[test_config.py] +[test_decorators.py] +[test_dispatcher.py] +[test_entry_point.py] +[test_error_output.py] +skip-if = python == 3 +[test_logger.py] +[test_mach.py] +[test_site.py] +[test_site_activation.py] +[test_site_compatibility.py] +# The Windows and Mac workers only use the internal PyPI mirror, +# which will be missing packages required for this test. +skip-if = + os == "win" + os == "mac" diff --git a/python/mach/mach/test/script_site_activation.py b/python/mach/mach/test/script_site_activation.py new file mode 100644 index 0000000000..8c23f1a19c --- /dev/null +++ b/python/mach/mach/test/script_site_activation.py @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script is used by "test_site_activation.py" to verify how site activations +# affect the sys.path. +# The sys.path is printed in three stages: +# 1. Once at the beginning +# 2. Once after Mach site activation +# 3. Once after the command site activation +# The output of this script should be an ast-parsable list with three nested lists: one +# for each sys.path state. +# Note that virtualenv-creation output may need to be filtered out - it can be done by +# only ast-parsing the last line of text outputted by this script. + +import os +import sys +from unittest.mock import patch + +from mach.requirements import MachEnvRequirements, PthSpecifier +from mach.site import CommandSiteManager, MachSiteManager + + +def main(): + # Should be set by calling test + topsrcdir = os.environ["TOPSRCDIR"] + command_site = os.environ["COMMAND_SITE"] + mach_site_requirements = os.environ["MACH_SITE_PTH_REQUIREMENTS"] + command_site_requirements = os.environ["COMMAND_SITE_PTH_REQUIREMENTS"] + work_dir = os.environ["WORK_DIR"] + + def resolve_requirements(topsrcdir, site_name): + req = MachEnvRequirements() + if site_name == "mach": + req.pth_requirements = [ + PthSpecifier(path) for path in mach_site_requirements.split(os.pathsep) + ] + else: + req.pth_requirements = [PthSpecifier(command_site_requirements)] + return req + + with patch("mach.site.resolve_requirements", resolve_requirements): + initial_sys_path = sys.path.copy() + + mach_site = MachSiteManager.from_environment( + topsrcdir, + lambda: work_dir, + ) + mach_site.activate() + mach_sys_path = sys.path.copy() + + command_site = CommandSiteManager.from_environment( + topsrcdir, lambda: work_dir, command_site, work_dir + ) + command_site.activate() + command_sys_path = sys.path.copy() + print( + [ + initial_sys_path, + mach_sys_path, + command_sys_path, + ] + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_commands.py b/python/mach/mach/test/test_commands.py new file mode 100644 index 0000000000..38191b0898 --- /dev/null +++ b/python/mach/mach/test/test_commands.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +from pathlib import Path + +import pytest +from buildconfig import topsrcdir +from mozunit import main + +import mach + +ALL_COMMANDS = [ + "cmd_bar", + "cmd_foo", + "cmd_foobar", + "mach-commands", + "mach-completion", + "mach-debug-commands", +] + + +@pytest.fixture +def run_completion(run_mach): + def inner(args=[]): + mach_dir = Path(mach.__file__).parent + providers = [ + Path("commands.py"), + mach_dir / "commands" / "commandinfo.py", + ] + + def context_handler(key): + if key == "topdir": + return topsrcdir + + args = ["mach-completion"] + args + return run_mach(args, providers, context_handler=context_handler) + + return inner + + +def format(targets): + return "\n".join(targets) + "\n" + + +def test_mach_completion(run_completion): + result, stdout, stderr = run_completion() + assert result == 0 + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_f"]) + assert result == 0 + # While it seems like this should return only commands that have + # 'cmd_f' as a prefix, the completion script will handle this case + # properly. + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_foo"]) + assert result == 0 + assert stdout == format(["help", "--arg"]) + + +@pytest.mark.parametrize("shell", ("bash", "fish", "zsh")) +def test_generate_mach_completion_script(run_completion, shell): + rv, out, err = run_completion([shell]) + print(out) + print(err, file=sys.stderr) + assert rv == 0 + assert err == "" + + assert "cmd_foo" in out + assert "arg" in out + assert "cmd_foobar" in out + assert "cmd_bar" in out + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py new file mode 100644 index 0000000000..5775790e69 --- /dev/null +++ b/python/mach/mach/test/test_conditions.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path + +from buildconfig import topsrcdir +from mozunit import main + +from mach.base import MachError +from mach.main import Mach +from mach.registrar import Registrar +from mach.test.conftest import PROVIDER_DIR, TestBase + + +def _make_populate_context(include_extra_attributes): + def _populate_context(key=None): + if key is None: + return + + attributes = { + "topdir": topsrcdir, + } + if include_extra_attributes: + attributes["foo"] = True + attributes["bar"] = False + + try: + return attributes[key] + except KeyError: + raise AttributeError(key) + + return _populate_context + + +_populate_bare_context = _make_populate_context(False) +_populate_context = _make_populate_context(True) + + +class TestConditions(TestBase): + """Tests for conditionally filtering commands.""" + + def _run(self, args, context_handler=_populate_bare_context): + return self._run_mach( + args, Path("conditions.py"), context_handler=context_handler + ) + + def test_conditions_pass(self): + """Test that a command which passes its conditions is runnable.""" + + self.assertEqual((0, "", ""), self._run(["cmd_condition_true"])) + self.assertEqual((0, "", ""), self._run(["cmd_foo_ctx"], _populate_context)) + + def test_invalid_context_message(self): + """Test that commands which do not pass all their conditions + print the proper failure message.""" + + def is_bar(): + """Bar must be true""" + + fail_conditions = [is_bar] + + for name in ("cmd_condition_false", "cmd_condition_true_and_false"): + result, stdout, stderr = self._run([name]) + self.assertEqual(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEqual(fail_msg.rstrip(), stdout.rstrip()) + + for name in ("cmd_bar_ctx", "cmd_foobar_ctx"): + result, stdout, stderr = self._run([name], _populate_context) + self.assertEqual(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEqual(fail_msg.rstrip(), stdout.rstrip()) + + def test_invalid_type(self): + """Test that a condition which is not callable raises an exception.""" + + m = Mach(str(Path.cwd())) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + self.assertRaises( + MachError, + m.load_commands_from_file, + PROVIDER_DIR / "conditions_invalid.py", + ) + + def test_help_message(self): + """Test that commands that are not runnable do not show up in help.""" + + result, stdout, stderr = self._run(["help"], _populate_context) + self.assertIn("cmd_condition_true", stdout) + self.assertNotIn("cmd_condition_false", stdout) + self.assertNotIn("cmd_condition_true_and_false", stdout) + self.assertIn("cmd_foo_ctx", stdout) + self.assertNotIn("cmd_bar_ctx", stdout) + self.assertNotIn("cmd_foobar_ctx", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py new file mode 100644 index 0000000000..25b75c8685 --- /dev/null +++ b/python/mach/mach/test/test_config.py @@ -0,0 +1,292 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +import sys +import unittest +from pathlib import Path + +from mozfile.mozfile import NamedTemporaryFile +from mozunit import main +from six import string_types + +from mach.config import ( + BooleanType, + ConfigException, + ConfigSettings, + IntegerType, + PathType, + PositiveIntegerType, + StringType, +) +from mach.decorators import SettingsProvider + +CONFIG1 = r""" +[foo] + +bar = bar_value +baz = /baz/foo.c +""" + +CONFIG2 = r""" +[foo] + +bar = value2 +""" + + +@SettingsProvider +class Provider1(object): + config_settings = [ + ("foo.bar", StringType, "desc"), + ("foo.baz", PathType, "desc"), + ] + + +@SettingsProvider +class ProviderDuplicate(object): + config_settings = [ + ("dupesect.foo", StringType, "desc"), + ("dupesect.foo", StringType, "desc"), + ] + + +@SettingsProvider +class Provider2(object): + config_settings = [ + ("a.string", StringType, "desc"), + ("a.boolean", BooleanType, "desc"), + ("a.pos_int", PositiveIntegerType, "desc"), + ("a.int", IntegerType, "desc"), + ("a.path", PathType, "desc"), + ] + + +@SettingsProvider +class Provider3(object): + @classmethod + def config_settings(cls): + return [ + ("a.string", "string", "desc"), + ("a.boolean", "boolean", "desc"), + ("a.pos_int", "pos_int", "desc"), + ("a.int", "int", "desc"), + ("a.path", "path", "desc"), + ] + + +@SettingsProvider +class Provider4(object): + config_settings = [ + ("foo.abc", StringType, "desc", "a", {"choices": set("abc")}), + ("foo.xyz", StringType, "desc", "w", {"choices": set("xyz")}), + ] + + +@SettingsProvider +class Provider5(object): + config_settings = [ + ("foo.*", "string", "desc"), + ("foo.bar", "string", "desc"), + ] + + +class TestConfigSettings(unittest.TestCase): + def test_empty(self): + s = ConfigSettings() + + self.assertEqual(len(s), 0) + self.assertNotIn("foo", s) + + def test_duplicate_option(self): + s = ConfigSettings() + + with self.assertRaises(ConfigException): + s.register_provider(ProviderDuplicate) + + def test_simple(self): + s = ConfigSettings() + s.register_provider(Provider1) + + self.assertEqual(len(s), 1) + self.assertIn("foo", s) + + foo = s["foo"] + foo = s.foo + + self.assertEqual(len(foo), 0) + self.assertEqual(len(foo._settings), 2) + + self.assertIn("bar", foo._settings) + self.assertIn("baz", foo._settings) + + self.assertNotIn("bar", foo) + foo["bar"] = "value1" + self.assertIn("bar", foo) + + self.assertEqual(foo["bar"], "value1") + self.assertEqual(foo.bar, "value1") + + def test_assignment_validation(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + # Assigning an undeclared setting raises. + exc_type = AttributeError if sys.version_info < (3, 0) else KeyError + with self.assertRaises(exc_type): + a.undefined = True + + with self.assertRaises(KeyError): + a["undefined"] = True + + # Basic type validation. + a.string = "foo" + a.string = "foo" + + with self.assertRaises(TypeError): + a.string = False + + a.boolean = True + a.boolean = False + + with self.assertRaises(TypeError): + a.boolean = "foo" + + a.pos_int = 5 + a.pos_int = 0 + + with self.assertRaises(ValueError): + a.pos_int = -1 + + with self.assertRaises(TypeError): + a.pos_int = "foo" + + a.int = 5 + a.int = 0 + a.int = -5 + + with self.assertRaises(TypeError): + a.int = 1.24 + + with self.assertRaises(TypeError): + a.int = "foo" + + a.path = "/home/gps" + a.path = "foo.c" + a.path = "foo/bar" + a.path = "./foo" + + def retrieval_type_helper(self, provider): + s = ConfigSettings() + s.register_provider(provider) + + a = s.a + + a.string = "foo" + a.boolean = True + a.pos_int = 12 + a.int = -4 + a.path = "./foo/bar" + + self.assertIsInstance(a.string, string_types) + self.assertIsInstance(a.boolean, bool) + self.assertIsInstance(a.pos_int, int) + self.assertIsInstance(a.int, int) + self.assertIsInstance(a.path, string_types) + + def test_retrieval_type(self): + self.retrieval_type_helper(Provider2) + self.retrieval_type_helper(Provider3) + + def test_choices_validation(self): + s = ConfigSettings() + s.register_provider(Provider4) + + foo = s.foo + foo.abc + with self.assertRaises(ValueError): + foo.xyz + + with self.assertRaises(ValueError): + foo.abc = "e" + + foo.abc = "b" + foo.xyz = "y" + + def test_wildcard_options(self): + s = ConfigSettings() + s.register_provider(Provider5) + + foo = s.foo + + self.assertIn("*", foo._settings) + self.assertNotIn("*", foo) + + foo.baz = "value1" + foo.bar = "value2" + + self.assertIn("baz", foo) + self.assertEqual(foo.baz, "value1") + + self.assertIn("bar", foo) + self.assertEqual(foo.bar, "value2") + + def test_file_reading_single(self): + temp = NamedTemporaryFile(mode="wt") + temp.write(CONFIG1) + temp.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_file(Path(temp.name)) + + self.assertEqual(s.foo.bar, "bar_value") + + def test_file_reading_multiple(self): + """Loading multiple files has proper overwrite behavior.""" + temp1 = NamedTemporaryFile(mode="wt") + temp1.write(CONFIG1) + temp1.flush() + + temp2 = NamedTemporaryFile(mode="wt") + temp2.write(CONFIG2) + temp2.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_files([Path(temp1.name), Path(temp2.name)]) + + self.assertEqual(s.foo.bar, "value2") + + def test_file_reading_missing(self): + """Missing files should silently be ignored.""" + + s = ConfigSettings() + + s.load_file("/tmp/foo.ini") + + def test_file_writing(self): + s = ConfigSettings() + s.register_provider(Provider2) + + s.a.string = "foo" + s.a.boolean = False + + temp = NamedTemporaryFile("wt") + s.write(temp) + temp.flush() + + s2 = ConfigSettings() + s2.register_provider(Provider2) + + s2.load_file(temp.name) + + self.assertEqual(s.a.string, s2.a.string) + self.assertEqual(s.a.boolean, s2.a.boolean) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_decorators.py b/python/mach/mach/test/test_decorators.py new file mode 100644 index 0000000000..f33b6e7d8f --- /dev/null +++ b/python/mach/mach/test/test_decorators.py @@ -0,0 +1,133 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path +from unittest import mock +from unittest.mock import Mock, patch + +import pytest +from mozbuild.base import MachCommandBase +from mozunit import main + +import mach.decorators +import mach.registrar +from mach.base import MachError +from mach.decorators import Command, CommandArgument, SubCommand +from mach.requirements import MachEnvRequirements +from mach.site import CommandSiteManager, MozSiteMetadata, SitePackagesSource + + +@pytest.fixture +def registrar(monkeypatch): + test_registrar = mach.registrar.MachRegistrar() + test_registrar.register_category( + "testing", "Mach unittest", "Testing for mach decorators" + ) + monkeypatch.setattr(mach.decorators, "Registrar", test_registrar) + return test_registrar + + +def test_register_command_with_argument(registrar): + inner_function = Mock() + context = Mock() + context.cwd = "." + + @Command("cmd_foo", category="testing") + @CommandArgument("--arg", default=None, help="Argument help.") + def run_foo(command_context, arg): + inner_function(arg) + + registrar.dispatch("cmd_foo", context, arg="argument") + + inner_function.assert_called_with("argument") + + +def test_register_command_with_metrics_path(registrar): + context = Mock() + context.cwd = "." + + metrics_path = "metrics/path" + metrics_mock = Mock() + context.telemetry.metrics.return_value = metrics_mock + + @Command("cmd_foo", category="testing", metrics_path=metrics_path) + def run_foo(command_context): + assert command_context.metrics == metrics_mock + + @SubCommand("cmd_foo", "sub_foo", metrics_path=metrics_path + "2") + def run_subfoo(command_context): + assert command_context.metrics == metrics_mock + + registrar.dispatch("cmd_foo", context) + + context.telemetry.metrics.assert_called_with(metrics_path) + assert context.handler.metrics_path == metrics_path + + registrar.dispatch("cmd_foo", context, subcommand="sub_foo") + assert context.handler.metrics_path == metrics_path + "2" + + +def test_register_command_sets_up_class_at_runtime(registrar): + inner_function = Mock() + + context = Mock() + context.cwd = "." + + # We test that the virtualenv is set up properly dynamically on + # the instance that actually runs the command. + @Command("cmd_foo", category="testing", virtualenv_name="env_foo") + def run_foo(command_context): + assert ( + Path(command_context.virtualenv_manager.virtualenv_root).name == "env_foo" + ) + inner_function("foo") + + @Command("cmd_bar", category="testing", virtualenv_name="env_bar") + def run_bar(command_context): + assert ( + Path(command_context.virtualenv_manager.virtualenv_root).name == "env_bar" + ) + inner_function("bar") + + def from_environment_patch( + topsrcdir: str, state_dir: str, virtualenv_name, directory: str + ): + return CommandSiteManager( + "", + "", + virtualenv_name, + virtualenv_name, + MozSiteMetadata(0, "mach", SitePackagesSource.VENV, "", ""), + True, + MachEnvRequirements(), + ) + + with mock.patch.object( + CommandSiteManager, "from_environment", from_environment_patch + ): + with patch.object(MachCommandBase, "activate_virtualenv"): + registrar.dispatch("cmd_foo", context) + inner_function.assert_called_with("foo") + registrar.dispatch("cmd_bar", context) + inner_function.assert_called_with("bar") + + +def test_cannot_create_command_nonexisting_category(registrar): + with pytest.raises(MachError): + + @Command("cmd_foo", category="bar") + def run_foo(command_context): + pass + + +def test_subcommand_requires_parent_to_exist(registrar): + with pytest.raises(MachError): + + @SubCommand("sub_foo", "foo") + def run_foo(command_context): + pass + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_dispatcher.py b/python/mach/mach/test/test_dispatcher.py new file mode 100644 index 0000000000..85c2e9a847 --- /dev/null +++ b/python/mach/mach/test/test_dispatcher.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from io import StringIO +from pathlib import Path + +import pytest +from mozunit import main +from six import string_types + +from mach.base import CommandContext +from mach.registrar import Registrar + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestDispatcher(unittest.TestCase): + """Tests dispatch related code""" + + def get_parser(self, config=None): + mach = self.get_mach(Path("basic.py")) + + for provider in Registrar.settings_providers: + mach.settings.register_provider(provider) + + if config: + if isinstance(config, string_types): + config = StringIO(config) + mach.settings.load_fps([config]) + + context = CommandContext(cwd="", settings=mach.settings) + return mach.get_argument_parser(context) + + def test_command_aliases(self): + config = """ +[alias] +foo = cmd_foo +bar = cmd_bar +baz = cmd_bar --baz +cmd_bar = cmd_bar --baz +""" + parser = self.get_parser(config=config) + + args = parser.parse_args(["foo"]) + self.assertEqual(args.command, "cmd_foo") + + def assert_bar_baz(argv): + args = parser.parse_args(argv) + self.assertEqual(args.command, "cmd_bar") + self.assertTrue(args.command_args.baz) + + # The following should all result in |cmd_bar --baz| + assert_bar_baz(["bar", "--baz"]) + assert_bar_baz(["baz"]) + assert_bar_baz(["cmd_bar"]) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py new file mode 100644 index 0000000000..1129eba476 --- /dev/null +++ b/python/mach/mach/test/test_entry_point.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import imp +import sys +from pathlib import Path +from unittest.mock import patch + +from mozunit import main + +from mach.base import MachError +from mach.test.conftest import TestBase + + +class Entry: + """Stub replacement for pkg_resources.EntryPoint""" + + def __init__(self, providers): + self.providers = providers + + def load(self): + def _providers(): + return self.providers + + return _providers + + +class TestEntryPoints(TestBase): + """Test integrating with setuptools entry points""" + + provider_dir = Path(__file__).parent.resolve() / "providers" + + def _run_help(self): + return self._run_mach(["help"], entry_point="mach.providers") + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_directory(self, mock): + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if "mach.commands" not in sys.modules: + mod = imp.new_module("mach.commands") + sys.modules["mach.commands"] = mod + + mock.return_value = [Entry([self.provider_dir])] + # Mach error raised due to conditions_invalid.py + with self.assertRaises(MachError): + self._run_help() + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_file(self, mock): + mock.return_value = [Entry([self.provider_dir / "basic.py"])] + + result, stdout, stderr = self._run_help() + self.assertIsNone(result) + self.assertIn("cmd_foo", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py new file mode 100644 index 0000000000..12eab65856 --- /dev/null +++ b/python/mach/mach/test/test_error_output.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path + +from mozunit import main + +from mach.main import COMMAND_ERROR_TEMPLATE, MODULE_ERROR_TEMPLATE + + +def test_command_error(run_mach): + result, stdout, stderr = run_mach( + ["throw", "--message", "Command Error"], provider_files=Path("throw.py") + ) + assert result == 1 + assert COMMAND_ERROR_TEMPLATE % "throw" in stdout + + +def test_invoked_error(run_mach): + result, stdout, stderr = run_mach( + ["throw_deep", "--message", "Deep stack"], provider_files=Path("throw.py") + ) + assert result == 1 + assert MODULE_ERROR_TEMPLATE % "throw_deep" in stdout + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py new file mode 100644 index 0000000000..643d890de8 --- /dev/null +++ b/python/mach/mach/test/test_logger.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import time +import unittest + +from mozunit import main + +from mach.logging import StructuredHumanFormatter + + +class DummyLogger(logging.Logger): + def __init__(self, cb): + logging.Logger.__init__(self, "test") + + self._cb = cb + + def handle(self, record): + self._cb(record) + + +class TestStructuredHumanFormatter(unittest.TestCase): + def test_non_ascii_logging(self): + # Ensures the formatter doesn't choke when non-ASCII characters are + # present in printed parameters. + formatter = StructuredHumanFormatter(time.time()) + + def on_record(record): + result = formatter.format(record) + relevant = result[9:] + + self.assertEqual(relevant, "Test: s\xe9curit\xe9") + + logger = DummyLogger(on_record) + + value = "s\xe9curit\xe9" + + logger.log( + logging.INFO, + "Test: {utf}", + extra={"action": "action", "params": {"utf": value}}, + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_mach.py b/python/mach/mach/test/test_mach.py new file mode 100644 index 0000000000..38379d1b49 --- /dev/null +++ b/python/mach/mach/test/test_mach.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozunit import main + + +def test_set_isatty_environ(monkeypatch, get_mach): + # Make sure the 'MACH_STDOUT_ISATTY' variable gets set. + monkeypatch.delenv("MACH_STDOUT_ISATTY", raising=False) + monkeypatch.setattr(os, "isatty", lambda fd: True) + + m = get_mach() + orig_run = m._run + env_is_set = [] + + def wrap_run(*args, **kwargs): + env_is_set.append("MACH_STDOUT_ISATTY" in os.environ) + return orig_run(*args, **kwargs) + + monkeypatch.setattr(m, "_run", wrap_run) + + ret = m.run([]) + assert ret == 0 + assert env_is_set[0] + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_site.py b/python/mach/mach/test/test_site.py new file mode 100644 index 0000000000..d7c3d8c489 --- /dev/null +++ b/python/mach/mach/test/test_site.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from unittest import mock + +import pytest +from buildconfig import topsrcdir +from mozunit import main + +from mach.site import ( + PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS, + SitePackagesSource, + resolve_requirements, +) + + +@pytest.mark.parametrize( + "env_native_package_source,env_use_system_python,env_moz_automation,expected", + [ + ("system", False, False, SitePackagesSource.SYSTEM), + ("pip", False, False, SitePackagesSource.VENV), + ("none", False, False, SitePackagesSource.NONE), + (None, False, False, SitePackagesSource.VENV), + (None, False, True, SitePackagesSource.NONE), + (None, True, False, SitePackagesSource.NONE), + (None, True, True, SitePackagesSource.NONE), + ], +) +def test_resolve_package_source( + env_native_package_source, env_use_system_python, env_moz_automation, expected +): + with mock.patch.dict( + os.environ, + { + "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": env_native_package_source or "", + "MACH_USE_SYSTEM_PYTHON": "1" if env_use_system_python else "", + "MOZ_AUTOMATION": "1" if env_moz_automation else "", + }, + ): + assert SitePackagesSource.for_mach() == expected + + +def test_all_restricted_sites_dont_have_pypi_requirements(): + for site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS: + requirements = resolve_requirements(topsrcdir, site_name) + assert not requirements.pypi_requirements, ( + 'Sites that must be able to operate without "pip install" must not have any ' + f'mandatory "pypi requirements". However, the "{site_name}" site depends on: ' + f"{requirements.pypi_requirements}" + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_site_activation.py b/python/mach/mach/test/test_site_activation.py new file mode 100644 index 0000000000..e034a27b76 --- /dev/null +++ b/python/mach/mach/test/test_site_activation.py @@ -0,0 +1,463 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import ast +import functools +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from subprocess import CompletedProcess +from typing import List + +import buildconfig +import mozunit +import pkg_resources +import pytest + +from mach.site import MozSiteMetadata, PythonVirtualenv, activate_virtualenv + + +class ActivationContext: + def __init__( + self, + topsrcdir: Path, + work_dir: Path, + original_python_path: str, + stdlib_paths: List[Path], + system_paths: List[Path], + required_mach_sys_paths: List[Path], + mach_requirement_paths: List[Path], + command_requirement_path: Path, + ): + self.topsrcdir = topsrcdir + self.work_dir = work_dir + self.original_python_path = original_python_path + self.stdlib_paths = stdlib_paths + self.system_paths = system_paths + self.required_moz_init_sys_paths = required_mach_sys_paths + self.mach_requirement_paths = mach_requirement_paths + self.command_requirement_path = command_requirement_path + + def virtualenv(self, name: str) -> PythonVirtualenv: + base_path = self.work_dir + + if name == "mach": + base_path = base_path / "_virtualenvs" + return PythonVirtualenv(str(base_path / name)) + + +def test_new_package_appears_in_pkg_resources(): + try: + # "carrot" was chosen as the package to use because: + # * It has to be a package that doesn't exist in-scope at the start (so, + # all vendored modules included in the test virtualenv aren't usage). + # * It must be on our internal PyPI mirror. + # Of the options, "carrot" is a small install that fits these requirements. + pkg_resources.get_distribution("carrot") + assert False, "Expected to not find 'carrot' as the initial state of the test" + except pkg_resources.DistributionNotFound: + pass + + with tempfile.TemporaryDirectory() as venv_dir: + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + venv_dir, + ] + ) + + venv = PythonVirtualenv(venv_dir) + venv.pip_install(["carrot==0.10.7"]) + + initial_metadata = MozSiteMetadata.from_runtime() + try: + metadata = MozSiteMetadata(None, None, None, None, venv.prefix) + with metadata.update_current_site(venv.python_path): + activate_virtualenv(venv) + + assert pkg_resources.get_distribution("carrot").version == "0.10.7" + finally: + MozSiteMetadata.current = initial_metadata + + +def test_sys_path_source_none_build(context): + original, mach, command = _run_activation_script_for_paths(context, "none", "build") + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + assert mach == [ + *context.stdlib_paths, + *context.mach_requirement_paths, + ] + + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(context.virtualenv("build")) + assert command_venv == [Path(""), *expected_command_paths] + + +def test_sys_path_source_none_other(context): + original, mach, command = _run_activation_script_for_paths(context, "none", "other") + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + assert mach == [ + *context.stdlib_paths, + *context.mach_requirement_paths, + ] + + command_virtualenv = PythonVirtualenv(str(context.work_dir / "other")) + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(context.virtualenv("other")) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venv_build(context): + original, mach, command = _run_activation_script_for_paths(context, "pip", "build") + _assert_original_python_sys_path(context, original) + + mach_virtualenv = context.virtualenv("mach") + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + mach_venv = _sys_path_of_virtualenv(mach_virtualenv) + assert mach_venv == [ + Path(""), + *expected_mach_paths, + ] + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venv_other(context): + original, mach, command = _run_activation_script_for_paths(context, "pip", "other") + _assert_original_python_sys_path(context, original) + + mach_virtualenv = context.virtualenv("mach") + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("other") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + mach_venv = _sys_path_of_virtualenv(mach_virtualenv) + assert mach_venv == [ + Path(""), + *expected_mach_paths, + ] + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_system_build(context): + original, mach, command = _run_activation_script_for_paths( + context, "system", "build" + ) + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *context.system_paths, + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *context.system_paths, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_system_other(context): + result = _run_activation_script( + context, + "system", + "other", + context.original_python_path, + stderr=subprocess.PIPE, + ) + assert result.returncode != 0 + assert ( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites ' + "other than" in result.stderr + ) + + +def test_sys_path_source_venvsystem_build(context): + venv_system_python = _create_venv_system_python( + context.work_dir, context.original_python_path + ) + venv_system_site_packages_dirs = [ + Path(p) for p in venv_system_python.site_packages_dirs() + ] + original, mach, command = _run_activation_script_for_paths( + context, "system", "build", venv_system_python.python_path + ) + + assert original == [ + Path(__file__).parent, + *context.required_moz_init_sys_paths, + *context.stdlib_paths, + *venv_system_site_packages_dirs, + ] + + assert not os.path.exists(context.virtualenv("mach").prefix) + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *venv_system_site_packages_dirs, + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *venv_system_site_packages_dirs, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venvsystem_other(context): + venv_system_python = _create_venv_system_python( + context.work_dir, context.original_python_path + ) + result = _run_activation_script( + context, + "system", + "other", + venv_system_python.python_path, + stderr=subprocess.PIPE, + ) + assert result.returncode != 0 + assert ( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites ' + "other than" in result.stderr + ) + + +@pytest.fixture(name="context") +def _activation_context(): + original_python_path, stdlib_paths, system_paths = _original_python() + topsrcdir = Path(buildconfig.topsrcdir) + required_mach_sys_paths = [ + topsrcdir / "python" / "mach", + topsrcdir / "third_party" / "python" / "packaging", + topsrcdir / "third_party" / "python" / "pyparsing", + topsrcdir / "third_party" / "python" / "pip", + ] + + with tempfile.TemporaryDirectory() as work_dir: + # Get "resolved" version of path to ease comparison against "site"-added sys.path + # entries, as "site" calculates the realpath of provided locations. + work_dir = Path(work_dir).resolve() + mach_requirement_paths = [ + *required_mach_sys_paths, + work_dir / "mach_site_path", + ] + command_requirement_path = work_dir / "command_site_path" + (work_dir / "mach_site_path").touch() + command_requirement_path.touch() + yield ActivationContext( + topsrcdir, + work_dir, + original_python_path, + stdlib_paths, + system_paths, + required_mach_sys_paths, + mach_requirement_paths, + command_requirement_path, + ) + + +@functools.lru_cache(maxsize=None) +def _original_python(): + current_site = MozSiteMetadata.from_runtime() + stdlib_paths, system_paths = current_site.original_python.sys_path() + stdlib_paths = [Path(path) for path in _filter_pydev_from_paths(stdlib_paths)] + system_paths = [Path(path) for path in system_paths] + return current_site.original_python.python_path, stdlib_paths, system_paths + + +def _run_activation_script( + context: ActivationContext, + source: str, + site_name: str, + invoking_python: str, + **kwargs +) -> CompletedProcess: + return subprocess.run( + [ + invoking_python, + str(Path(__file__).parent / "script_site_activation.py"), + ], + stdout=subprocess.PIPE, + universal_newlines=True, + env={ + "TOPSRCDIR": str(context.topsrcdir), + "COMMAND_SITE": site_name, + "PYTHONPATH": os.pathsep.join( + str(p) for p in context.required_moz_init_sys_paths + ), + "MACH_SITE_PTH_REQUIREMENTS": os.pathsep.join( + str(p) for p in context.mach_requirement_paths + ), + "COMMAND_SITE_PTH_REQUIREMENTS": str(context.command_requirement_path), + "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": source, + "WORK_DIR": str(context.work_dir), + # These two variables are needed on Windows so that Python initializes + # properly and adds the "user site packages" to the sys.path like normal. + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "APPDATA": os.environ.get("APPDATA", ""), + }, + **kwargs, + ) + + +def _run_activation_script_for_paths( + context: ActivationContext, source: str, site_name: str, invoking_python: str = None +) -> List[List[Path]]: + """Return the states of the sys.path when activating Mach-managed sites + + Three sys.path states are returned: + * The initial sys.path, equivalent to "path_to_python -c "import sys; print(sys.path)" + * The sys.path after activating the Mach site + * The sys.path after activating the command site + """ + + output = _run_activation_script( + context, + source, + site_name, + invoking_python or context.original_python_path, + check=True, + ).stdout + # Filter to the last line, which will have our nested list that we want to + # parse. This will avoid unrelated output, such as from virtualenv creation + output = output.splitlines()[-1] + return [ + [Path(path) for path in _filter_pydev_from_paths(paths)] + for paths in ast.literal_eval(output) + ] + + +def _assert_original_python_sys_path(context: ActivationContext, original: List[Path]): + # Assert that initial sys.path (prior to any activations) matches expectations. + assert original == [ + Path(__file__).parent, + *context.required_moz_init_sys_paths, + *context.stdlib_paths, + *context.system_paths, + ] + + +def _sys_path_of_virtualenv(virtualenv: PythonVirtualenv) -> List[Path]: + output = subprocess.run( + [virtualenv.python_path, "-c", "import sys; print(sys.path)"], + stdout=subprocess.PIPE, + universal_newlines=True, + env={ + # Needed for python to initialize properly + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + }, + check=True, + ).stdout + return [Path(path) for path in _filter_pydev_from_paths(ast.literal_eval(output))] + + +def _filter_pydev_from_paths(paths: List[str]) -> List[str]: + # Filter out injected "pydev" debugging tool if running within a JetBrains + # debugging context. + return [path for path in paths if "pydev" not in path and "JetBrains" not in path] + + +def _create_venv_system_python( + work_dir: Path, invoking_python: str +) -> PythonVirtualenv: + virtualenv = PythonVirtualenv(str(work_dir / "system_python")) + subprocess.run( + [ + invoking_python, + "-m", + "venv", + virtualenv.prefix, + "--without-pip", + ], + check=True, + ) + return virtualenv + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mach/mach/test/test_site_compatibility.py b/python/mach/mach/test/test_site_compatibility.py new file mode 100644 index 0000000000..4c1b6d5efa --- /dev/null +++ b/python/mach/mach/test/test_site_compatibility.py @@ -0,0 +1,189 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import mozunit +from buildconfig import topsrcdir + +from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv + + +def _resolve_command_site_names(): + site_names = [] + for child in (Path(topsrcdir) / "python" / "sites").iterdir(): + if not child.is_file(): + continue + + if child.suffix != ".txt": + continue + + if child.name == "mach.txt": + continue + + site_names.append(child.stem) + return site_names + + +def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env): + """Convert from parsed requirements object to pip-consumable format""" + requirements_path = Path(topsrcdir) / "python" / "sites" / f"{site_name}.txt" + requirements = MachEnvRequirements.from_requirements_definition( + topsrcdir, False, not is_mach_or_build_env, requirements_path + ) + + lines = [] + for pypi in ( + requirements.pypi_requirements + requirements.pypi_optional_requirements + ): + lines.append(str(pypi.requirement)) + + for vendored in requirements.vendored_requirements: + lines.append(str(cache.package_for_vendor_dir(Path(vendored.path)))) + + for pth in requirements.pth_requirements: + path = Path(pth.path) + + if "third_party" not in (p.name for p in path.parents): + continue + + for child in path.iterdir(): + if child.name.endswith(".dist-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has a ' + '".dist-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + if child.name.endswith(".egg-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has an ' + '".egg-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + + return "\n".join(lines) + + +class PackageCache: + def __init__(self, storage_dir: Path): + self._cache = {} + self._storage_dir = storage_dir + + def package_for_vendor_dir(self, vendor_path: Path): + if vendor_path in self._cache: + return self._cache[vendor_path] + + if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))): + # This vendored package is not a wheel. It may be a source package (with + # a setup.py), or just some Python code that was manually copied into the + # tree. If it's a source package, the setup.py file may be up a few levels + # from the referenced Python module path. + package_dir = vendor_path + while True: + if (package_dir / "setup.py").exists(): + break + elif package_dir.parent == package_dir: + raise Exception( + f'Package "{vendor_path}" is not a wheel and does not have a ' + 'setup.py file. Perhaps it should be "pth:" instead of ' + '"vendored:"?' + ) + package_dir = package_dir.parent + + self._cache[vendor_path] = package_dir + return package_dir + + # Pip requires that wheels have a version number in their name, even if + # it ignores it. We should parse out the version and put it in here + # so that failure debugging is easier, but that's non-trivial work. + # So, this "0" satisfies pip's naming requirement while being relatively + # obvious that it's a placeholder. + output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any" + shutil.make_archive(str(output_path), "zip", vendor_path) + + whl_path = output_path.parent / (output_path.name + ".whl") + (output_path.parent / (output_path.name + ".zip")).rename(whl_path) + self._cache[vendor_path] = whl_path + + return whl_path + + +def test_sites_compatible(tmpdir: str): + command_site_names = _resolve_command_site_names() + work_dir = Path(tmpdir) + cache = PackageCache(work_dir) + mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) + + # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + "--without-pip", + virtualenv.prefix, + ] + ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) + + for name in command_site_names: + print(f'Checking compatibility of "{name}" site') + command_requirements = _requirement_definition_to_pip_format( + name, cache, name == "build" + ) + with open(work_dir / "requirements.txt", "w") as requirements_txt: + requirements_txt.write(mach_requirements) + requirements_txt.write("\n") + requirements_txt.write(command_requirements) + + # Attempt to install combined set of dependencies (global Mach + current + # command) + proc = subprocess.run( + [ + virtualenv.python_path, + "-m", + "pip", + "install", + "-r", + str(work_dir / "requirements.txt"), + ], + cwd=topsrcdir, + ) + if proc.returncode != 0: + print( + dedent( + f""" + Error: The '{name}' site contains dependencies that are not + compatible with the 'mach' site. Check the following files for + any conflicting packages mentioned in the prior error message: + + python/sites/mach.txt + python/sites/{name}.txt + """ + ) + ) + assert False + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mach/mach/test/zero_microseconds.py b/python/mach/mach/test/zero_microseconds.py new file mode 100644 index 0000000000..b1d523071f --- /dev/null +++ b/python/mach/mach/test/zero_microseconds.py @@ -0,0 +1,12 @@ +# This code is loaded via `mach python --exec-file`, so it runs in the scope of +# the `mach python` command. +old = self._mach_context.post_dispatch_handler # noqa: F821 + + +def handler(context, handler, instance, result, start_time, end_time, depth, args): + global old + # Round off sub-second precision. + old(context, handler, instance, result, int(start_time), end_time, depth, args) + + +self._mach_context.post_dispatch_handler = handler # noqa: F821 diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py new file mode 100644 index 0000000000..4ed303cf3b --- /dev/null +++ b/python/mach/mach/util.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import os +import sys +from pathlib import Path, PurePosixPath +from typing import Optional, Union + + +class UserError(Exception): + """Represents an error caused by something the user did wrong rather than + an internal `mach` failure. Exceptions that are subclasses of this class + will not be reported as failures to Sentry. + """ + + +def setenv(key, value): + """Compatibility shim to ensure the proper string type is used with + os.environ for the version of Python being used. + """ + from six import text_type + + encoding = "mbcs" if sys.platform == "win32" else "utf-8" + + if sys.version_info[0] == 2: + if isinstance(key, text_type): + key = key.encode(encoding) + if isinstance(value, text_type): + value = value.encode(encoding) + else: + if isinstance(key, bytes): + key = key.decode(encoding) + if isinstance(value, bytes): + value = value.decode(encoding) + + os.environ[key] = value + + +def get_state_dir( + specific_to_topsrcdir=False, topsrcdir: Optional[Union[str, Path]] = None +): + """Obtain path to a directory to hold state. + + Args: + specific_to_topsrcdir (bool): If True, return a state dir specific to the current + srcdir instead of the global state dir (default: False) + + Returns: + A path to the state dir (str) + """ + state_dir = Path(os.environ.get("MOZBUILD_STATE_PATH", Path.home() / ".mozbuild")) + if not specific_to_topsrcdir: + return str(state_dir) + + if not topsrcdir: + # Only import MozbuildObject if topsrcdir isn't provided. This is to cover + # the Mach initialization stage, where "mozbuild" isn't in the import scope. + from mozbuild.base import MozbuildObject + + topsrcdir = Path( + MozbuildObject.from_environment(cwd=str(Path(__file__).parent)).topsrcdir + ) + + # Ensure that the topsrcdir is a consistent string before hashing it. + topsrcdir = Path(topsrcdir).resolve() + + # Shortening to 12 characters makes these directories a bit more manageable + # in a terminal and is more than good enough for this purpose. + srcdir_hash = hashlib.sha256(str(topsrcdir).encode("utf-8")).hexdigest()[:12] + + state_dir = state_dir / "srcdirs" / f"{topsrcdir.name}-{srcdir_hash}" + + if not state_dir.is_dir(): + # We create the srcdir here rather than 'mach_initialize.py' so direct + # consumers of this function don't create the directory inconsistently. + print(f"Creating local state directory: {state_dir}") + state_dir.mkdir(mode=0o770, parents=True) + # Save the topsrcdir that this state dir corresponds to so we can clean + # it up in the event its srcdir was deleted. + with (state_dir / "topsrcdir.txt").open(mode="w") as fh: + fh.write(str(topsrcdir)) + + return str(state_dir) + + +def win_to_msys_path(path: Path): + """Convert a windows-style path to msys-style.""" + drive, path = os.path.splitdrive(path) + path = "/".join(path.split("\\")) + if drive: + if path[0] == "/": + path = path[1:] + path = f"/{drive[:-1]}/{path}" + return PurePosixPath(path) + + +def to_optional_path(path: Optional[Path]): + if path: + return Path(path) + else: + return None + + +def to_optional_str(path: Optional[Path]): + if path: + return str(path) + else: + return None diff --git a/python/mach/metrics.yaml b/python/mach/metrics.yaml new file mode 100644 index 0000000000..16b2aa2877 --- /dev/null +++ b/python/mach/metrics.yaml @@ -0,0 +1,206 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# If this file is changed, update the generated docs: +# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs + +# Adding a new metric? We have docs for that! +# https://mozilla.github.io/glean/book/user/metrics/adding-new-metrics.html +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 + +mach: + command: + type: string + description: > + The name of the mach command that was invoked, such as "build", + "doc", or "try". + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + argv: + type: string_list + description: > + Parameters provided to mach. Absolute paths are sanitized to be relative + to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", + or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" + would be replaced with "$topsrcdir/python/mozbuild". + If a valid replacement base path cannot be found, the path is replaced + with "<path omitted>". + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + success: + type: boolean + description: True if the mach invocation succeeded. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + duration: + type: timespan + description: How long it took for the command to complete. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + +mach.system: + cpu_brand: + type: string + description: CPU brand string from CPUID. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + distro: + type: string + description: The name of the operating system distribution. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + distro_version: + type: string + description: The high-level OS version. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1655845#c3 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + logical_cores: + type: counter + description: Number of logical CPU cores present. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + physical_cores: + type: counter + description: Number of physical CPU cores present. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + memory: + type: memory_distribution + memory_unit: gigabyte + description: Amount of system memory. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com + expires: never + send_in_pings: + - usage + vscode_terminal: + type: boolean + description: True if the current terminal is opened via Visual Studio Code. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172#c4 + notification_emails: + - build-telemetry@mozilla.com + - andi@mozilla.com + expires: never + send_in_pings: + - usage + ssh_connection: + type: boolean + description: True if the current shell is a remote SSH connection. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1702172#c4 + notification_emails: + - build-telemetry@mozilla.com + - andi@mozilla.com + expires: never + send_in_pings: + - usage + vscode_running: + type: boolean + description: True if there is an instance of vscode running. + lifetime: application + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1717801 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1717801#c1 + notification_emails: + - build-telemetry@mozilla.com + - andi@mozilla.com + expires: never + send_in_pings: + - usage diff --git a/python/mach/pings.yaml b/python/mach/pings.yaml new file mode 100644 index 0000000000..c975437237 --- /dev/null +++ b/python/mach/pings.yaml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# If this file is changed, update the generated docs: +# https://firefox-source-docs.mozilla.org/mach/telemetry.html#updating-generated-metrics-docs +--- +$schema: moz://mozilla.org/schemas/glean/pings/2-0-0 + +usage: + description: > + Sent when the mach invocation is completed (regardless of result). + Contains information about the mach invocation that was made, its result, + and some details about the current environment and hardware. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 + include_client_id: true + notification_emails: + - build-telemetry@mozilla.com + - mhentges@mozilla.com diff --git a/python/mach/setup.cfg b/python/mach/setup.cfg new file mode 100644 index 0000000000..3c6e79cf31 --- /dev/null +++ b/python/mach/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/python/mach/setup.py b/python/mach/setup.py new file mode 100644 index 0000000000..80426b6e00 --- /dev/null +++ b/python/mach/setup.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + + +VERSION = "1.0.0" +HERE = os.path.dirname(__file__) +README = open(os.path.join(HERE, "README.rst")).read() + +setup( + name="mach", + description="Generic command line command dispatching framework.", + long_description=README, + license="MPL 2.0", + author="Gregory Szorc", + author_email="gregory.szorc@gmail.com", + url="https://developer.mozilla.org/en-US/docs/Developer_Guide/mach", + packages=["mach", "mach.mixin"], + version=VERSION, + classifiers=[ + "Environment :: Console", + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Natural Language :: English", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + ], + install_requires=[ + "blessed", + "mozfile", + "mozprocess", + "six", + ], + tests_require=["mock"], +) diff --git a/python/mach_commands.py b/python/mach_commands.py new file mode 100644 index 0000000000..d4f1f67efe --- /dev/null +++ b/python/mach_commands.py @@ -0,0 +1,366 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import os +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed, thread +from multiprocessing import cpu_count + +import mozinfo +from mach.decorators import Command, CommandArgument +from manifestparser import TestManifest +from manifestparser import filters as mpf +from mozfile import which +from tqdm import tqdm + + +@Command("python", category="devenv", description="Run Python.") +@CommandArgument( + "--exec-file", default=None, help="Execute this Python file using `exec`" +) +@CommandArgument( + "--ipython", + action="store_true", + default=False, + help="Use ipython instead of the default Python REPL.", +) +@CommandArgument( + "--virtualenv", + default=None, + help="Prepare and use the virtualenv with the provided name. If not specified, " + "then the Mach context is used instead.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def python( + command_context, + exec_file, + ipython, + virtualenv, + args, +): + # Avoid logging the command + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + + # Note: subprocess requires native strings in os.environ on Windows. + append_env = {"PYTHONDONTWRITEBYTECODE": str("1")} + + if virtualenv: + command_context._virtualenv_name = virtualenv + + if exec_file: + command_context.activate_virtualenv() + exec(open(exec_file).read()) + return 0 + + if ipython: + if virtualenv: + command_context.virtualenv_manager.ensure() + python_path = which( + "ipython", path=command_context.virtualenv_manager.bin_path + ) + if not python_path: + raise Exception( + "--ipython was specified, but the provided " + '--virtualenv doesn\'t have "ipython" installed.' + ) + else: + command_context._virtualenv_name = "ipython" + command_context.virtualenv_manager.ensure() + python_path = which( + "ipython", path=command_context.virtualenv_manager.bin_path + ) + else: + command_context.virtualenv_manager.ensure() + python_path = command_context.virtualenv_manager.python_path + + return command_context.run_process( + [python_path] + args, + pass_thru=True, # Allow user to run Python interactively. + ensure_exit_code=False, # Don't throw on non-zero exit code. + python_unbuffered=False, # Leave input buffered. + append_env=append_env, + ) + + +@Command( + "python-test", + category="testing", + virtualenv_name="python-test", + description="Run Python unit tests with pytest.", +) +@CommandArgument( + "-v", "--verbose", default=False, action="store_true", help="Verbose output." +) +@CommandArgument( + "-j", + "--jobs", + default=None, + type=int, + help="Number of concurrent jobs to run. Default is the number of CPUs " + "in the system.", +) +@CommandArgument( + "-x", + "--exitfirst", + default=False, + action="store_true", + help="Runs all tests sequentially and breaks at the first failure.", +) +@CommandArgument( + "--subsuite", + default=None, + help=( + "Python subsuite to run. If not specified, all subsuites are run. " + "Use the string `default` to only run tests without a subsuite." + ), +) +@CommandArgument( + "tests", + nargs="*", + metavar="TEST", + help=( + "Tests to run. Each test can be a single file or a directory. " + "Default test resolution relies on PYTHON_UNITTEST_MANIFESTS." + ), +) +@CommandArgument( + "extra", + nargs=argparse.REMAINDER, + metavar="PYTEST ARGS", + help=( + "Arguments that aren't recognized by mach. These will be " + "passed as it is to pytest" + ), +) +def python_test(command_context, *args, **kwargs): + try: + tempdir = str(tempfile.mkdtemp(suffix="-python-test")) + os.environ["PYTHON_TEST_TMP"] = tempdir + return run_python_tests(command_context, *args, **kwargs) + finally: + import mozfile + + mozfile.remove(tempdir) + + +def run_python_tests( + command_context, + tests=None, + test_objects=None, + subsuite=None, + verbose=False, + jobs=None, + exitfirst=False, + extra=None, + **kwargs, +): + if test_objects is None: + from moztest.resolve import TestResolver + + resolver = command_context._spawn(TestResolver) + # If we were given test paths, try to find tests matching them. + test_objects = resolver.resolve_tests(paths=tests, flavor="python") + else: + # We've received test_objects from |mach test|. We need to ignore + # the subsuite because python-tests don't use this key like other + # harnesses do and |mach test| doesn't realize this. + subsuite = None + + mp = TestManifest() + mp.tests.extend(test_objects) + + filters = [] + if subsuite == "default": + filters.append(mpf.subsuite(None)) + elif subsuite: + filters.append(mpf.subsuite(subsuite)) + + tests = mp.active_tests(filters=filters, disabled=False, python=3, **mozinfo.info) + + if not tests: + submsg = "for subsuite '{}' ".format(subsuite) if subsuite else "" + message = ( + "TEST-UNEXPECTED-FAIL | No tests collected " + + "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg) + ) + command_context.log(logging.WARN, "python-test", {}, message) + return 1 + + parallel = [] + sequential = [] + os.environ.setdefault("PYTEST_ADDOPTS", "") + + if extra: + os.environ["PYTEST_ADDOPTS"] += " " + " ".join(extra) + + installed_requirements = set() + for test in tests: + if ( + test.get("requirements") + and test["requirements"] not in installed_requirements + ): + command_context.virtualenv_manager.install_pip_requirements( + test["requirements"], quiet=True + ) + installed_requirements.add(test["requirements"]) + + if exitfirst: + sequential = tests + os.environ["PYTEST_ADDOPTS"] += " -x" + else: + for test in tests: + if test.get("sequential"): + sequential.append(test) + else: + parallel.append(test) + + jobs = jobs or cpu_count() + + return_code = 0 + failure_output = [] + + def on_test_finished(result): + output, ret, test_path = result + + if ret: + # Log the output of failed tests at the end so it's easy to find. + failure_output.extend(output) + + if not return_code: + command_context.log( + logging.ERROR, + "python-test", + {"test_path": test_path, "ret": ret}, + "Setting retcode to {ret} from {test_path}", + ) + else: + for line in output: + command_context.log( + logging.INFO, "python-test", {"line": line.rstrip()}, "{line}" + ) + + return return_code or ret + + with tqdm( + total=(len(parallel) + len(sequential)), + unit="Test", + desc="Tests Completed", + initial=0, + ) as progress_bar: + try: + with ThreadPoolExecutor(max_workers=jobs) as executor: + futures = [] + + for test in parallel: + command_context.log( + logging.DEBUG, + "python-test", + {"line": f"Launching thread for test {test['file_relpath']}"}, + "{line}", + ) + futures.append( + executor.submit( + _run_python_test, command_context, test, jobs, verbose + ) + ) + + try: + for future in as_completed(futures): + progress_bar.clear() + return_code = on_test_finished(future.result()) + progress_bar.update(1) + except KeyboardInterrupt: + # Hack to force stop currently running threads. + # https://gist.github.com/clchiou/f2608cbe54403edb0b13 + executor._threads.clear() + thread._threads_queues.clear() + raise + + for test in sequential: + test_result = _run_python_test(command_context, test, jobs, verbose) + + progress_bar.clear() + return_code = on_test_finished(test_result) + if return_code and exitfirst: + break + + progress_bar.update(1) + finally: + progress_bar.clear() + # Now log all failures (even if there was a KeyboardInterrupt or other exception). + for line in failure_output: + command_context.log( + logging.INFO, "python-test", {"line": line.rstrip()}, "{line}" + ) + + command_context.log( + logging.INFO, + "python-test", + {"return_code": return_code}, + "Return code from mach python-test: {return_code}", + ) + + return return_code + + +def _run_python_test(command_context, test, jobs, verbose): + output = [] + + def _log(line): + # Buffer messages if more than one worker to avoid interleaving + if jobs > 1: + output.append(line) + else: + command_context.log( + logging.INFO, "python-test", {"line": line.rstrip()}, "{line}" + ) + + _log(test["path"]) + python = command_context.virtualenv_manager.python_path + cmd = [python, test["path"]] + env = os.environ.copy() + env["PYTHONDONTWRITEBYTECODE"] = "1" + + result = subprocess.run( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + encoding="UTF-8", + ) + + return_code = result.returncode + + file_displayed_test = False + + for line in result.stdout.split(os.linesep): + if not file_displayed_test: + test_ran = "Ran" in line or "collected" in line or line.startswith("TEST-") + if test_ran: + file_displayed_test = True + + # Hack to make sure treeherder highlights pytest failures + if "FAILED" in line.rsplit(" ", 1)[-1]: + line = line.replace("FAILED", "TEST-UNEXPECTED-FAIL") + + _log(line) + + if not file_displayed_test: + return_code = 1 + _log( + "TEST-UNEXPECTED-FAIL | No test output (missing mozunit.main() " + "call?): {}".format(test["path"]) + ) + + if verbose: + if return_code != 0: + _log("Test failed: {}".format(test["path"])) + else: + _log("Test passed: {}".format(test["path"])) + + return output, return_code, test["path"] |