diff options
Diffstat (limited to 'comm/python/mutlh')
| -rw-r--r-- | comm/python/mutlh/README.md | 56 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/__init__.py | 0 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/decorators.py | 197 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/site.py | 120 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/__init__.py | 0 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/conftest.py | 11 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/python.ini | 11 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_decorators.py | 82 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_site.py | 33 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_site_compatibility.py | 194 |
10 files changed, 704 insertions, 0 deletions
diff --git a/comm/python/mutlh/README.md b/comm/python/mutlh/README.md new file mode 100644 index 0000000000..0c3d3c4387 --- /dev/null +++ b/comm/python/mutlh/README.md @@ -0,0 +1,56 @@ +# mutlh + +Mutlh (Klingon for *construct, assemble, put together*) is an extension of +Mozilla's Mach to meet the needs of Thunderbird developers. + +### Why you might need this + +When implementing a `mach` command in comm-central, you may need to utilize +some of the Python code in the repository. Often, `mach` commands have difficulty +importing these modules as they're not in sys.path usually. + +### Use case: mach command needs to import a library not on sys.path + +- In your `mach_commands.py` file, instead of importing from `mach.decorators`, + import from `mutlh.decorators`. +- Implement your command as usual. `@Command`, `@CommandArgument`, `@SubCommand`, + and `@CommandArgumentGroup` are available and work just like `mach.decorators` + equivalents. +- By default, the "tb_common" site is used for MutlhCommands. + +```python +from mutlh.decorators import Command, CommandArgument + +@Command( + "tb-add-missing-ftls", + category="thunderbird", + description="Add missing FTL files after l10n merge.", +) +@CommandArgument( + "--merge", + type=Path, + help="Merge path base", +) +@CommandArgument( + "locale", + type=str, + help="Locale code", +) +def tb_add_missing_ftls(command_context, merge, locale): + """implementation""" +``` + +- The default "tb_common" virtualenv can be overridden by passing `virtualenv_name` + to `@Command`. + +```python +@Command( + "crazytb", + category="thunderbird", + description="Something insane", + virtualenv_name="crazyenv" +) +def crazytb(command_context, *args, **kwargs): + command_context.activate_virtualenv() + """Do stuff""" +``` diff --git a/comm/python/mutlh/mutlh/__init__.py b/comm/python/mutlh/mutlh/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/comm/python/mutlh/mutlh/__init__.py diff --git a/comm/python/mutlh/mutlh/decorators.py b/comm/python/mutlh/mutlh/decorators.py new file mode 100644 index 0000000000..9a5f537009 --- /dev/null +++ b/comm/python/mutlh/mutlh/decorators.py @@ -0,0 +1,197 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import argparse +import os + +from mach.decorators import _MachCommand +from mozbuild.base import MachCommandBase + + +class MutlhCommandBase(MachCommandBase): + @property + def virtualenv_manager(self): + from mozboot.util import get_state_dir + + from .site import MutlhCommandSiteManager + + if self._virtualenv_manager is None: + self._virtualenv_manager = MutlhCommandSiteManager.from_environment( + self.topsrcdir, + lambda: get_state_dir(specific_to_topsrcdir=True, topsrcdir=self.topsrcdir), + self._virtualenv_name, + os.path.join(self.topobjdir, "_virtualenvs"), + ) + + return self._virtualenv_manager + + +class _MutlhCommand(_MachCommand): + def create_instance(self, context, virtualenv_name): + metrics = None + if self.metrics_path: + metrics = context.telemetry.metrics(self.metrics_path) + + # This ensures the resulting class is defined inside `mach` so that logging + # works as expected, and has a meaningful name + subclass = type(self.name, (MutlhCommandBase,), {}) + + if virtualenv_name is None: + virtualenv_name = "tb_common" + + return subclass( + context, + virtualenv_name=virtualenv_name, + metrics=metrics, + no_auto_log=self.no_auto_log, + ) + + +class Command(object): + def __init__(self, name, metrics_path=None, **kwargs): + self._mach_command = _MutlhCommand(name=name, **kwargs) + self._mach_command.metrics_path = metrics_path + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MutlhCommand() + + func._mach_command |= self._mach_command + func._mach_command.register(func) + + return func + + +class SubCommand(object): + global_order = 0 + + def __init__( + self, + command, + subcommand, + description=None, + parser=None, + metrics_path=None, + virtualenv_name=None, + ): + self._mach_command = _MutlhCommand( + name=command, + subcommand=subcommand, + description=description, + parser=parser, + virtualenv_name=virtualenv_name, + ) + self._mach_command.decl_order = SubCommand.global_order + SubCommand.global_order += 1 + + self._mach_command.metrics_path = metrics_path + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MutlhCommand() + + func._mach_command |= self._mach_command + func._mach_command.register(func) + + return func + + +class CommandArgument(object): + def __init__(self, *args, **kwargs): + if kwargs.get("nargs") == argparse.REMAINDER: + # These are the assertions we make in dispatcher.py about + # those types of CommandArguments. + assert len(args) == 1 + assert all(k in ("default", "nargs", "help", "group", "metavar") for k in kwargs) + self._command_args = (args, kwargs) + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MutlhCommand() + + func._mach_command.arguments.insert(0, self._command_args) + + return func + + +class CommandArgumentGroup(object): + def __init__(self, group_name): + self._group_name = group_name + + def __call__(self, func): + if not hasattr(func, "_mach_command"): + func._mach_command = _MutlhCommand() + + func._mach_command.argument_group_names.insert(0, self._group_name) + + return func + + +def mach2MutlhCommand(cmd: str, new_func=None, **replacekws): + """ + Change a registered _MachCommand to a _MutlhCommand + + :param str cmd: The name of the existing command + :param function new_func: New implementation function + :param dict replacekws: keyword arguments to replace + :return _MutlhCommand: replacement + """ + from mach.registrar import Registrar + + def get_mach_command(cmd_name): + mach_cmd = Registrar.command_handlers.get(cmd_name) + if mach_cmd: + del Registrar.command_handlers[cmd_name] + return mach_cmd + raise Exception(f"{cmd_name} unknown!") + + mach_cmd = get_mach_command(cmd) + + if mach_cmd.subcommand_handlers: + raise Exception("Commands with SubCommands not implemented!") + + if "parser" in replacekws: + replacekws["_parser"] = replacekws["parser"] + del replacekws["parser"] + + arg_names = ( + "name", + "subcommand", + "category", + "description", + "conditions", + "_parser", + "virtualenv_name", + "ok_if_tests_disabled", + "order", + "no_auto_log", + ) + kwargs = dict([(k, getattr(cmd, k)) for k in arg_names]) + kwargs.update(dict([(k, v) for k, v in replacekws.items() if k in arg_names])) + if "_parser" in kwargs: + kwargs["parser"] = kwargs["_parser"] + del kwargs["_parser"] + + mutlh_cmd = _MutlhCommand(**kwargs) + post_args = ( + "arguments", + "argument_group_names", + "metrics_path", + "subcommand_handlers", + "decl_order", + ) + for arg in post_args: + value = replacekws.get(arg, getattr(mach_cmd, arg)) + setattr(mutlh_cmd, arg, value) + + if new_func is None: + new_func = mach_cmd.func + delattr(new_func, "_mach_command") + if not hasattr(new_func, "_mach_command"): + new_func._mach_command = _MutlhCommand() + + new_func._mach_command |= mutlh_cmd + mutlh_cmd.register(new_func) + + return mutlh_cmd diff --git a/comm/python/mutlh/mutlh/site.py b/comm/python/mutlh/mutlh/site.py new file mode 100644 index 0000000000..ab93e874e9 --- /dev/null +++ b/comm/python/mutlh/mutlh/site.py @@ -0,0 +1,120 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import functools +import os +from typing import Callable, Optional + +from mach.requirements import MachEnvRequirements, UnexpectedFlexibleRequirementException +from mach.site import ( + PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS, + CommandSiteManager, + MozSiteMetadata, + SitePackagesSource, + _mach_virtualenv_root, +) + + +class SiteNotFoundException(Exception): + def __init__(self, site_name, manifest_paths): + self.site_name = site_name + self.manifest_paths = manifest_paths + self.args = (site_name, manifest_paths) + + +@functools.lru_cache(maxsize=None) +def find_manifest(topsrcdir, site_name): + manifest_paths = ( + os.path.join(topsrcdir, "comm", "python", "sites", f"{site_name}.txt"), + os.path.join(topsrcdir, "python", "sites", f"{site_name}.txt"), + ) + + for check_path in manifest_paths: + if os.path.exists(check_path): + return check_path + + raise SiteNotFoundException(site_name, manifest_paths) + + +@functools.lru_cache(maxsize=None) +def resolve_requirements(topsrcdir, site_name): + try: + manifest_path = find_manifest(topsrcdir, site_name) + except SiteNotFoundException as e: + raise Exception( + f'The current command is using the "{e.site_name}" ' + "site. However, that site is missing its associated " + f"requirements definition file in one of the supported " + f"paths: {e.manifest_paths}." + ) + is_thunderbird = True + try: + return MachEnvRequirements.from_requirements_definition( + topsrcdir, + is_thunderbird, + site_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS, + manifest_path, + ) + except UnexpectedFlexibleRequirementException as e: + raise Exception( + f'The "{site_name}" site does not have all pypi packages pinned ' + f'in the format "package==version" (found "{e.raw_requirement}").\n' + f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} sites are " + "allowed to have unpinned packages." + ) + + +class MutlhCommandSiteManager(CommandSiteManager): + @classmethod + def from_environment( + cls, + topsrcdir: str, + get_state_dir: Callable[[], Optional[str]], + site_name: str, + command_virtualenvs_dir: str, + ): + """ + Args: + topsrcdir: The path to the Firefox repo + get_state_dir: A function that resolves the path to the checkout-scoped + state_dir, generally ~/.mozbuild/srcdirs/<checkout-based-dir>/ + site_name: The name of this site, such as "build" + command_virtualenvs_dir: The location under which this site's virtualenv + should be created + """ + active_metadata = MozSiteMetadata.from_runtime() + assert ( + active_metadata + ), "A Mach-managed site must be active before doing work with command sites" + + mach_site_packages_source = active_metadata.mach_site_packages_source + pip_restricted_site = site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS + if not pip_restricted_site and mach_site_packages_source == SitePackagesSource.SYSTEM: + # Sites that aren't pip-network-install-restricted are likely going to be + # incompatible with the system. Besides, this use case shouldn't exist, since + # using the system packages is supposed to only be needed to lower risk of + # important processes like building Firefox. + raise Exception( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any ' + f"sites other than {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS}. The " + f'current attempted site is "{site_name}".' + ) + + mach_virtualenv_root = ( + _mach_virtualenv_root(get_state_dir()) + if mach_site_packages_source == SitePackagesSource.VENV + else None + ) + populate_virtualenv = ( + mach_site_packages_source == SitePackagesSource.VENV or not pip_restricted_site + ) + return cls( + topsrcdir, + mach_virtualenv_root, + os.path.join(command_virtualenvs_dir, site_name), + site_name, + active_metadata, + populate_virtualenv, + resolve_requirements(topsrcdir, site_name), + ) diff --git a/comm/python/mutlh/mutlh/test/__init__.py b/comm/python/mutlh/mutlh/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/__init__.py diff --git a/comm/python/mutlh/mutlh/test/conftest.py b/comm/python/mutlh/mutlh/test/conftest.py new file mode 100644 index 0000000000..6fcac64308 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/conftest.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import sys + +HERE = os.path.dirname(__file__) +EXT_PATH = os.path.abspath(os.path.join(HERE, "..", "..")) + +sys.path.insert(0, EXT_PATH) diff --git a/comm/python/mutlh/mutlh/test/python.ini b/comm/python/mutlh/mutlh/test/python.ini new file mode 100644 index 0000000000..3dafe825cd --- /dev/null +++ b/comm/python/mutlh/mutlh/test/python.ini @@ -0,0 +1,11 @@ +[DEFAULT] +subsuite = mutlh + +[test_decorators.py] +[test_site.py] +[test_site_compatibility.py] +# The Windows and Mac workers only use the internal PyPI mirror, +# which will be missing packages required for this test. +skip-if = + os == "win" + os == "mac" diff --git a/comm/python/mutlh/mutlh/test/test_decorators.py b/comm/python/mutlh/mutlh/test/test_decorators.py new file mode 100644 index 0000000000..7e27cd6383 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_decorators.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path +from unittest import mock +from unittest.mock import Mock, patch + +import conftest # noqa: F401 +import pytest +from mozunit import main + +import mach.decorators +import mach.registrar +from mach.requirements import MachEnvRequirements +from mach.site import MozSiteMetadata, SitePackagesSource +from mutlh.decorators import Command, CommandArgument, MutlhCommandBase +from mutlh.site import MutlhCommandSiteManager + + +@pytest.fixture +def registrar(monkeypatch): + test_registrar = mach.registrar.MachRegistrar() + test_registrar.register_category("testing", "Mach unittest", "Testing for mach decorators") + monkeypatch.setattr(mach.decorators, "Registrar", test_registrar) + return test_registrar + + +def test_register_command_with_argument(registrar): + inner_function = Mock() + context = Mock() + context.cwd = "." + + @Command("cmd_foo", category="testing") + @CommandArgument("--arg", default=None, help="Argument help.") + def run_foo(command_context, arg): + inner_function(arg) + + registrar.dispatch("cmd_foo", context, arg="argument") + + inner_function.assert_called_with("argument") + + +def test_register_command_sets_up_class_at_runtime(registrar): + inner_function = Mock() + + context = Mock() + context.cwd = "." + + # We test that the virtualenv is set up properly dynamically on + # the instance that actually runs the command. + @Command("cmd_foo", category="testing", virtualenv_name="env_foo") + def run_foo(command_context): + assert Path(command_context.virtualenv_manager.virtualenv_root).name == "env_foo" + inner_function("foo") + + @Command("cmd_bar", category="testing", virtualenv_name="env_bar") + def run_bar(command_context): + assert Path(command_context.virtualenv_manager.virtualenv_root).name == "env_bar" + inner_function("bar") + + def from_environment_patch(topsrcdir: str, state_dir: str, virtualenv_name, directory: str): + return MutlhCommandSiteManager( + "", + "", + virtualenv_name, + virtualenv_name, + MozSiteMetadata(0, "mach", SitePackagesSource.VENV, "", ""), + True, + MachEnvRequirements(), + ) + + with mock.patch.object(MutlhCommandSiteManager, "from_environment", from_environment_patch): + with patch.object(MutlhCommandBase, "activate_virtualenv"): + registrar.dispatch("cmd_foo", context) + inner_function.assert_called_with("foo") + registrar.dispatch("cmd_bar", context) + inner_function.assert_called_with("bar") + + +if __name__ == "__main__": + main() diff --git a/comm/python/mutlh/mutlh/test/test_site.py b/comm/python/mutlh/mutlh/test/test_site.py new file mode 100644 index 0000000000..f5c11cadd7 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_site.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from contextlib import nullcontext as does_not_raise + +import conftest # noqa: F401 +import mozunit +import pytest + +from buildconfig import topsrcdir +from mutlh.site import SiteNotFoundException, find_manifest + + +@pytest.mark.parametrize( + "site_name,expected", + [ + ("tb_common", does_not_raise("comm/python/sites/tb_common.txt")), + ("lint", does_not_raise("python/sites/lint.txt")), + ("not_a_real_site_name", pytest.raises(SiteNotFoundException)), + ], +) +def test_find_manifest(site_name, expected): + def get_path(result): + return os.path.relpath(result, topsrcdir) + + with expected: + assert get_path(find_manifest(topsrcdir, site_name)) == expected.enter_result + + +if __name__ == "__main__": + mozunit.main() diff --git a/comm/python/mutlh/mutlh/test/test_site_compatibility.py b/comm/python/mutlh/mutlh/test/test_site_compatibility.py new file mode 100644 index 0000000000..c316e54240 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_site_compatibility.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import mozunit + +from buildconfig import topsrcdir +from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv + +MUTLH_REQUIREMENTS_PATH = Path(topsrcdir) / "comm" / "python" / "sites" +MACH_REQUIREMENTS_PATH = Path(topsrcdir) / "python" / "sites" + + +def _resolve_command_site_names(): + site_names = [] + for child in MUTLH_REQUIREMENTS_PATH.iterdir(): + if not child.is_file(): + continue + + if child.suffix != ".txt": + continue + + if child.name == "mach.txt": + continue + + site_names.append(child.stem) + return site_names + + +def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env): + """Convert from parsed requirements object to pip-consumable format""" + if site_name == "mach": + requirements_path = MACH_REQUIREMENTS_PATH / f"{site_name}.txt" + else: + requirements_path = MUTLH_REQUIREMENTS_PATH / f"{site_name}.txt" + is_thunderbird = True + + requirements = MachEnvRequirements.from_requirements_definition( + topsrcdir, is_thunderbird, not is_mach_or_build_env, requirements_path + ) + + lines = [] + for pypi in requirements.pypi_requirements + requirements.pypi_optional_requirements: + lines.append(str(pypi.requirement)) + + for vendored in requirements.vendored_requirements: + lines.append(str(cache.package_for_vendor_dir(Path(vendored.path)))) + + for pth in requirements.pth_requirements: + path = Path(pth.path) + + if "third_party" not in (p.name for p in path.parents): + continue + + for child in path.iterdir(): + if child.name.endswith(".dist-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has a ' + '".dist-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + if child.name.endswith(".egg-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has an ' + '".egg-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + + return "\n".join(lines) + + +class PackageCache: + def __init__(self, storage_dir: Path): + self._cache = {} + self._storage_dir = storage_dir + + def package_for_vendor_dir(self, vendor_path: Path): + if vendor_path in self._cache: + return self._cache[vendor_path] + + if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))): + # This vendored package is not a wheel. It may be a source package (with + # a setup.py), or just some Python code that was manually copied into the + # tree. If it's a source package, the setup.py file may be up a few levels + # from the referenced Python module path. + package_dir = vendor_path + while True: + if (package_dir / "setup.py").exists(): + break + elif package_dir.parent == package_dir: + raise Exception( + f'Package "{vendor_path}" is not a wheel and does not have a ' + 'setup.py file. Perhaps it should be "pth:" instead of ' + '"vendored:"?' + ) + package_dir = package_dir.parent + + self._cache[vendor_path] = package_dir + return package_dir + + # Pip requires that wheels have a version number in their name, even if + # it ignores it. We should parse out the version and put it in here + # so that failure debugging is easier, but that's non-trivial work. + # So, this "0" satisfies pip's naming requirement while being relatively + # obvious that it's a placeholder. + output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any" + shutil.make_archive(str(output_path), "zip", vendor_path) + + whl_path = output_path.parent / (output_path.name + ".whl") + (output_path.parent / (output_path.name + ".zip")).rename(whl_path) + self._cache[vendor_path] = whl_path + + return whl_path + + +def test_sites_compatible(tmpdir: str): + command_site_names = _resolve_command_site_names() + work_dir = Path(tmpdir) + cache = PackageCache(work_dir) + mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) + + # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + "--without-pip", + virtualenv.prefix, + ] + ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) + + for name in command_site_names: + print(f'Checking compatibility of "{name}" site') + command_requirements = _requirement_definition_to_pip_format(name, cache, False) + with open(work_dir / "requirements.txt", "w") as requirements_txt: + requirements_txt.write(mach_requirements) + requirements_txt.write("\n") + requirements_txt.write(command_requirements) + + # Attempt to install combined set of dependencies (global Mach + current + # command) + proc = subprocess.run( + [ + virtualenv.python_path, + "-m", + "pip", + "install", + "-r", + str(work_dir / "requirements.txt"), + ], + cwd=topsrcdir, + ) + if proc.returncode != 0: + print( + dedent( + f""" + Error: The '{name}' site contains dependencies that are not + compatible with the 'mach' site. Check the following files for + any conflicting packages mentioned in the prior error message: + + python/sites/mach.txt + comm/python/sites/{name}.txt + """ + ) + ) + assert False + + +if __name__ == "__main__": + mozunit.main() |