diff options
Diffstat (limited to 'comm/python/mutlh/mutlh/test')
| -rw-r--r-- | comm/python/mutlh/mutlh/test/__init__.py | 0 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/conftest.py | 11 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/python.ini | 11 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_decorators.py | 82 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_site.py | 33 | ||||
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_site_compatibility.py | 194 |
6 files changed, 331 insertions, 0 deletions
diff --git a/comm/python/mutlh/mutlh/test/__init__.py b/comm/python/mutlh/mutlh/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/__init__.py diff --git a/comm/python/mutlh/mutlh/test/conftest.py b/comm/python/mutlh/mutlh/test/conftest.py new file mode 100644 index 0000000000..6fcac64308 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/conftest.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import sys + +HERE = os.path.dirname(__file__) +EXT_PATH = os.path.abspath(os.path.join(HERE, "..", "..")) + +sys.path.insert(0, EXT_PATH) diff --git a/comm/python/mutlh/mutlh/test/python.ini b/comm/python/mutlh/mutlh/test/python.ini new file mode 100644 index 0000000000..3dafe825cd --- /dev/null +++ b/comm/python/mutlh/mutlh/test/python.ini @@ -0,0 +1,11 @@ +[DEFAULT] +subsuite = mutlh + +[test_decorators.py] +[test_site.py] +[test_site_compatibility.py] +# The Windows and Mac workers only use the internal PyPI mirror, +# which will be missing packages required for this test. +skip-if = + os == "win" + os == "mac" diff --git a/comm/python/mutlh/mutlh/test/test_decorators.py b/comm/python/mutlh/mutlh/test/test_decorators.py new file mode 100644 index 0000000000..7e27cd6383 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_decorators.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path +from unittest import mock +from unittest.mock import Mock, patch + +import conftest # noqa: F401 +import pytest +from mozunit import main + +import mach.decorators +import mach.registrar +from mach.requirements import MachEnvRequirements +from mach.site import MozSiteMetadata, SitePackagesSource +from mutlh.decorators import Command, CommandArgument, MutlhCommandBase +from mutlh.site import MutlhCommandSiteManager + + +@pytest.fixture +def registrar(monkeypatch): + test_registrar = mach.registrar.MachRegistrar() + test_registrar.register_category("testing", "Mach unittest", "Testing for mach decorators") + monkeypatch.setattr(mach.decorators, "Registrar", test_registrar) + return test_registrar + + +def test_register_command_with_argument(registrar): + inner_function = Mock() + context = Mock() + context.cwd = "." + + @Command("cmd_foo", category="testing") + @CommandArgument("--arg", default=None, help="Argument help.") + def run_foo(command_context, arg): + inner_function(arg) + + registrar.dispatch("cmd_foo", context, arg="argument") + + inner_function.assert_called_with("argument") + + +def test_register_command_sets_up_class_at_runtime(registrar): + inner_function = Mock() + + context = Mock() + context.cwd = "." + + # We test that the virtualenv is set up properly dynamically on + # the instance that actually runs the command. + @Command("cmd_foo", category="testing", virtualenv_name="env_foo") + def run_foo(command_context): + assert Path(command_context.virtualenv_manager.virtualenv_root).name == "env_foo" + inner_function("foo") + + @Command("cmd_bar", category="testing", virtualenv_name="env_bar") + def run_bar(command_context): + assert Path(command_context.virtualenv_manager.virtualenv_root).name == "env_bar" + inner_function("bar") + + def from_environment_patch(topsrcdir: str, state_dir: str, virtualenv_name, directory: str): + return MutlhCommandSiteManager( + "", + "", + virtualenv_name, + virtualenv_name, + MozSiteMetadata(0, "mach", SitePackagesSource.VENV, "", ""), + True, + MachEnvRequirements(), + ) + + with mock.patch.object(MutlhCommandSiteManager, "from_environment", from_environment_patch): + with patch.object(MutlhCommandBase, "activate_virtualenv"): + registrar.dispatch("cmd_foo", context) + inner_function.assert_called_with("foo") + registrar.dispatch("cmd_bar", context) + inner_function.assert_called_with("bar") + + +if __name__ == "__main__": + main() diff --git a/comm/python/mutlh/mutlh/test/test_site.py b/comm/python/mutlh/mutlh/test/test_site.py new file mode 100644 index 0000000000..f5c11cadd7 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_site.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from contextlib import nullcontext as does_not_raise + +import conftest # noqa: F401 +import mozunit +import pytest + +from buildconfig import topsrcdir +from mutlh.site import SiteNotFoundException, find_manifest + + +@pytest.mark.parametrize( + "site_name,expected", + [ + ("tb_common", does_not_raise("comm/python/sites/tb_common.txt")), + ("lint", does_not_raise("python/sites/lint.txt")), + ("not_a_real_site_name", pytest.raises(SiteNotFoundException)), + ], +) +def test_find_manifest(site_name, expected): + def get_path(result): + return os.path.relpath(result, topsrcdir) + + with expected: + assert get_path(find_manifest(topsrcdir, site_name)) == expected.enter_result + + +if __name__ == "__main__": + mozunit.main() diff --git a/comm/python/mutlh/mutlh/test/test_site_compatibility.py b/comm/python/mutlh/mutlh/test/test_site_compatibility.py new file mode 100644 index 0000000000..c316e54240 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_site_compatibility.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import mozunit + +from buildconfig import topsrcdir +from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv + +MUTLH_REQUIREMENTS_PATH = Path(topsrcdir) / "comm" / "python" / "sites" +MACH_REQUIREMENTS_PATH = Path(topsrcdir) / "python" / "sites" + + +def _resolve_command_site_names(): + site_names = [] + for child in MUTLH_REQUIREMENTS_PATH.iterdir(): + if not child.is_file(): + continue + + if child.suffix != ".txt": + continue + + if child.name == "mach.txt": + continue + + site_names.append(child.stem) + return site_names + + +def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env): + """Convert from parsed requirements object to pip-consumable format""" + if site_name == "mach": + requirements_path = MACH_REQUIREMENTS_PATH / f"{site_name}.txt" + else: + requirements_path = MUTLH_REQUIREMENTS_PATH / f"{site_name}.txt" + is_thunderbird = True + + requirements = MachEnvRequirements.from_requirements_definition( + topsrcdir, is_thunderbird, not is_mach_or_build_env, requirements_path + ) + + lines = [] + for pypi in requirements.pypi_requirements + requirements.pypi_optional_requirements: + lines.append(str(pypi.requirement)) + + for vendored in requirements.vendored_requirements: + lines.append(str(cache.package_for_vendor_dir(Path(vendored.path)))) + + for pth in requirements.pth_requirements: + path = Path(pth.path) + + if "third_party" not in (p.name for p in path.parents): + continue + + for child in path.iterdir(): + if child.name.endswith(".dist-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has a ' + '".dist-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + if child.name.endswith(".egg-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has an ' + '".egg-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + + return "\n".join(lines) + + +class PackageCache: + def __init__(self, storage_dir: Path): + self._cache = {} + self._storage_dir = storage_dir + + def package_for_vendor_dir(self, vendor_path: Path): + if vendor_path in self._cache: + return self._cache[vendor_path] + + if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))): + # This vendored package is not a wheel. It may be a source package (with + # a setup.py), or just some Python code that was manually copied into the + # tree. If it's a source package, the setup.py file may be up a few levels + # from the referenced Python module path. + package_dir = vendor_path + while True: + if (package_dir / "setup.py").exists(): + break + elif package_dir.parent == package_dir: + raise Exception( + f'Package "{vendor_path}" is not a wheel and does not have a ' + 'setup.py file. Perhaps it should be "pth:" instead of ' + '"vendored:"?' + ) + package_dir = package_dir.parent + + self._cache[vendor_path] = package_dir + return package_dir + + # Pip requires that wheels have a version number in their name, even if + # it ignores it. We should parse out the version and put it in here + # so that failure debugging is easier, but that's non-trivial work. + # So, this "0" satisfies pip's naming requirement while being relatively + # obvious that it's a placeholder. + output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any" + shutil.make_archive(str(output_path), "zip", vendor_path) + + whl_path = output_path.parent / (output_path.name + ".whl") + (output_path.parent / (output_path.name + ".zip")).rename(whl_path) + self._cache[vendor_path] = whl_path + + return whl_path + + +def test_sites_compatible(tmpdir: str): + command_site_names = _resolve_command_site_names() + work_dir = Path(tmpdir) + cache = PackageCache(work_dir) + mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) + + # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + "--without-pip", + virtualenv.prefix, + ] + ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) + + for name in command_site_names: + print(f'Checking compatibility of "{name}" site') + command_requirements = _requirement_definition_to_pip_format(name, cache, False) + with open(work_dir / "requirements.txt", "w") as requirements_txt: + requirements_txt.write(mach_requirements) + requirements_txt.write("\n") + requirements_txt.write(command_requirements) + + # Attempt to install combined set of dependencies (global Mach + current + # command) + proc = subprocess.run( + [ + virtualenv.python_path, + "-m", + "pip", + "install", + "-r", + str(work_dir / "requirements.txt"), + ], + cwd=topsrcdir, + ) + if proc.returncode != 0: + print( + dedent( + f""" + Error: The '{name}' site contains dependencies that are not + compatible with the 'mach' site. Check the following files for + any conflicting packages mentioned in the prior error message: + + python/sites/mach.txt + comm/python/sites/{name}.txt + """ + ) + ) + assert False + + +if __name__ == "__main__": + mozunit.main() |