diff options
Diffstat (limited to 'comm/python/mutlh/mutlh/test/test_site_compatibility.py')
| -rw-r--r-- | comm/python/mutlh/mutlh/test/test_site_compatibility.py | 194 |
1 files changed, 194 insertions, 0 deletions
diff --git a/comm/python/mutlh/mutlh/test/test_site_compatibility.py b/comm/python/mutlh/mutlh/test/test_site_compatibility.py new file mode 100644 index 0000000000..c316e54240 --- /dev/null +++ b/comm/python/mutlh/mutlh/test/test_site_compatibility.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import mozunit + +from buildconfig import topsrcdir +from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv + +MUTLH_REQUIREMENTS_PATH = Path(topsrcdir) / "comm" / "python" / "sites" +MACH_REQUIREMENTS_PATH = Path(topsrcdir) / "python" / "sites" + + +def _resolve_command_site_names(): + site_names = [] + for child in MUTLH_REQUIREMENTS_PATH.iterdir(): + if not child.is_file(): + continue + + if child.suffix != ".txt": + continue + + if child.name == "mach.txt": + continue + + site_names.append(child.stem) + return site_names + + +def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env): + """Convert from parsed requirements object to pip-consumable format""" + if site_name == "mach": + requirements_path = MACH_REQUIREMENTS_PATH / f"{site_name}.txt" + else: + requirements_path = MUTLH_REQUIREMENTS_PATH / f"{site_name}.txt" + is_thunderbird = True + + requirements = MachEnvRequirements.from_requirements_definition( + topsrcdir, is_thunderbird, not is_mach_or_build_env, requirements_path + ) + + lines = [] + for pypi in requirements.pypi_requirements + requirements.pypi_optional_requirements: + lines.append(str(pypi.requirement)) + + for vendored in requirements.vendored_requirements: + lines.append(str(cache.package_for_vendor_dir(Path(vendored.path)))) + + for pth in requirements.pth_requirements: + path = Path(pth.path) + + if "third_party" not in (p.name for p in path.parents): + continue + + for child in path.iterdir(): + if child.name.endswith(".dist-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has a ' + '".dist-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + if child.name.endswith(".egg-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has an ' + '".egg-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + + return "\n".join(lines) + + +class PackageCache: + def __init__(self, storage_dir: Path): + self._cache = {} + self._storage_dir = storage_dir + + def package_for_vendor_dir(self, vendor_path: Path): + if vendor_path in self._cache: + return self._cache[vendor_path] + + if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))): + # This vendored package is not a wheel. It may be a source package (with + # a setup.py), or just some Python code that was manually copied into the + # tree. If it's a source package, the setup.py file may be up a few levels + # from the referenced Python module path. + package_dir = vendor_path + while True: + if (package_dir / "setup.py").exists(): + break + elif package_dir.parent == package_dir: + raise Exception( + f'Package "{vendor_path}" is not a wheel and does not have a ' + 'setup.py file. Perhaps it should be "pth:" instead of ' + '"vendored:"?' + ) + package_dir = package_dir.parent + + self._cache[vendor_path] = package_dir + return package_dir + + # Pip requires that wheels have a version number in their name, even if + # it ignores it. We should parse out the version and put it in here + # so that failure debugging is easier, but that's non-trivial work. + # So, this "0" satisfies pip's naming requirement while being relatively + # obvious that it's a placeholder. + output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any" + shutil.make_archive(str(output_path), "zip", vendor_path) + + whl_path = output_path.parent / (output_path.name + ".whl") + (output_path.parent / (output_path.name + ".zip")).rename(whl_path) + self._cache[vendor_path] = whl_path + + return whl_path + + +def test_sites_compatible(tmpdir: str): + command_site_names = _resolve_command_site_names() + work_dir = Path(tmpdir) + cache = PackageCache(work_dir) + mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) + + # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + "--without-pip", + virtualenv.prefix, + ] + ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) + + for name in command_site_names: + print(f'Checking compatibility of "{name}" site') + command_requirements = _requirement_definition_to_pip_format(name, cache, False) + with open(work_dir / "requirements.txt", "w") as requirements_txt: + requirements_txt.write(mach_requirements) + requirements_txt.write("\n") + requirements_txt.write(command_requirements) + + # Attempt to install combined set of dependencies (global Mach + current + # command) + proc = subprocess.run( + [ + virtualenv.python_path, + "-m", + "pip", + "install", + "-r", + str(work_dir / "requirements.txt"), + ], + cwd=topsrcdir, + ) + if proc.returncode != 0: + print( + dedent( + f""" + Error: The '{name}' site contains dependencies that are not + compatible with the 'mach' site. Check the following files for + any conflicting packages mentioned in the prior error message: + + python/sites/mach.txt + comm/python/sites/{name}.txt + """ + ) + ) + assert False + + +if __name__ == "__main__": + mozunit.main() |