1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import shutil
import subprocess
import sys
from pathlib import Path
from textwrap import dedent
import mozunit
from buildconfig import topsrcdir
from mach.requirements import MachEnvRequirements
from mach.site import PythonVirtualenv
MUTLH_REQUIREMENTS_PATH = Path(topsrcdir) / "comm" / "python" / "sites"
MACH_REQUIREMENTS_PATH = Path(topsrcdir) / "python" / "sites"
def _resolve_command_site_names():
site_names = []
for child in MUTLH_REQUIREMENTS_PATH.iterdir():
if not child.is_file():
continue
if child.suffix != ".txt":
continue
if child.name == "mach.txt":
continue
site_names.append(child.stem)
return site_names
def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env):
"""Convert from parsed requirements object to pip-consumable format"""
if site_name == "mach":
requirements_path = MACH_REQUIREMENTS_PATH / f"{site_name}.txt"
else:
requirements_path = MUTLH_REQUIREMENTS_PATH / f"{site_name}.txt"
is_thunderbird = True
requirements = MachEnvRequirements.from_requirements_definition(
topsrcdir, is_thunderbird, not is_mach_or_build_env, requirements_path
)
lines = []
for pypi in requirements.pypi_requirements + requirements.pypi_optional_requirements:
lines.append(str(pypi.requirement))
for vendored in requirements.vendored_requirements:
lines.append(str(cache.package_for_vendor_dir(Path(vendored.path))))
for pth in requirements.pth_requirements:
path = Path(pth.path)
if "third_party" not in (p.name for p in path.parents):
continue
for child in path.iterdir():
if child.name.endswith(".dist-info"):
raise Exception(
f'In {requirements_path}, the "pth:" pointing to "{path}" has a '
'".dist-info" file.\n'
'Perhaps it should change to start with "vendored:" instead of '
'"pth:".'
)
if child.name.endswith(".egg-info"):
raise Exception(
f'In {requirements_path}, the "pth:" pointing to "{path}" has an '
'".egg-info" file.\n'
'Perhaps it should change to start with "vendored:" instead of '
'"pth:".'
)
return "\n".join(lines)
class PackageCache:
def __init__(self, storage_dir: Path):
self._cache = {}
self._storage_dir = storage_dir
def package_for_vendor_dir(self, vendor_path: Path):
if vendor_path in self._cache:
return self._cache[vendor_path]
if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))):
# This vendored package is not a wheel. It may be a source package (with
# a setup.py), or just some Python code that was manually copied into the
# tree. If it's a source package, the setup.py file may be up a few levels
# from the referenced Python module path.
package_dir = vendor_path
while True:
if (package_dir / "setup.py").exists():
break
elif package_dir.parent == package_dir:
raise Exception(
f'Package "{vendor_path}" is not a wheel and does not have a '
'setup.py file. Perhaps it should be "pth:" instead of '
'"vendored:"?'
)
package_dir = package_dir.parent
self._cache[vendor_path] = package_dir
return package_dir
# Pip requires that wheels have a version number in their name, even if
# it ignores it. We should parse out the version and put it in here
# so that failure debugging is easier, but that's non-trivial work.
# So, this "0" satisfies pip's naming requirement while being relatively
# obvious that it's a placeholder.
output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any"
shutil.make_archive(str(output_path), "zip", vendor_path)
whl_path = output_path.parent / (output_path.name + ".whl")
(output_path.parent / (output_path.name + ".zip")).rename(whl_path)
self._cache[vendor_path] = whl_path
return whl_path
def test_sites_compatible(tmpdir: str):
command_site_names = _resolve_command_site_names()
work_dir = Path(tmpdir)
cache = PackageCache(work_dir)
mach_requirements = _requirement_definition_to_pip_format("mach", cache, True)
# Create virtualenv to try to install all dependencies into.
virtualenv = PythonVirtualenv(str(work_dir / "env"))
subprocess.check_call(
[
sys.executable,
"-m",
"venv",
"--without-pip",
virtualenv.prefix,
]
)
platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib")
third_party = Path(topsrcdir) / "third_party" / "python"
with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile:
pthfile.write(
"\n".join(
[
str(third_party / "pip"),
str(third_party / "wheel"),
str(third_party / "setuptools"),
]
)
)
for name in command_site_names:
print(f'Checking compatibility of "{name}" site')
command_requirements = _requirement_definition_to_pip_format(name, cache, False)
with open(work_dir / "requirements.txt", "w") as requirements_txt:
requirements_txt.write(mach_requirements)
requirements_txt.write("\n")
requirements_txt.write(command_requirements)
# Attempt to install combined set of dependencies (global Mach + current
# command)
proc = subprocess.run(
[
virtualenv.python_path,
"-m",
"pip",
"install",
"-r",
str(work_dir / "requirements.txt"),
],
cwd=topsrcdir,
)
if proc.returncode != 0:
print(
dedent(
f"""
Error: The '{name}' site contains dependencies that are not
compatible with the 'mach' site. Check the following files for
any conflicting packages mentioned in the prior error message:
python/sites/mach.txt
comm/python/sites/{name}.txt
"""
)
)
assert False
if __name__ == "__main__":
mozunit.main()
|
