1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
|
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
option(
env="MOZ_FETCHES_DIR",
nargs=1,
when="MOZ_AUTOMATION",
help="Directory containing fetched artifacts",
)
@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
def moz_fetches_dir(value):
if value:
return value[0]
@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
def bootstrap_default(vcs_checkout_type, is_nightly, automation):
if automation:
return False
# We only enable if building off a VCS checkout of central.
if is_nightly and vcs_checkout_type:
return True
option(
"--enable-bootstrap",
default=bootstrap_default,
help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
)
@depends(developer_options, "--enable-bootstrap", moz_fetches_dir)
def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir):
if moz_fetches_dir:
log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.")
return "prepend"
if bootstrap:
log.debug(
"Prioritizing mozbuild state dir in toolchain paths because "
"bootstrap mode is enabled."
)
return "prepend"
if developer_options:
log.debug(
"Prioritizing mozbuild state dir in toolchain paths because "
"you are not building in release mode."
)
return "prepend"
log.debug(
"Prioritizing system over mozbuild state dir in "
"toolchain paths because you are building in "
"release mode."
)
return "append"
toolchains_base_dir = moz_fetches_dir | mozbuild_state_path
@dependable
@imports("os")
@imports(_from="os", _import="environ")
def original_path():
return environ["PATH"].split(os.pathsep)
@depends(host, when="--enable-bootstrap")
@imports("os")
@imports("traceback")
@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_toolchain_tasks(host):
prefix = {
("x86_64", "GNU", "Linux"): "linux64",
("x86_64", "OSX", "Darwin"): "macosx64",
("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
("x86_64", "WINNT", "WINNT"): "win64",
("aarch64", "WINNT", "WINNT"): "win64-aarch64",
}.get((host.cpu, host.os, host.kernel))
try:
tasks = toolchain_task_definitions()
except Exception as e:
message = traceback.format_exc()
log.warning(str(e))
log.debug(message)
return None
# We only want to use toolchains annotated with "local-toolchain". We also limit the
# amount of data to what we use, so that trace logs can be more useful.
tasks = {
k: {
"index": t.optimization["index-search"],
"artifact": t.attributes["toolchain-artifact"],
}
for k, t in tasks.items()
if t.attributes.get("local-toolchain") and "index-search" in t.optimization
}
return namespace(prefix=prefix, tasks=tasks)
@template
def bootstrap_path(path, **kwargs):
when = kwargs.pop("when", None)
if kwargs:
configure_error("bootstrap_path only takes `when` as a keyword argument")
@depends(
"--enable-bootstrap",
toolchains_base_dir,
moz_fetches_dir,
bootstrap_toolchain_tasks,
build_environment,
dependable(path),
when=when,
)
@imports("os")
@imports("subprocess")
@imports("sys")
@imports(_from="mozbuild.util", _import="ensureParentDir")
@imports(_from="importlib", _import="import_module")
@imports(_from="__builtin__", _import="open")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_path(
bootstrap, toolchains_base_dir, moz_fetches_dir, tasks, build_env, path
):
if not path:
return
path_parts = path.split("/")
def try_bootstrap(exists):
if not tasks:
return False
prefixes = [""]
if tasks.prefix:
prefixes.insert(0, "{}-".format(tasks.prefix))
for prefix in prefixes:
label = "toolchain-{}{}".format(prefix, path_parts[0])
task = tasks.tasks.get(label)
if task:
break
log.debug("Trying to bootstrap %s", label)
if not task:
return False
task_index = task["index"]
log.debug("Resolved %s to %s", label, task_index[0])
task_index = task_index[0].split(".")[-1]
artifact = task["artifact"]
# `mach artifact toolchain` doesn't support authentication for
# private artifacts.
if not artifact.startswith("public/"):
log.debug("Cannot bootstrap %s: not a public artifact", label)
return False
index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
try:
with open(index_file) as fh:
index = fh.read().strip()
except Exception:
# On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
# up-to-date.
index = task_index if moz_fetches_dir else None
if index == task_index and exists:
log.debug("%s is up-to-date", label)
return True
# Manually import with import_module so that we can gracefully disable bootstrap
# when e.g. building from a js standalone tarball, that doesn't contain the
# taskgraph code. In those cases, `mach artifact toolchain --from-build` would
# also fail.
try:
IndexSearch = import_module(
"gecko_taskgraph.optimize.strategies"
).IndexSearch
except Exception:
log.debug("Cannot bootstrap %s: missing taskgraph module", label)
return False
task_id = IndexSearch().should_replace_task(task, {}, None, task["index"])
if task_id:
# If we found the task in the index, use the `mach artifact toolchain`
# fast path.
flags = ["--from-task", f"{task_id}:{artifact}"]
else:
# Otherwise, use the slower path, which will print a better error than
# we would be able to.
flags = ["--from-build", label]
log.info(
"%s bootstrapped toolchain in %s",
"Updating" if exists else "Installing",
os.path.join(toolchains_base_dir, path_parts[0]),
)
os.makedirs(toolchains_base_dir, exist_ok=True)
subprocess.run(
[
sys.executable,
os.path.join(build_env.topsrcdir, "mach"),
"--log-no-times",
"artifact",
"toolchain",
]
+ flags,
cwd=toolchains_base_dir,
check=True,
)
ensureParentDir(index_file)
with open(index_file, "w") as fh:
fh.write(task_index)
return True
path = os.path.join(toolchains_base_dir, *path_parts)
if bootstrap:
try:
if not try_bootstrap(os.path.exists(path)):
# If there aren't toolchain artifacts to use for this build,
# don't return a path.
return None
except Exception as e:
log.error("%s", e)
die("If you can't fix the above, retry with --disable-bootstrap.")
# We re-test whether the path exists because it may have been created by
# try_bootstrap. Automation will not have gone through the bootstrap
# process, but we want to return the path if it exists.
if os.path.exists(path):
return path
return bootstrap_path
@template
def bootstrap_search_path(path, paths=original_path, **kwargs):
@depends(
bootstrap_path(path, **kwargs),
bootstrap_search_path_order,
paths,
original_path,
)
def bootstrap_search_path(path, order, paths, original_path):
if paths is None:
paths = original_path
if not path:
return paths
if order == "prepend":
return [path] + paths
return paths + [path]
return bootstrap_search_path
|
