1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running jobs via mozharness. Ideally, most stuff gets run this
way, and certainly anything using mozharness should use this approach.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
import six
from six import text_type
from textwrap import dedent
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional, Any
from voluptuous.validators import Match
from mozpack import path as mozpath
from taskgraph.transforms.job import (
configure_taskdesc_for_run,
run_job_using,
)
from taskgraph.transforms.job.common import (
setup_secrets,
docker_worker_add_artifacts,
generic_worker_add_artifacts,
)
from taskgraph.transforms.task import (
get_branch_repo,
get_branch_rev,
)
mozharness_run_schema = Schema(
{
Required("using"): "mozharness",
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
Required("script"): text_type,
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional("config-paths"): [text_type],
# the config files required for the task, relative to
# testing/mozharness/configs or one of the paths specified in
# `config-paths` and using forward slashes even on Windows
Required("config"): [text_type],
# any additional actions to pass to the mozharness command
Optional("actions"): [
Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings")
],
# any additional options (without leading --) to be passed to mozharness
Optional("options"): [
Match(
"^[a-z0-9-]+(=[^ ]+)?$",
"options must be `-` seperated alphanumeric strings (with optional argument)",
)
],
# --custom-build-variant-cfg value
Optional("custom-build-variant-cfg"): text_type,
# Extra configuration options to pass to mozharness.
Optional("extra-config"): dict,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required("tooltool-downloads"): Any(
False,
"public",
"internal",
),
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required("secrets"): Any(bool, [text_type]),
# If true, taskcluster proxy will be enabled; note that it may also be enabled
# automatically e.g., for secrets support. Not supported on Windows.
Required("taskcluster-proxy"): bool,
# If true, the build scripts will start Xvfb. Not supported on Windows.
Required("need-xvfb"): bool,
# If false, indicate that builds should skip producing artifacts. Not
# supported on Windows.
Required("keep-artifacts"): bool,
# If specified, use the in-tree job script specified.
Optional("job-script"): text_type,
Required("requires-signed-builds"): bool,
# Whether or not to use caches.
Optional("use-caches"): bool,
# If false, don't set MOZ_SIMPLE_PACKAGE_NAME
# Only disableable on windows
Required("use-simple-package"): bool,
# If false don't pass --branch mozharness script
# Only disableable on windows
Required("use-magic-mh-args"): bool,
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required("comm-checkout"): bool,
# Base work directory used to set up the task.
Optional("workdir"): text_type,
}
)
mozharness_defaults = {
"tooltool-downloads": False,
"secrets": False,
"taskcluster-proxy": False,
"need-xvfb": False,
"keep-artifacts": True,
"requires-signed-builds": False,
"use-simple-package": True,
"use-magic-mh-args": True,
"comm-checkout": False,
}
@run_job_using(
"docker-worker",
"mozharness",
schema=mozharness_run_schema,
defaults=mozharness_defaults,
)
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
if not run.pop("use-simple-package", None):
raise NotImplementedError(
"Simple packaging cannot be disabled via"
"'use-simple-package' on docker-workers"
)
if not run.pop("use-magic-mh-args", None):
raise NotImplementedError(
"Cannot disabled mh magic arg passing via"
"'use-magic-mh-args' on docker-workers"
)
# Running via mozharness assumes an image that contains build.sh:
# by default, debian8-amd64-build, but it could be another image (like
# android-build).
worker.setdefault("docker-image", {"in-tree": "debian8-amd64-build"})
worker.setdefault("artifacts", []).append(
{
"name": "public/logs",
"path": "{workdir}/logs/".format(**run),
"type": "directory",
}
)
worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None)
docker_worker_add_artifacts(config, job, taskdesc)
env = worker.setdefault("env", {})
env.update(
{
"WORKSPACE": "{workdir}/workspace".format(**run),
"MOZHARNESS_CONFIG": " ".join(run.pop("config")),
"MOZHARNESS_SCRIPT": run.pop("script"),
"MH_BRANCH": config.params["project"],
"MOZ_SOURCE_CHANGESET": get_branch_rev(config),
"MOZ_SOURCE_REPO": get_branch_repo(config),
"MH_BUILD_POOL": "taskcluster",
"MOZ_BUILD_DATE": config.params["moz_build_date"],
"MOZ_SCM_LEVEL": config.params["level"],
"PYTHONUNBUFFERED": "1",
}
)
worker.setdefault("required-volumes", []).append(env["WORKSPACE"])
if "actions" in run:
env["MOZHARNESS_ACTIONS"] = " ".join(run.pop("actions"))
if "options" in run:
env["MOZHARNESS_OPTIONS"] = " ".join(run.pop("options"))
if "config-paths" in run:
env["MOZHARNESS_CONFIG_PATHS"] = " ".join(run.pop("config-paths"))
if "custom-build-variant-cfg" in run:
env["MH_CUSTOM_BUILD_VARIANT_CFG"] = run.pop("custom-build-variant-cfg")
extra_config = run.pop("extra-config", {})
extra_config["objdir"] = "obj-build"
env["EXTRA_MOZHARNESS_CONFIG"] = six.ensure_text(
json.dumps(extra_config, sort_keys=True)
)
if "job-script" in run:
env["JOB_SCRIPT"] = run["job-script"]
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"]
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run.pop("keep-artifacts"):
env["DIST_TARGET_UPLOADS"] = ""
env["DIST_UPLOADS"] = ""
# Xvfb
if run.pop("need-xvfb"):
env["NEED_XVFB"] = "true"
else:
env["NEED_XVFB"] = "false"
# Retry if mozharness returns TBPL_RETRY
worker["retry-exit-status"] = [4]
setup_secrets(config, job, taskdesc)
run["using"] = "run-task"
run["command"] = mozpath.join(
"${GECKO_PATH}",
run.pop("job-script", "taskcluster/scripts/builder/build-linux.sh"),
)
run.pop("secrets")
run.pop("requires-signed-builds")
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
@run_job_using(
"generic-worker",
"mozharness",
schema=mozharness_run_schema,
defaults=mozharness_defaults,
)
def mozharness_on_generic_worker(config, job, taskdesc):
assert job["worker"]["os"] in (
"windows",
"macosx",
), "only supports windows and macOS right now: {}".format(job["label"])
run = job["run"]
# fail if invalid run options are included
invalid = []
for prop in ["need-xvfb"]:
if prop in run and run.pop(prop):
invalid.append(prop)
if not run.pop("keep-artifacts", True):
invalid.append("keep-artifacts")
if invalid:
raise Exception(
"Jobs run using mozharness on Windows do not support properties "
+ ", ".join(invalid)
)
worker = taskdesc["worker"] = job["worker"]
worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None)
setup_secrets(config, job, taskdesc)
taskdesc["worker"].setdefault("artifacts", []).append(
{"name": "public/logs", "path": "logs", "type": "directory"}
)
if not worker.get("skip-artifacts", False):
generic_worker_add_artifacts(config, job, taskdesc)
env = worker["env"]
env.update(
{
"MOZ_BUILD_DATE": config.params["moz_build_date"],
"MOZ_SCM_LEVEL": config.params["level"],
"MH_BRANCH": config.params["project"],
"MOZ_SOURCE_CHANGESET": get_branch_rev(config),
"MOZ_SOURCE_REPO": get_branch_repo(config),
}
)
if run.pop("use-simple-package"):
env.update({"MOZ_SIMPLE_PACKAGE_NAME": "target"})
extra_config = run.pop("extra-config", {})
extra_config["objdir"] = "obj-build"
env["EXTRA_MOZHARNESS_CONFIG"] = six.ensure_text(
json.dumps(extra_config, sort_keys=True)
)
# The windows generic worker uses batch files to pass environment variables
# to commands. Setting a variable to empty in a batch file unsets, so if
# there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that
# mozharness doesn't try to find the commit message on its own.
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"] or "no commit message"
if not job["attributes"]["build_platform"].startswith(("win", "macosx")):
raise Exception(
"Task generation for mozharness build jobs currently only supported on "
"Windows and macOS"
)
mh_command = []
if job["worker"]["os"] == "windows":
mh_command.append("c:/mozilla-build/python3/python3.exe")
gecko_path = "%GECKO_PATH%"
else:
gecko_path = "$GECKO_PATH"
mh_command += [
"{}/mach".format(gecko_path),
"python",
"--no-activate",
"{}/testing/{}".format(gecko_path, run.pop("script")),
]
for path in run.pop("config-paths", []):
mh_command.append("--extra-config-path {}/{}".format(gecko_path, path))
for cfg in run.pop("config"):
mh_command.extend(("--config", cfg))
if run.pop("use-magic-mh-args"):
mh_command.extend(("--branch", config.params["project"]))
if job["worker"]["os"] == "windows":
mh_command.extend(("--work-dir", r"%cd:Z:=z:%\workspace"))
for action in run.pop("actions", []):
mh_command.append("--" + action)
for option in run.pop("options", []):
mh_command.append("--" + option)
if run.get("custom-build-variant-cfg"):
mh_command.append("--custom-build-variant")
mh_command.append(run.pop("custom-build-variant-cfg"))
if job["worker"]["os"] == "macosx":
# Ideally, we'd use shellutil.quote, but that would single-quote
# $GECKO_PATH, which would defeat having the variable in the command
# in the first place, as it wouldn't be expanded.
# In practice, arguments are expected not to contain characters that
# would require quoting.
mh_command = " ".join(mh_command)
run["using"] = "run-task"
run["command"] = mh_command
run.pop("secrets")
run.pop("requires-signed-builds")
run.pop("job-script", None)
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
# Everything past this point is Windows-specific.
if job["worker"]["os"] == "macosx":
return
if taskdesc.get("use-sccache"):
worker["command"] = (
[
# Make the comment part of the first command, as it will help users to
# understand what is going on, and why these steps are implemented.
dedent(
"""\
:: sccache currently uses the full compiler commandline as input to the
:: cache hash key, so create a symlink to the task dir and build from
:: the symlink dir to get consistent paths.
if exist z:\\build rmdir z:\\build"""
),
r"mklink /d z:\build %cd%",
# Grant delete permission on the link to everyone.
r"icacls z:\build /grant *S-1-1-0:D /L",
r"cd /d z:\build",
]
+ worker["command"]
)
|