1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from taskgraph.task import Task
from taskgraph.util.schema import Schema
from voluptuous import Required
from gecko_taskgraph.util.copy_task import copy_task
from ..util.attributes import sorted_unique_list
schema = Schema(
{
Required("primary-dependency", "primary dependency task"): Task,
Required(
"dependent-tasks",
"dictionary of dependent tasks, keyed by kind",
): {str: Task},
}
)
# Define a collection of group_by functions
GROUP_BY_MAP = {}
def group_by(name):
def wrapper(func):
GROUP_BY_MAP[name] = func
return func
return wrapper
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds, designed for use as
multiple-dependent needs.
Required ``group-by-fn`` is used to define how we coalesce the
multiple deps together to pass to transforms, e.g. all kinds specified get
collapsed by platform with `platform`
Optional ``primary-dependency`` (ordered list or string) is used to determine
which upstream kind to inherit attrs from. See ``get_primary_dep``.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a job will be created. Alternatively there is
'not-for-build-platforms' kind configuration which will be consulted only after
'only-for-build-platforms' is checked (if present), and omit any jobs where the
build platform matches.
Optional ``job-template`` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
job_template = config.get("job-template")
for dep_tasks in group_tasks(config, loaded_tasks):
job = {"dependent-tasks": dep_tasks}
job["primary-dependency"] = get_primary_dep(config, dep_tasks)
if job_template:
job.update(copy_task(job_template))
# copy shipping_product from upstream
product = job["primary-dependency"].attributes.get(
"shipping_product", job["primary-dependency"].task.get("shipping-product")
)
if product:
job.setdefault("shipping-product", product)
job.setdefault("attributes", {})["required_signoffs"] = sorted_unique_list(
*(
task.attributes.get("required_signoffs", [])
for task in dep_tasks.values()
)
)
yield job
def skip_only_or_not(config, task):
"""Return True if we should skip this task based on `only_` or `not_` config."""
only_platforms = config.get("only-for-build-platforms")
not_platforms = config.get("not-for-build-platforms")
only_attributes = config.get("only-for-attributes")
not_attributes = config.get("not-for-attributes")
task_attrs = task.attributes
if only_platforms or not_platforms:
platform = task_attrs.get("build_platform")
build_type = task_attrs.get("build_type")
if not platform or not build_type:
return True
combined_platform = f"{platform}/{build_type}"
if only_platforms and combined_platform not in only_platforms:
return True
if not_platforms and combined_platform in not_platforms:
return True
if only_attributes:
if not set(only_attributes) & set(task_attrs):
# make sure any attribute exists
return True
if not_attributes:
if set(not_attributes) & set(task_attrs):
return True
return False
def group_tasks(config, tasks):
group_by_fn = GROUP_BY_MAP[config["group-by"]]
groups = group_by_fn(config, tasks)
for combinations in groups.values():
kinds = [f.kind for f in combinations]
assert_unique_members(
kinds,
error_msg=("Multi_dep.py should have filtered down to one task per kind"),
)
dependencies = {t.kind: copy_task(t) for t in combinations}
yield dependencies
@group_by("platform")
def platform_grouping(config, tasks):
groups = {}
for task in tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if skip_only_or_not(config, task):
continue
platform = task.attributes.get("build_platform")
build_type = task.attributes.get("build_type")
product = task.attributes.get(
"shipping_product", task.task.get("shipping-product")
)
groups.setdefault((platform, build_type, product), []).append(task)
return groups
@group_by("single-locale")
def single_locale_grouping(config, tasks):
"""Split by a single locale (but also by platform, build-type, product)
The locale can be `None` (en-US build/signing/repackage), a single locale,
or multiple locales per task, e.g. for l10n chunking. In the case of a task
with, say, five locales, the task will show up in all five locale groupings.
This grouping is written for non-partner-repack beetmover, but might also
be useful elsewhere.
"""
groups = {}
for task in tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if skip_only_or_not(config, task):
continue
platform = task.attributes.get("build_platform")
build_type = task.attributes.get("build_type")
product = task.attributes.get(
"shipping_product", task.task.get("shipping-product")
)
task_locale = task.attributes.get("locale")
chunk_locales = task.attributes.get("chunk_locales")
locales = chunk_locales or [task_locale]
for locale in locales:
locale_key = (platform, build_type, product, locale)
groups.setdefault(locale_key, [])
if task not in groups[locale_key]:
groups[locale_key].append(task)
return groups
@group_by("chunk-locales")
def chunk_locale_grouping(config, tasks):
"""Split by a chunk_locale (but also by platform, build-type, product)
This grouping is written for mac signing with notarization, but might also
be useful elsewhere.
"""
groups = {}
for task in tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if skip_only_or_not(config, task):
continue
platform = task.attributes.get("build_platform")
build_type = task.attributes.get("build_type")
product = task.attributes.get(
"shipping_product", task.task.get("shipping-product")
)
chunk_locales = tuple(sorted(task.attributes.get("chunk_locales", [])))
chunk_locale_key = (platform, build_type, product, chunk_locales)
groups.setdefault(chunk_locale_key, [])
if task not in groups[chunk_locale_key]:
groups[chunk_locale_key].append(task)
return groups
@group_by("partner-repack-ids")
def partner_repack_ids_grouping(config, tasks):
"""Split by partner_repack_ids (but also by platform, build-type, product)
This grouping is written for release-{eme-free,partner}-repack-signing.
"""
groups = {}
for task in tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if skip_only_or_not(config, task):
continue
platform = task.attributes.get("build_platform")
build_type = task.attributes.get("build_type")
product = task.attributes.get(
"shipping_product", task.task.get("shipping-product")
)
partner_repack_ids = tuple(
sorted(task.task.get("extra", {}).get("repack_ids", []))
)
partner_repack_ids_key = (platform, build_type, product, partner_repack_ids)
groups.setdefault(partner_repack_ids_key, [])
if task not in groups[partner_repack_ids_key]:
groups[partner_repack_ids_key].append(task)
return groups
def assert_unique_members(kinds, error_msg=None):
if len(kinds) != len(set(kinds)):
raise Exception(error_msg)
def get_primary_dep(config, dep_tasks):
"""Find the dependent task to inherit attributes from.
If ``primary-dependency`` is defined in ``kind.yml`` and is a string,
then find the first dep with that task kind and return it. If it is
defined and is a list, the first kind in that list with a matching dep
is the primary dependency. If it's undefined, return the first dep.
"""
primary_dependencies = config.get("primary-dependency")
if isinstance(primary_dependencies, str):
primary_dependencies = [primary_dependencies]
if not primary_dependencies:
assert len(dep_tasks) == 1, "Must define a primary-dependency!"
return list(dep_tasks.values())[0]
primary_dep = None
for primary_kind in primary_dependencies:
for dep_kind in dep_tasks:
if dep_kind == primary_kind:
assert (
primary_dep is None
), "Too many primary dependent tasks in dep_tasks: {}!".format(
[t.label for t in dep_tasks]
)
primary_dep = dep_tasks[dep_kind]
if primary_dep is None:
raise Exception(
"Can't find dependency of {}: {}".format(
config["primary-dependency"], config
)
)
return primary_dep
|
