1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Chunk the partner repack tasks by subpartner and locale
"""
import copy
from mozbuild.chunkify import chunkify
from taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.partners import (
apply_partner_priority,
get_repack_ids_by_platform,
)
transforms = TransformSequence()
transforms.add(apply_partner_priority)
@transforms.add
def chunk_partners(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
build_platform = dep_job.attributes["build_platform"]
repack_id = dep_job.task.get("extra", {}).get("repack_id")
repack_ids = dep_job.task.get("extra", {}).get("repack_ids")
copy_repack_ids = job.pop("copy-repack-ids", False)
if copy_repack_ids:
assert repack_ids, "dep_job {} doesn't have repack_ids!".format(
dep_job.label
)
job.setdefault("extra", {})["repack_ids"] = repack_ids
yield job
# first downstream of the repack task, no chunking or fanout has been done yet
elif not any([repack_id, repack_ids]):
platform_repack_ids = get_repack_ids_by_platform(config, build_platform)
# we chunk mac signing
if config.kind in (
"release-partner-repack-signing",
"release-eme-free-repack-signing",
"release-eme-free-repack-mac-signing",
"release-partner-repack-mac-signing",
):
repacks_per_chunk = job.get("repacks-per-chunk")
chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk)
if remainder:
chunks = int(chunks + 1)
for this_chunk in range(1, chunks + 1):
chunk = chunkify(platform_repack_ids, this_chunk, chunks)
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_ids", chunk)
partner_job["extra"]["repack_suffix"] = str(this_chunk)
yield partner_job
# linux and windows we fan out immediately to one task per partner-sub_partner-locale
else:
for repack_id in platform_repack_ids:
partner_job = copy.deepcopy(job) # don't overwrite dict values here
partner_job.setdefault("extra", {})
partner_job["extra"]["repack_id"] = repack_id
yield partner_job
# fan out chunked mac signing for repackage
elif repack_ids:
for repack_id in repack_ids:
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
yield partner_job
# otherwise we've fully fanned out already, continue by passing repack_id on
else:
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
yield partner_job
|
