summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/transforms/chunk_partners.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /taskcluster/gecko_taskgraph/transforms/chunk_partners.py
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/chunk_partners.py')
-rw-r--r--taskcluster/gecko_taskgraph/transforms/chunk_partners.py75
1 files changed, 75 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/chunk_partners.py b/taskcluster/gecko_taskgraph/transforms/chunk_partners.py
new file mode 100644
index 0000000000..ed74cc6232
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/chunk_partners.py
@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Chunk the partner repack tasks by subpartner and locale
+"""
+
+
+import copy
+
+from mozbuild.chunkify import chunkify
+from taskgraph.transforms.base import TransformSequence
+
+from gecko_taskgraph.util.partners import (
+ apply_partner_priority,
+ get_repack_ids_by_platform,
+)
+
+transforms = TransformSequence()
+transforms.add(apply_partner_priority)
+
+
+@transforms.add
+def chunk_partners(config, jobs):
+ for job in jobs:
+ dep_job = job["primary-dependency"]
+ build_platform = dep_job.attributes["build_platform"]
+ repack_id = dep_job.task.get("extra", {}).get("repack_id")
+ repack_ids = dep_job.task.get("extra", {}).get("repack_ids")
+ copy_repack_ids = job.pop("copy-repack-ids", False)
+
+ if copy_repack_ids:
+ assert repack_ids, "dep_job {} doesn't have repack_ids!".format(
+ dep_job.label
+ )
+ job.setdefault("extra", {})["repack_ids"] = repack_ids
+ yield job
+ # first downstream of the repack task, no chunking or fanout has been done yet
+ elif not any([repack_id, repack_ids]):
+ platform_repack_ids = get_repack_ids_by_platform(config, build_platform)
+ # we chunk mac signing
+ if config.kind in (
+ "release-partner-repack-signing",
+ "release-eme-free-repack-signing",
+ "release-eme-free-repack-mac-signing",
+ "release-partner-repack-mac-signing",
+ ):
+ repacks_per_chunk = job.get("repacks-per-chunk")
+ chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk)
+ if remainder:
+ chunks = int(chunks + 1)
+ for this_chunk in range(1, chunks + 1):
+ chunk = chunkify(platform_repack_ids, this_chunk, chunks)
+ partner_job = copy.deepcopy(job)
+ partner_job.setdefault("extra", {}).setdefault("repack_ids", chunk)
+ partner_job["extra"]["repack_suffix"] = str(this_chunk)
+ yield partner_job
+ # linux and windows we fan out immediately to one task per partner-sub_partner-locale
+ else:
+ for repack_id in platform_repack_ids:
+ partner_job = copy.deepcopy(job) # don't overwrite dict values here
+ partner_job.setdefault("extra", {})
+ partner_job["extra"]["repack_id"] = repack_id
+ yield partner_job
+ # fan out chunked mac signing for repackage
+ elif repack_ids:
+ for repack_id in repack_ids:
+ partner_job = copy.deepcopy(job)
+ partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
+ yield partner_job
+ # otherwise we've fully fanned out already, continue by passing repack_id on
+ else:
+ partner_job = copy.deepcopy(job)
+ partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
+ yield partner_job