summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /taskcluster/gecko_taskgraph/transforms/beetmover_apt.py
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms/beetmover_apt.py')
-rw-r--r--taskcluster/gecko_taskgraph/transforms/beetmover_apt.py114
1 files changed, 114 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py b/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py
new file mode 100644
index 0000000000..8c56f1a968
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py
@@ -0,0 +1,114 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from itertools import islice
+
+from taskgraph import MAX_DEPENDENCIES
+from taskgraph.transforms.base import TransformSequence
+
+from gecko_taskgraph.util.platforms import architecture
+from gecko_taskgraph.util.scriptworker import (
+ generate_artifact_registry_gcs_sources,
+ get_beetmover_apt_repo_scope,
+ get_beetmover_repo_action_scope,
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def beetmover_apt(config, tasks):
+ product = (
+ "firefox"
+ if not config.params["release_type"] # try
+ or config.params["release_type"] == "nightly"
+ else config.params["release_product"]
+ )
+ filtered_tasks = filter_beetmover_apt_tasks(tasks, product)
+ # There are too many beetmover-repackage dependencies for a single task
+ # and we hit the taskgraph dependencies limit.
+ # To work around this limitation, we chunk the would be task
+ # into tasks dependendent on, at most, half of MAX_DEPENDENCIES.
+ batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2)
+ for index, batch in enumerate(batches):
+ dependencies = {}
+ gcs_sources = []
+ for task in batch:
+ dep = task["primary-dependency"]
+ dependencies[dep.label] = dep.label
+ gcs_sources.extend(generate_artifact_registry_gcs_sources(dep))
+ description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages"
+ platform = "firefox-release/opt"
+ treeherder = {
+ "platform": platform,
+ "tier": 1,
+ "kind": "other",
+ "symbol": f"BM-apt(batch-{index + 1})",
+ }
+ apt_repo_scope = get_beetmover_apt_repo_scope(config)
+ repo_action_scope = get_beetmover_repo_action_scope(config)
+ attributes = {
+ "required_signoffs": ["mar-signing"],
+ "shippable": True,
+ "shipping_product": product,
+ }
+ task = {
+ "label": f"{config.kind}-{index + 1}-{platform}",
+ "description": description,
+ "worker-type": "beetmover",
+ "treeherder": treeherder,
+ "scopes": [apt_repo_scope, repo_action_scope],
+ "attributes": attributes,
+ "shipping-phase": "ship",
+ "shipping-product": product,
+ "dependencies": dependencies,
+ }
+ worker = {
+ "implementation": "beetmover-import-from-gcs-to-artifact-registry",
+ "product": product,
+ "gcs-sources": gcs_sources,
+ }
+ task["worker"] = worker
+ yield task
+
+
+def batched(iterable, n):
+ "Batch data into tuples of length n. The last batch may be shorter."
+ # batched('ABCDEFG', 3) --> ABC DEF G
+ if n < 1:
+ raise ValueError("n must be at least one")
+ it = iter(iterable)
+ batch = tuple(islice(it, n))
+ while batch:
+ yield batch
+ batch = tuple(islice(it, n))
+
+
+def filter_beetmover_apt_tasks(tasks, product):
+ return (task for task in tasks if filter_beetmover_apt_task(task, product))
+
+
+def filter_beetmover_apt_task(task, product):
+ # We only create beetmover-apt tasks for l10n beetmover-repackage tasks that
+ # beetmove langpack .deb packages. The langpack .deb packages support all
+ # architectures, so we generate them only on x86_64 tasks.
+ return (
+ is_x86_64_l10n_task(task) or is_not_l10n_task(task)
+ ) and is_task_for_product(task, product)
+
+
+def is_x86_64_l10n_task(task):
+ dep = task["primary-dependency"]
+ locale = dep.attributes.get("locale")
+ return locale and architecture(dep.attributes["build_platform"]) == "x86_64"
+
+
+def is_not_l10n_task(task):
+ dep = task["primary-dependency"]
+ locale = dep.attributes.get("locale")
+ return not locale
+
+
+def is_task_for_product(task, product):
+ dep = task["primary-dependency"]
+ return dep.attributes.get("shipping_product") == product