119 lines
4.2 KiB
Python
119 lines
4.2 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
from itertools import islice
|
|
|
|
from taskgraph import MAX_DEPENDENCIES
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.util.dependencies import get_primary_dependency
|
|
|
|
from gecko_taskgraph.util.platforms import architecture
|
|
from gecko_taskgraph.util.scriptworker import (
|
|
generate_artifact_registry_gcs_sources,
|
|
get_beetmover_apt_repo_scope,
|
|
get_beetmover_repo_action_scope,
|
|
)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
@transforms.add
|
|
def beetmover_apt(config, tasks):
|
|
product = (
|
|
"firefox"
|
|
if config.params["release_type"] == "nightly"
|
|
else config.params["release_product"]
|
|
)
|
|
filtered_tasks = filter_beetmover_apt_tasks(config, tasks, product)
|
|
# There are too many beetmover-repackage dependencies for a single task
|
|
# and we hit the taskgraph dependencies limit.
|
|
# To work around this limitation, we chunk the would be task
|
|
# into tasks dependendent on, at most, half of MAX_DEPENDENCIES.
|
|
batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2)
|
|
for index, batch in enumerate(batches):
|
|
dependencies = {}
|
|
gcs_sources = []
|
|
for task in batch:
|
|
dep = get_primary_dependency(config, task)
|
|
assert dep
|
|
|
|
dependencies[dep.label] = dep.label
|
|
gcs_sources.extend(generate_artifact_registry_gcs_sources(dep))
|
|
description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages"
|
|
platform = f"{product}-release/opt"
|
|
treeherder = {
|
|
"platform": platform,
|
|
"tier": 1,
|
|
"kind": "other",
|
|
"symbol": f"BM-apt(batch-{index + 1})",
|
|
}
|
|
apt_repo_scope = get_beetmover_apt_repo_scope(config)
|
|
repo_action_scope = get_beetmover_repo_action_scope(config)
|
|
attributes = {
|
|
"required_signoffs": ["mar-signing"],
|
|
"shippable": True,
|
|
"shipping_product": product,
|
|
}
|
|
task = {
|
|
"label": f"{config.kind}-{index + 1}-{platform}",
|
|
"description": description,
|
|
"worker-type": "beetmover",
|
|
"treeherder": treeherder,
|
|
"scopes": [apt_repo_scope, repo_action_scope],
|
|
"attributes": attributes,
|
|
"shipping-phase": "ship",
|
|
"shipping-product": product,
|
|
"dependencies": dependencies,
|
|
}
|
|
worker = {
|
|
"implementation": "beetmover-import-from-gcs-to-artifact-registry",
|
|
"product": product,
|
|
"gcs-sources": gcs_sources,
|
|
}
|
|
task["worker"] = worker
|
|
yield task
|
|
|
|
|
|
def batched(iterable, n):
|
|
"Batch data into tuples of length n. The last batch may be shorter."
|
|
# batched('ABCDEFG', 3) --> ABC DEF G
|
|
if n < 1:
|
|
raise ValueError("n must be at least one")
|
|
it = iter(iterable)
|
|
batch = tuple(islice(it, n))
|
|
while batch:
|
|
yield batch
|
|
batch = tuple(islice(it, n))
|
|
|
|
|
|
def filter_beetmover_apt_tasks(config, tasks, product):
|
|
for task in tasks:
|
|
task["primary-dependency"] = get_primary_dependency(config, task)
|
|
if filter_beetmover_apt_task(task, product):
|
|
yield task
|
|
|
|
|
|
def filter_beetmover_apt_task(task, product):
|
|
# We only create beetmover-apt tasks for l10n beetmover-repackage tasks that
|
|
# beetmove langpack .deb packages. The langpack .deb packages support all
|
|
# architectures, so we generate them only on x86_64 tasks.
|
|
return (
|
|
is_x86_64_l10n_task(task) or is_not_l10n_task(task)
|
|
) and is_task_for_product(task, product)
|
|
|
|
|
|
def is_x86_64_l10n_task(task):
|
|
dep = task["primary-dependency"]
|
|
locale = dep.attributes.get("locale")
|
|
return locale and architecture(dep.attributes["build_platform"]) == "x86_64"
|
|
|
|
|
|
def is_not_l10n_task(task):
|
|
dep = task["primary-dependency"]
|
|
locale = dep.attributes.get("locale")
|
|
return not locale
|
|
|
|
|
|
def is_task_for_product(task, product):
|
|
dep = task["primary-dependency"]
|
|
return dep.attributes.get("shipping_product") == product
|