1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from itertools import islice
from taskgraph import MAX_DEPENDENCIES
from taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.platforms import architecture
from gecko_taskgraph.util.scriptworker import (
generate_artifact_registry_gcs_sources,
get_beetmover_apt_repo_scope,
get_beetmover_repo_action_scope,
)
transforms = TransformSequence()
@transforms.add
def beetmover_apt(config, tasks):
product = (
"firefox"
if not config.params["release_type"] # try
or config.params["release_type"] == "nightly"
else config.params["release_product"]
)
filtered_tasks = filter_beetmover_apt_tasks(tasks, product)
# There are too many beetmover-repackage dependencies for a single task
# and we hit the taskgraph dependencies limit.
# To work around this limitation, we chunk the would be task
# into tasks dependendent on, at most, half of MAX_DEPENDENCIES.
batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2)
for index, batch in enumerate(batches):
dependencies = {}
gcs_sources = []
for task in batch:
dep = task["primary-dependency"]
dependencies[dep.label] = dep.label
gcs_sources.extend(generate_artifact_registry_gcs_sources(dep))
description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages"
platform = "firefox-release/opt"
treeherder = {
"platform": platform,
"tier": 1,
"kind": "other",
"symbol": f"BM-apt(batch-{index + 1})",
}
apt_repo_scope = get_beetmover_apt_repo_scope(config)
repo_action_scope = get_beetmover_repo_action_scope(config)
attributes = {
"required_signoffs": ["mar-signing"],
"shippable": True,
"shipping_product": product,
}
task = {
"label": f"{config.kind}-{index + 1}-{platform}",
"description": description,
"worker-type": "beetmover",
"treeherder": treeherder,
"scopes": [apt_repo_scope, repo_action_scope],
"attributes": attributes,
"shipping-phase": "ship",
"shipping-product": product,
"dependencies": dependencies,
}
worker = {
"implementation": "beetmover-import-from-gcs-to-artifact-registry",
"product": product,
"gcs-sources": gcs_sources,
}
task["worker"] = worker
yield task
def batched(iterable, n):
"Batch data into tuples of length n. The last batch may be shorter."
# batched('ABCDEFG', 3) --> ABC DEF G
if n < 1:
raise ValueError("n must be at least one")
it = iter(iterable)
batch = tuple(islice(it, n))
while batch:
yield batch
batch = tuple(islice(it, n))
def filter_beetmover_apt_tasks(tasks, product):
return (task for task in tasks if filter_beetmover_apt_task(task, product))
def filter_beetmover_apt_task(task, product):
# We only create beetmover-apt tasks for l10n beetmover-repackage tasks that
# beetmove langpack .deb packages. The langpack .deb packages support all
# architectures, so we generate them only on x86_64 tasks.
return (
is_x86_64_l10n_task(task) or is_not_l10n_task(task)
) and is_task_for_product(task, product)
def is_x86_64_l10n_task(task):
dep = task["primary-dependency"]
locale = dep.attributes.get("locale")
return locale and architecture(dep.attributes["build_platform"]) == "x86_64"
def is_not_l10n_task(task):
dep = task["primary-dependency"]
locale = dep.attributes.get("locale")
return not locale
def is_task_for_product(task, product):
dep = task["primary-dependency"]
return dep.attributes.get("shipping_product") == product
|
