361 lines
13 KiB
Python
361 lines
13 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Transform the beetmover task into an actual task description.
|
|
"""
|
|
|
|
import logging
|
|
from typing import List
|
|
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.util.dependencies import get_dependencies, get_primary_dependency
|
|
from taskgraph.util.schema import Schema
|
|
from taskgraph.util.taskcluster import get_artifact_prefix
|
|
from taskgraph.util.treeherder import inherit_treeherder_from_dep, replace_group
|
|
from voluptuous import Optional, Required
|
|
|
|
from gecko_taskgraph.transforms.beetmover import craft_release_properties
|
|
from gecko_taskgraph.transforms.task import task_description_schema
|
|
from gecko_taskgraph.util.attributes import (
|
|
copy_attributes_from_dependent_job,
|
|
sorted_unique_list,
|
|
)
|
|
from gecko_taskgraph.util.partials import (
|
|
get_balrog_platform_name,
|
|
get_partials_artifacts_from_params,
|
|
get_partials_info_from_params,
|
|
)
|
|
from gecko_taskgraph.util.scriptworker import (
|
|
generate_beetmover_artifact_map,
|
|
generate_beetmover_partials_artifact_map,
|
|
generate_beetmover_upstream_artifacts,
|
|
get_beetmover_action_scope,
|
|
get_beetmover_bucket_scope,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
beetmover_description_schema = Schema(
|
|
{
|
|
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
|
|
Required("label"): str,
|
|
Required("dependencies"): task_description_schema["dependencies"],
|
|
# treeherder is allowed here to override any defaults we use for beetmover. See
|
|
# taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the
|
|
# below transforms for defaults of various values.
|
|
Optional("treeherder"): task_description_schema["treeherder"],
|
|
Optional("attributes"): task_description_schema["attributes"],
|
|
# locale is passed only for l10n beetmoving
|
|
Optional("locale"): str,
|
|
Required("shipping-phase"): task_description_schema["shipping-phase"],
|
|
Optional("task-from"): task_description_schema["task-from"],
|
|
}
|
|
)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
@transforms.add
|
|
def remove_name(config, jobs):
|
|
for job in jobs:
|
|
if "name" in job:
|
|
del job["name"]
|
|
yield job
|
|
|
|
|
|
transforms.add_validate(beetmover_description_schema)
|
|
|
|
|
|
def get_label_by_suffix(labels: List, suffix: str):
|
|
"""
|
|
Given list of labels, returns the label with provided suffix
|
|
Raises exception if more than one label is found.
|
|
|
|
Args:
|
|
labels (List): List of labels
|
|
suffix (str): Suffix for the desired label
|
|
|
|
Returns
|
|
str: The desired label
|
|
"""
|
|
labels = [l for l in labels if l.endswith(suffix)]
|
|
if len(labels) > 1:
|
|
raise Exception(
|
|
f"There should only be a single label with suffix: {suffix} - found {len(labels)}"
|
|
)
|
|
return labels[0]
|
|
|
|
|
|
@transforms.add
|
|
def gather_required_signoffs(config, jobs):
|
|
for job in jobs:
|
|
job.setdefault("attributes", {})["required_signoffs"] = sorted_unique_list(
|
|
*(
|
|
dep.attributes.get("required_signoffs", [])
|
|
for dep in get_dependencies(config, job)
|
|
)
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def make_task_description(config, jobs):
|
|
for job in jobs:
|
|
dep_job = get_primary_dependency(config, job)
|
|
assert dep_job
|
|
|
|
attributes = dep_job.attributes
|
|
|
|
treeherder = inherit_treeherder_from_dep(job, dep_job)
|
|
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
|
|
if "build" in job["dependencies"]:
|
|
build_label = job["dependencies"]["build"]
|
|
build_dep = config.kind_dependencies_tasks[build_label]
|
|
upstream_symbol = build_dep.task["extra"]["treeherder"]["symbol"]
|
|
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
|
|
label = job["label"]
|
|
description = (
|
|
"Beetmover submission for locale '{locale}' for build '"
|
|
"{build_platform}/{build_type}'".format(
|
|
locale=attributes.get("locale", "en-US"),
|
|
build_platform=attributes.get("build_platform"),
|
|
build_type=attributes.get("build_type"),
|
|
)
|
|
)
|
|
|
|
upstream_deps = {
|
|
k: config.kind_dependencies_tasks[v] for k, v in job["dependencies"].items()
|
|
}
|
|
|
|
signing_name = "build-signing"
|
|
build_name = "build"
|
|
repackage_name = "repackage"
|
|
repackage_signing_name = "repackage-signing"
|
|
msi_signing_name = "repackage-signing-msi"
|
|
msix_signing_name = "repackage-signing-shippable-l10n-msix"
|
|
mar_signing_name = "mar-signing"
|
|
attribution_name = "attribution"
|
|
repackage_deb_name = "repackage-deb"
|
|
if job.get("locale"):
|
|
signing_name = "shippable-l10n-signing"
|
|
build_name = "shippable-l10n"
|
|
repackage_name = "repackage-l10n"
|
|
repackage_signing_name = "repackage-signing-l10n"
|
|
mar_signing_name = "mar-signing-l10n"
|
|
attribution_name = "attribution-l10n"
|
|
repackage_deb_name = "repackage-deb-l10n"
|
|
|
|
# The upstream "signing" task for macosx is either *-mac-signing or *-mac-notarization
|
|
if attributes.get("build_platform", "").startswith("macosx"):
|
|
signing_name = None
|
|
# We use the signing task on level 1 and notarization on level 3
|
|
if int(config.params.get("level", 0)) < 3:
|
|
signing_name = get_label_by_suffix(job["dependencies"], "-mac-signing")
|
|
else:
|
|
signing_name = get_label_by_suffix(
|
|
job["dependencies"], "-mac-notarization"
|
|
)
|
|
if not signing_name:
|
|
raise Exception("Could not find upstream kind for mac signing.")
|
|
|
|
dependencies = {
|
|
"build": upstream_deps[build_name],
|
|
"repackage": upstream_deps[repackage_name],
|
|
"signing": upstream_deps[signing_name],
|
|
"mar-signing": upstream_deps[mar_signing_name],
|
|
}
|
|
if "partials-signing" in upstream_deps:
|
|
dependencies["partials-signing"] = upstream_deps["partials-signing"]
|
|
if msi_signing_name in upstream_deps:
|
|
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
|
|
if msix_signing_name in upstream_deps:
|
|
dependencies[msix_signing_name] = upstream_deps[msix_signing_name]
|
|
if repackage_signing_name in upstream_deps:
|
|
dependencies["repackage-signing"] = upstream_deps[repackage_signing_name]
|
|
if attribution_name in upstream_deps:
|
|
dependencies[attribution_name] = upstream_deps[attribution_name]
|
|
if repackage_deb_name in upstream_deps:
|
|
dependencies[repackage_deb_name] = upstream_deps[repackage_deb_name]
|
|
for kind in ("upload-symbols", "upload-symbols-dummy"):
|
|
if kind in upstream_deps:
|
|
dependencies[kind] = upstream_deps[kind]
|
|
|
|
attributes = copy_attributes_from_dependent_job(dep_job)
|
|
attributes.update(job.get("attributes", {}))
|
|
if job.get("locale"):
|
|
attributes["locale"] = job["locale"]
|
|
|
|
bucket_scope = get_beetmover_bucket_scope(config)
|
|
action_scope = get_beetmover_action_scope(config)
|
|
|
|
task = {
|
|
"label": label,
|
|
"description": description,
|
|
"worker-type": "beetmover",
|
|
"scopes": [bucket_scope, action_scope],
|
|
"dependencies": dependencies,
|
|
"attributes": attributes,
|
|
"run-on-projects": dep_job.attributes.get("run_on_projects"),
|
|
"treeherder": treeherder,
|
|
"shipping-phase": job["shipping-phase"],
|
|
"shipping-product": job.get("shipping-product"),
|
|
}
|
|
|
|
yield task
|
|
|
|
|
|
def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):
|
|
artifact_prefix = get_artifact_prefix(job)
|
|
if locale and locale != "en-US":
|
|
artifact_prefix = f"{artifact_prefix}/{locale}"
|
|
|
|
upstream_artifacts = [
|
|
{
|
|
"taskId": {"task-reference": "<partials-signing>"},
|
|
"taskType": "signing",
|
|
"paths": [f"{artifact_prefix}/{path}" for path, _ in artifacts],
|
|
"locale": locale or "en-US",
|
|
}
|
|
]
|
|
|
|
return upstream_artifacts
|
|
|
|
|
|
@transforms.add
|
|
def make_task_worker(config, jobs):
|
|
for job in jobs:
|
|
locale = job["attributes"].get("locale")
|
|
platform = job["attributes"]["build_platform"]
|
|
|
|
worker = {
|
|
"implementation": "beetmover",
|
|
"release-properties": craft_release_properties(config, job),
|
|
"upstream-artifacts": generate_beetmover_upstream_artifacts(
|
|
config, job, platform, locale
|
|
),
|
|
"artifact-map": generate_beetmover_artifact_map(
|
|
config, job, platform=platform, locale=locale
|
|
),
|
|
}
|
|
|
|
if locale:
|
|
worker["locale"] = locale
|
|
job["worker"] = worker
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def strip_unwanted_langpacks_from_worker(config, jobs):
|
|
"""Strips out langpacks where we didn't sign them.
|
|
|
|
This explicitly deletes langpacks from upstream artifacts and from artifact-maps.
|
|
Due to limitations in declarative artifacts, doing this was our easiest way right now.
|
|
"""
|
|
ALWAYS_OK_PLATFORMS = {"linux64-shippable", "linux64-devedition"}
|
|
OSX_OK_PLATFORMS = {"macosx64-shippable", "macosx64-devedition"}
|
|
for job in jobs:
|
|
platform = job["attributes"].get("build_platform")
|
|
if platform in ALWAYS_OK_PLATFORMS:
|
|
# No need to strip anything
|
|
yield job
|
|
continue
|
|
|
|
for map in job["worker"].get("artifact-map", [])[:]:
|
|
if not any([path.endswith("target.langpack.xpi") for path in map["paths"]]):
|
|
continue
|
|
if map["locale"] == "ja-JP-mac":
|
|
# This locale should only exist on mac
|
|
assert platform in OSX_OK_PLATFORMS
|
|
continue
|
|
# map[paths] is being modified while iterating, so we need to resolve the
|
|
# ".keys()" iterator up front by throwing it into a list.
|
|
for path in list(map["paths"].keys()):
|
|
if path.endswith("target.langpack.xpi"):
|
|
del map["paths"][path]
|
|
if map["paths"] == {}:
|
|
job["worker"]["artifact-map"].remove(map)
|
|
|
|
for artifact in job["worker"].get("upstream-artifacts", []):
|
|
if not any(
|
|
[path.endswith("target.langpack.xpi") for path in artifact["paths"]]
|
|
):
|
|
continue
|
|
if artifact["locale"] == "ja-JP-mac":
|
|
# This locale should only exist on mac
|
|
assert platform in OSX_OK_PLATFORMS
|
|
continue
|
|
artifact["paths"] = [
|
|
path
|
|
for path in artifact["paths"]
|
|
if not path.endswith("target.langpack.xpi")
|
|
]
|
|
if artifact["paths"] == []:
|
|
job["worker"]["upstream-artifacts"].remove(artifact)
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def make_partials_artifacts(config, jobs):
|
|
for job in jobs:
|
|
locale = job["attributes"].get("locale")
|
|
if not locale:
|
|
locale = "en-US"
|
|
|
|
platform = job["attributes"]["build_platform"]
|
|
|
|
if "partials-signing" not in job["dependencies"]:
|
|
yield job
|
|
continue
|
|
|
|
balrog_platform = get_balrog_platform_name(platform)
|
|
artifacts = get_partials_artifacts_from_params(
|
|
config.params.get("release_history"), balrog_platform, locale
|
|
)
|
|
|
|
upstream_artifacts = generate_partials_upstream_artifacts(
|
|
job, artifacts, balrog_platform, locale
|
|
)
|
|
|
|
job["worker"]["upstream-artifacts"].extend(upstream_artifacts)
|
|
|
|
extra = list()
|
|
|
|
partials_info = get_partials_info_from_params(
|
|
config.params.get("release_history"), balrog_platform, locale
|
|
)
|
|
|
|
job["worker"]["artifact-map"].extend(
|
|
generate_beetmover_partials_artifact_map(
|
|
config, job, partials_info, platform=platform, locale=locale
|
|
)
|
|
)
|
|
|
|
for artifact in partials_info:
|
|
artifact_extra = {
|
|
"locale": locale,
|
|
"artifact_name": artifact,
|
|
"buildid": partials_info[artifact]["buildid"],
|
|
"platform": balrog_platform,
|
|
}
|
|
for rel_attr in ("previousBuildNumber", "previousVersion"):
|
|
if partials_info[artifact].get(rel_attr):
|
|
artifact_extra[rel_attr] = partials_info[artifact][rel_attr]
|
|
extra.append(artifact_extra)
|
|
|
|
job.setdefault("extra", {})
|
|
job["extra"]["partials"] = extra
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def convert_deps(config, jobs):
|
|
for job in jobs:
|
|
job["dependencies"] = {
|
|
name: dep_job.label for name, dep_job in job["dependencies"].items()
|
|
}
|
|
yield job
|