236 lines
7.8 KiB
Python
236 lines
7.8 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Apply some defaults and minor modifications to the jobs defined in the build
|
|
kind.
|
|
"""
|
|
import logging
|
|
|
|
from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.util.schema import resolve_keyed_by
|
|
from taskgraph.util.treeherder import add_suffix
|
|
|
|
from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level
|
|
from gecko_taskgraph.util.workertypes import worker_type_implementation
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
@transforms.add
|
|
def set_defaults(config, jobs):
|
|
"""Set defaults, including those that differ per worker implementation"""
|
|
for job in jobs:
|
|
job["treeherder"].setdefault("kind", "build")
|
|
job["treeherder"].setdefault("tier", 1)
|
|
_, worker_os = worker_type_implementation(
|
|
config.graph_config, config.params, job["worker-type"]
|
|
)
|
|
worker = job.setdefault("worker", {})
|
|
worker.setdefault("env", {})
|
|
worker["chain-of-trust"] = True
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def stub_installer(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job,
|
|
"stub-installer",
|
|
item_name=job["name"],
|
|
project=config.params["project"],
|
|
**{
|
|
"release-type": config.params["release_type"],
|
|
},
|
|
)
|
|
job.setdefault("attributes", {})
|
|
if job.get("stub-installer"):
|
|
job["attributes"]["stub-installer"] = job["stub-installer"]
|
|
job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"})
|
|
if "stub-installer" in job:
|
|
del job["stub-installer"]
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def resolve_shipping_product(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job,
|
|
"shipping-product",
|
|
item_name=job["name"],
|
|
**{
|
|
"release-type": config.params["release_type"],
|
|
},
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def update_channel(config, jobs):
|
|
keys = [
|
|
"run.update-channel",
|
|
"run.mar-channel-id",
|
|
"run.accepted-mar-channel-ids",
|
|
]
|
|
for job in jobs:
|
|
job["worker"].setdefault("env", {})
|
|
for key in keys:
|
|
resolve_keyed_by(
|
|
job,
|
|
key,
|
|
item_name=job["name"],
|
|
**{
|
|
"project": config.params["project"],
|
|
"release-type": config.params["release_type"],
|
|
},
|
|
)
|
|
update_channel = job["run"].pop("update-channel", None)
|
|
if update_channel:
|
|
job["run"].setdefault("extra-config", {})["update_channel"] = update_channel
|
|
job["attributes"]["update-channel"] = update_channel
|
|
mar_channel_id = job["run"].pop("mar-channel-id", None)
|
|
if mar_channel_id:
|
|
job["attributes"]["mar-channel-id"] = mar_channel_id
|
|
job["worker"]["env"]["MAR_CHANNEL_ID"] = mar_channel_id
|
|
accepted_mar_channel_ids = job["run"].pop("accepted-mar-channel-ids", None)
|
|
if accepted_mar_channel_ids:
|
|
job["attributes"]["accepted-mar-channel-ids"] = accepted_mar_channel_ids
|
|
job["worker"]["env"]["ACCEPTED_MAR_CHANNEL_IDS"] = accepted_mar_channel_ids
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def mozconfig(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job,
|
|
"run.mozconfig-variant",
|
|
item_name=job["name"],
|
|
**{
|
|
"release-type": config.params["release_type"],
|
|
},
|
|
)
|
|
mozconfig_variant = job["run"].pop("mozconfig-variant", None)
|
|
if mozconfig_variant:
|
|
job["run"].setdefault("extra-config", {})[
|
|
"mozconfig_variant"
|
|
] = mozconfig_variant
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def use_artifact(config, jobs):
|
|
if is_try(config.params):
|
|
use_artifact = config.params["try_task_config"].get(
|
|
"use-artifact-builds", False
|
|
)
|
|
else:
|
|
use_artifact = False
|
|
for job in jobs:
|
|
if (
|
|
config.kind == "build"
|
|
and use_artifact
|
|
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
|
|
# If tests aren't packaged, then we are not able to rebuild all the packages
|
|
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1"
|
|
# Android shippable artifact builds are not supported
|
|
and not (
|
|
"android" in job["name"] and job["attributes"].get("shippable", False)
|
|
)
|
|
):
|
|
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a")
|
|
job["worker"]["env"]["USE_ARTIFACT"] = "1"
|
|
job["attributes"]["artifact-build"] = True
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def use_profile_data(config, jobs):
|
|
for job in jobs:
|
|
use_pgo = job.pop("use-pgo", False)
|
|
disable_pgo = config.params["try_task_config"].get("disable-pgo", False)
|
|
artifact_build = job["attributes"].get("artifact-build")
|
|
if not use_pgo or disable_pgo or artifact_build:
|
|
yield job
|
|
continue
|
|
|
|
# If use_pgo is True, the task uses the generate-profile task of the
|
|
# same name. Otherwise a task can specify a specific generate-profile
|
|
# task to use in the use_pgo field.
|
|
if use_pgo is True:
|
|
name = job["name"]
|
|
else:
|
|
name = use_pgo
|
|
dependencies = f"generate-profile-{name}"
|
|
job.setdefault("dependencies", {})["generate-profile"] = dependencies
|
|
job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"]
|
|
job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
|
|
|
|
_, worker_os = worker_type_implementation(
|
|
config.graph_config, config.params, job["worker-type"]
|
|
)
|
|
if worker_os == "linux":
|
|
# LTO linkage needs more open files than the default from run-task.
|
|
job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"})
|
|
|
|
if job.get("use-sccache"):
|
|
raise Exception(
|
|
"use-sccache is incompatible with use-pgo in {}".format(job["name"])
|
|
)
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def resolve_keys(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job,
|
|
"use-sccache",
|
|
item_name=job["name"],
|
|
**{"release-level": release_level(config.params["project"])},
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def enable_full_crashsymbols(config, jobs):
|
|
"""Enable full crashsymbols on jobs with
|
|
'enable-full-crashsymbols' set to True and on release branches, or
|
|
on try"""
|
|
branches = RELEASE_PROJECTS | {"toolchains", "try", "try-comm-central"}
|
|
for job in jobs:
|
|
enable_full_crashsymbols = job["attributes"].get("enable-full-crashsymbols")
|
|
if enable_full_crashsymbols and config.params["project"] in branches:
|
|
logger.debug("Enabling full symbol generation for %s", job["name"])
|
|
job["worker"]["env"]["MOZ_ENABLE_FULL_SYMBOLS"] = "1"
|
|
else:
|
|
logger.debug("Disabling full symbol generation for %s", job["name"])
|
|
job["attributes"].pop("enable-full-crashsymbols", None)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def set_expiry(config, jobs):
|
|
for job in jobs:
|
|
attributes = job["attributes"]
|
|
if (
|
|
"shippable" in attributes
|
|
and attributes["shippable"]
|
|
and config.kind
|
|
in {
|
|
"build",
|
|
}
|
|
):
|
|
expiration_policy = "long"
|
|
else:
|
|
expiration_policy = "medium"
|
|
|
|
job["expiration-policy"] = expiration_policy
|
|
yield job
|