118 lines
4.7 KiB
Python
118 lines
4.7 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml)
|
|
"""
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import yaml
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from yaml import YAMLError
|
|
|
|
from gecko_taskgraph.transforms.job.common import get_expiration
|
|
from gecko_taskgraph.util.workertypes import worker_type_implementation
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
def read_artifact_manifest(manifest_path):
|
|
"""Read the artifacts.yml manifest file and return it."""
|
|
# logger.info(f"The current directory is {os.getcwd()}")
|
|
try:
|
|
with open(manifest_path) as ymlf:
|
|
yml = yaml.safe_load(ymlf.read())
|
|
return yml
|
|
except YAMLError as ye:
|
|
err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:'
|
|
err += ye
|
|
raise SystemExit(err)
|
|
except FileNotFoundError:
|
|
err = f'Failed to load manifest "{manifest_path}". File not found'
|
|
raise SystemExit(err)
|
|
except PermissionError:
|
|
err = f'Failed to load manifest "{manifest_path}". Permission Error'
|
|
raise SystemExit(err)
|
|
|
|
|
|
@transforms.add
|
|
def set_artifact_expiration(config, jobs):
|
|
"""Set the expiration for certain artifacts based on a manifest file."""
|
|
"""---
|
|
win:
|
|
- build_resources.json: short
|
|
|
|
linux:
|
|
- target.crashreporter-symbols-full.tar.zst: medium
|
|
"""
|
|
transform_dir = os.path.dirname(__file__)
|
|
manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml"))
|
|
|
|
for job in jobs:
|
|
try:
|
|
platform = job["attributes"]["build_platform"]
|
|
except KeyError:
|
|
err = "Tried to get build_platfrom for job, but it does not exist. Exiting."
|
|
raise SystemExit(err)
|
|
if "worker" in job:
|
|
if "env" in job["worker"]:
|
|
if isinstance(job["worker"]["env"], dict):
|
|
job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform
|
|
else:
|
|
raise SystemExit(
|
|
f"Expected env to be a dict, but it was {type(job['worker']['env'])}"
|
|
)
|
|
if "artifacts" in job["worker"]:
|
|
plat = platform.lower()
|
|
if "plain" in plat or "ccov" in plat or "rusttest" in plat:
|
|
art_dict = None
|
|
elif (
|
|
plat == "toolchain-wasm32-wasi-compiler-rt-trunk"
|
|
or plat == "toolchain-linux64-x64-compiler-rt-trunk"
|
|
or plat == "toolchain-linux64-x86-compiler-rt-trunk"
|
|
or plat == "android-geckoview-docs"
|
|
):
|
|
art_dict = None
|
|
elif plat.startswith("win"):
|
|
art_dict = manifest["win"]
|
|
elif plat.startswith("linux"):
|
|
art_dict = manifest["linux"]
|
|
elif plat.startswith("mac"):
|
|
art_dict = manifest["macos"]
|
|
elif plat.startswith("android"):
|
|
art_dict = manifest["android"]
|
|
elif plat.startswith("ios"):
|
|
art_dict = manifest["ios"]
|
|
else:
|
|
print(
|
|
f'The platform name "{plat}" didn\'t start with',
|
|
'"win", "mac", "android", or "linux".',
|
|
file=sys.stderr,
|
|
)
|
|
art_dict = None
|
|
worker_implementation, _ = worker_type_implementation(
|
|
config.graph_config, config.params, job["worker-type"]
|
|
)
|
|
if worker_implementation == "docker-worker":
|
|
artifact_dest = "/builds/worker/cidata/{}"
|
|
else:
|
|
artifact_dest = "cidata/{}"
|
|
|
|
if art_dict is not None:
|
|
for art_name in art_dict.keys():
|
|
# The 'artifacts' key of a job is a list at this stage.
|
|
# So, must append a new dict to the list
|
|
expiry_policy = art_dict[art_name]
|
|
expires = get_expiration(config, policy=expiry_policy)
|
|
new_art = {
|
|
"name": f"public/cidata/{art_name}",
|
|
"path": artifact_dest.format(art_name),
|
|
"type": "file",
|
|
"expires-after": expires,
|
|
}
|
|
job["worker"]["artifacts"].append(new_art)
|
|
yield job
|