summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/transforms/artifact.py
blob: b14f723d4a440e247fbec73ff962e8b5ecb0d91a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml)
"""
import logging
import os
import sys

import yaml
from taskgraph.transforms.base import TransformSequence
from yaml import YAMLError

from gecko_taskgraph.transforms.job.common import get_expiration
from gecko_taskgraph.util.workertypes import worker_type_implementation

logger = logging.getLogger(__name__)

transforms = TransformSequence()


def read_artifact_manifest(manifest_path):
    """Read the artifacts.yml manifest file and return it."""
    # logger.info(f"The current directory is {os.getcwd()}")
    try:
        with open(manifest_path, "r") as ymlf:
            yml = yaml.safe_load(ymlf.read())
            return yml
    except YAMLError as ye:
        err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:'
        err += ye
        raise SystemExit(err)
    except FileNotFoundError:
        err = f'Failed to load manifest "{manifest_path}". File not found'
        raise SystemExit(err)
    except PermissionError:
        err = f'Failed to load manifest "{manifest_path}". Permission Error'
        raise SystemExit(err)


@transforms.add
def set_artifact_expiration(config, jobs):
    """Set the expiration for certain artifacts based on a manifest file."""
    """---
       win:
         - build_resources.json: short

       linux:
         - target.crashreporter-symbols-full.tar.zst: medium
    """
    transform_dir = os.path.dirname(__file__)
    manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml"))

    for job in jobs:
        try:
            platform = job["attributes"]["build_platform"]
        except KeyError:
            err = "Tried to get build_platfrom for job, but it does not exist. Exiting."
            raise SystemExit(err)
        if "worker" in job:
            if "env" in job["worker"]:
                if isinstance(job["worker"]["env"], dict):
                    job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform
                else:
                    raise SystemExit(
                        f"Expected env to be a dict, but it was {type(job['worker']['env'])}"
                    )
            if "artifacts" in job["worker"]:
                plat = platform.lower()
                if "plain" in plat or "ccov" in plat or "rusttest" in plat:
                    art_dict = None
                elif (
                    plat == "toolchain-wasm32-wasi-compiler-rt-trunk"
                    or plat == "toolchain-linux64-x64-compiler-rt-trunk"
                    or plat == "toolchain-linux64-x86-compiler-rt-trunk"
                    or plat == "android-geckoview-docs"
                ):
                    art_dict = None
                elif plat.startswith("win"):
                    art_dict = manifest["win"]
                elif plat.startswith("linux"):
                    art_dict = manifest["linux"]
                elif plat.startswith("mac"):
                    art_dict = manifest["macos"]
                elif plat.startswith("android"):
                    art_dict = manifest["android"]
                elif plat.startswith("ios"):
                    art_dict = manifest["ios"]
                else:
                    print(
                        f'The platform name "{plat}" didn\'t start with',
                        '"win", "mac", "android", or "linux".',
                        file=sys.stderr,
                    )
                    art_dict = None
                worker_implementation, _ = worker_type_implementation(
                    config.graph_config, config.params, job["worker-type"]
                )
                if worker_implementation == "docker-worker":
                    artifact_dest = "/builds/worker/cidata/{}"
                else:
                    artifact_dest = "cidata/{}"

                if art_dict is not None:
                    for art_name in art_dict.keys():
                        # The 'artifacts' key of a job is a list at this stage.
                        # So, must append a new dict to the list
                        expiry_policy = art_dict[art_name]
                        expires = get_expiration(config, policy=expiry_policy)
                        new_art = {
                            "name": f"public/cidata/{art_name}",
                            "path": artifact_dest.format(art_name),
                            "type": "file",
                            "expires-after": expires,
                        }
                        job["worker"]["artifacts"].append(new_art)
        yield job