summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/util/partials.py
blob: b04fc64e17a3192d47d809e0fa61b28fce5ad2de (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


import logging

import redo
import requests

from gecko_taskgraph.util.scriptworker import (
    BALROG_SCOPE_ALIAS_TO_PROJECT,
    BALROG_SERVER_SCOPES,
)

logger = logging.getLogger(__name__)

PLATFORM_RENAMES = {
    "windows2012-32": "win32",
    "windows2012-64": "win64",
    "windows2012-aarch64": "win64-aarch64",
    "osx-cross": "macosx64",
    "osx": "macosx64",
}

BALROG_PLATFORM_MAP = {
    "linux": ["Linux_x86-gcc3"],
    "linux32": ["Linux_x86-gcc3"],
    "linux64": ["Linux_x86_64-gcc3"],
    "linux64-asan-reporter": ["Linux_x86_64-gcc3-asan"],
    "macosx64": [
        "Darwin_x86_64-gcc3-u-i386-x86_64",
        "Darwin_x86-gcc3-u-i386-x86_64",
        "Darwin_aarch64-gcc3",
        "Darwin_x86-gcc3",
        "Darwin_x86_64-gcc3",
    ],
    "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"],
    "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"],
    "win64-asan-reporter": ["WINNT_x86_64-msvc-x64-asan"],
    "win64-aarch64": [
        "WINNT_aarch64-msvc-aarch64",
    ],
}

FTP_PLATFORM_MAP = {
    "Darwin_x86-gcc3": "mac",
    "Darwin_x86-gcc3-u-i386-x86_64": "mac",
    "Darwin_x86_64-gcc3": "mac",
    "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
    "Darwin_aarch64-gcc3": "mac",
    "Linux_x86-gcc3": "linux-i686",
    "Linux_x86_64-gcc3": "linux-x86_64",
    "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter",
    "WINNT_x86_64-msvc-x64-asan": "win64-asan-reporter",
    "WINNT_x86-msvc": "win32",
    "WINNT_x86-msvc-x64": "win32",
    "WINNT_x86-msvc-x86": "win32",
    "WINNT_x86_64-msvc": "win64",
    "WINNT_x86_64-msvc-x64": "win64",
    "WINNT_aarch64-msvc-aarch64": "win64-aarch64",
}


def get_balrog_platform_name(platform):
    """Convert build platform names into balrog platform names.

    Remove known values instead to catch aarch64 and other platforms
    that may be added.
    """
    removals = ["-devedition", "-shippable"]
    for remove in removals:
        platform = platform.replace(remove, "")
    return PLATFORM_RENAMES.get(platform, platform)


def _sanitize_platform(platform):
    platform = get_balrog_platform_name(platform)
    if platform not in BALROG_PLATFORM_MAP:
        return platform
    return BALROG_PLATFORM_MAP[platform][0]


def get_builds(release_history, platform, locale):
    """Examine cached balrog release history and return the list of
    builds we need to generate diffs from"""
    platform = _sanitize_platform(platform)
    return release_history.get(platform, {}).get(locale, {})


def get_partials_artifacts_from_params(release_history, platform, locale):
    platform = _sanitize_platform(platform)
    return [
        (artifact, details.get("previousVersion", None))
        for artifact, details in release_history.get(platform, {})
        .get(locale, {})
        .items()
    ]


def get_partials_info_from_params(release_history, platform, locale):
    platform = _sanitize_platform(platform)

    artifact_map = {}
    for k in release_history.get(platform, {}).get(locale, {}):
        details = release_history[platform][locale][k]
        attributes = ("buildid", "previousBuildNumber", "previousVersion")
        artifact_map[k] = {
            attr: details[attr] for attr in attributes if attr in details
        }
    return artifact_map


def _retry_on_http_errors(url, verify, params, errors):
    if params:
        params_str = "&".join("=".join([k, str(v)]) for k, v in params.items())
    else:
        params_str = ""
    logger.info("Connecting to %s?%s", url, params_str)
    for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
        try:
            req = requests.get(url, verify=verify, params=params, timeout=10)
            req.raise_for_status()
            return req
        except requests.HTTPError as e:
            if e.response.status_code in errors:
                logger.exception(
                    "Got HTTP %s trying to reach %s", e.response.status_code, url
                )
            else:
                raise
    else:
        raise Exception(f"Cannot connect to {url}!")


def get_sorted_releases(product, branch):
    """Returns a list of release names from Balrog.
    :param product: product name, AKA appName
    :param branch: branch name, e.g. mozilla-central
    :return: a sorted list of release names, most recent first.
    """
    url = f"{_get_balrog_api_root(branch)}/releases"
    params = {
        "product": product,
        # Adding -nightly-2 (2 stands for the beginning of build ID
        # based on date) should filter out release and latest blobs.
        # This should be changed to -nightly-3 in 3000 ;)
        "name_prefix": f"{product}-{branch}-nightly-2",
        "names_only": True,
    }
    req = _retry_on_http_errors(url=url, verify=True, params=params, errors=[500])
    releases = req.json()["names"]
    releases = sorted(releases, reverse=True)
    return releases


def get_release_builds(release, branch):
    url = f"{_get_balrog_api_root(branch)}/releases/{release}"
    req = _retry_on_http_errors(url=url, verify=True, params=None, errors=[500])
    return req.json()


def _get_balrog_api_root(branch):
    # Query into the scopes scriptworker uses to make sure we check against the same balrog server
    # That our jobs would use.
    scope = None
    for alias, projects in BALROG_SCOPE_ALIAS_TO_PROJECT:
        if branch in projects and alias in BALROG_SERVER_SCOPES:
            scope = BALROG_SERVER_SCOPES[alias]
            break
    else:
        scope = BALROG_SERVER_SCOPES["default"]

    if scope == "balrog:server:dep":
        return "https://stage.balrog.nonprod.cloudops.mozgcp.net/api/v1"
    return "https://aus5.mozilla.org/api/v1"


def find_localtest(fileUrls):
    for channel in fileUrls:
        if "-localtest" in channel:
            return channel


def populate_release_history(
    product, branch, maxbuilds=4, maxsearch=10, partial_updates=None
):
    # Assuming we are using release branches when we know the list of previous
    # releases in advance
    if partial_updates is not None:
        return _populate_release_history(
            product, branch, partial_updates=partial_updates
        )
    return _populate_nightly_history(
        product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch
    )


def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10):
    """Find relevant releases in Balrog
    Not all releases have all platforms and locales, due
    to Taskcluster migration.

        Args:
            product (str): capitalized product name, AKA appName, e.g. Firefox
            branch (str): branch name (mozilla-central)
            maxbuilds (int): Maximum number of historical releases to populate
            maxsearch(int): Traverse at most this many releases, to avoid
                working through the entire history.
        Returns:
            json object based on data from balrog api

            results = {
                'platform1': {
                    'locale1': {
                        'buildid1': mar_url,
                        'buildid2': mar_url,
                        'buildid3': mar_url,
                    },
                    'locale2': {
                        'target.partial-1.mar': {'buildid1': 'mar_url'},
                    }
                },
                'platform2': {
                }
            }
    """
    last_releases = get_sorted_releases(product, branch)

    partial_mar_tmpl = "target.partial-{}.mar"

    builds = dict()
    for release in last_releases[:maxsearch]:
        # maxbuilds in all categories, don't make any more queries
        full = len(builds) > 0 and all(
            len(builds[platform][locale]) >= maxbuilds
            for platform in builds
            for locale in builds[platform]
        )
        if full:
            break
        history = get_release_builds(release, branch)

        for platform in history["platforms"]:
            if "alias" in history["platforms"][platform]:
                continue
            if platform not in builds:
                builds[platform] = dict()
            for locale in history["platforms"][platform]["locales"]:
                if locale not in builds[platform]:
                    builds[platform][locale] = dict()
                if len(builds[platform][locale]) >= maxbuilds:
                    continue
                if "buildID" not in history["platforms"][platform]["locales"][locale]:
                    continue
                buildid = history["platforms"][platform]["locales"][locale]["buildID"]
                if (
                    "completes" not in history["platforms"][platform]["locales"][locale]
                    or len(
                        history["platforms"][platform]["locales"][locale]["completes"]
                    )
                    == 0
                ):
                    continue
                url = history["platforms"][platform]["locales"][locale]["completes"][0][
                    "fileUrl"
                ]
                nextkey = len(builds[platform][locale]) + 1
                builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
                    "buildid": buildid,
                    "mar_url": url,
                }
    return builds


def _populate_release_history(product, branch, partial_updates):
    builds = dict()
    for version, release in partial_updates.items():
        prev_release_blob = "{product}-{version}-build{build_number}".format(
            product=product, version=version, build_number=release["buildNumber"]
        )
        partial_mar_key = f"target-{version}.partial.mar"
        history = get_release_builds(prev_release_blob, branch)
        # use one of the localtest channels to avoid relying on bouncer
        localtest = find_localtest(history["fileUrls"])
        url_pattern = history["fileUrls"][localtest]["completes"]["*"]

        for platform in history["platforms"]:
            if "alias" in history["platforms"][platform]:
                continue
            if platform not in builds:
                builds[platform] = dict()
            for locale in history["platforms"][platform]["locales"]:
                if locale not in builds[platform]:
                    builds[platform][locale] = dict()
                buildid = history["platforms"][platform]["locales"][locale]["buildID"]
                url = url_pattern.replace(
                    "%OS_FTP%", FTP_PLATFORM_MAP[platform]
                ).replace("%LOCALE%", locale)
                builds[platform][locale][partial_mar_key] = {
                    "buildid": buildid,
                    "mar_url": url,
                    "previousVersion": version,
                    "previousBuildNumber": release["buildNumber"],
                    "product": product,
                }
    return builds