Adding upstream version 115.7.0esr.upstream/115.7.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 297 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/util/partials.py b/taskcluster/gecko_taskgraph/util/partials.py
new file mode 100644
index 0000000000..1a3affcc42
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/util/partials.py
@@ -0,0 +1,297 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+import redo
+import requests
+
+from gecko_taskgraph.util.scriptworker import (
+    BALROG_SCOPE_ALIAS_TO_PROJECT,
+    BALROG_SERVER_SCOPES,
+)
+
+logger = logging.getLogger(__name__)
+
+PLATFORM_RENAMES = {
+    "windows2012-32": "win32",
+    "windows2012-64": "win64",
+    "windows2012-aarch64": "win64-aarch64",
+    "osx-cross": "macosx64",
+    "osx": "macosx64",
+}
+
+BALROG_PLATFORM_MAP = {
+    "linux": ["Linux_x86-gcc3"],
+    "linux32": ["Linux_x86-gcc3"],
+    "linux64": ["Linux_x86_64-gcc3"],
+    "linux64-asan-reporter": ["Linux_x86_64-gcc3-asan"],
+    "macosx64": [
+        "Darwin_x86_64-gcc3-u-i386-x86_64",
+        "Darwin_x86-gcc3-u-i386-x86_64",
+        "Darwin_aarch64-gcc3",
+        "Darwin_x86-gcc3",
+        "Darwin_x86_64-gcc3",
+    ],
+    "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"],
+    "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"],
+    "win64-asan-reporter": ["WINNT_x86_64-msvc-x64-asan"],
+    "win64-aarch64": [
+        "WINNT_aarch64-msvc-aarch64",
+    ],
+}
+
+FTP_PLATFORM_MAP = {
+    "Darwin_x86-gcc3": "mac",
+    "Darwin_x86-gcc3-u-i386-x86_64": "mac",
+    "Darwin_x86_64-gcc3": "mac",
+    "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
+    "Darwin_aarch64-gcc3": "mac",
+    "Linux_x86-gcc3": "linux-i686",
+    "Linux_x86_64-gcc3": "linux-x86_64",
+    "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter",
+    "WINNT_x86_64-msvc-x64-asan": "win64-asan-reporter",
+    "WINNT_x86-msvc": "win32",
+    "WINNT_x86-msvc-x64": "win32",
+    "WINNT_x86-msvc-x86": "win32",
+    "WINNT_x86_64-msvc": "win64",
+    "WINNT_x86_64-msvc-x64": "win64",
+    "WINNT_aarch64-msvc-aarch64": "win64-aarch64",
+}
+
+
+def get_balrog_platform_name(platform):
+    """Convert build platform names into balrog platform names.
+
+    Remove known values instead to catch aarch64 and other platforms
+    that may be added.
+    """
+    removals = ["-devedition", "-shippable"]
+    for remove in removals:
+        platform = platform.replace(remove, "")
+    return PLATFORM_RENAMES.get(platform, platform)
+
+
+def _sanitize_platform(platform):
+    platform = get_balrog_platform_name(platform)
+    if platform not in BALROG_PLATFORM_MAP:
+        return platform
+    return BALROG_PLATFORM_MAP[platform][0]
+
+
+def get_builds(release_history, platform, locale):
+    """Examine cached balrog release history and return the list of
+    builds we need to generate diffs from"""
+    platform = _sanitize_platform(platform)
+    return release_history.get(platform, {}).get(locale, {})
+
+
+def get_partials_artifacts_from_params(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+    return [
+        (artifact, details.get("previousVersion", None))
+        for artifact, details in release_history.get(platform, {})
+        .get(locale, {})
+        .items()
+    ]
+
+
+def get_partials_info_from_params(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+
+    artifact_map = {}
+    for k in release_history.get(platform, {}).get(locale, {}):
+        details = release_history[platform][locale][k]
+        attributes = ("buildid", "previousBuildNumber", "previousVersion")
+        artifact_map[k] = {
+            attr: details[attr] for attr in attributes if attr in details
+        }
+    return artifact_map
+
+
+def _retry_on_http_errors(url, verify, params, errors):
+    if params:
+        params_str = "&".join("=".join([k, str(v)]) for k, v in params.items())
+    else:
+        params_str = ""
+    logger.info("Connecting to %s?%s", url, params_str)
+    for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
+        try:
+            req = requests.get(url, verify=verify, params=params, timeout=10)
+            req.raise_for_status()
+            return req
+        except requests.HTTPError as e:
+            if e.response.status_code in errors:
+                logger.exception(
+                    "Got HTTP %s trying to reach %s", e.response.status_code, url
+                )
+            else:
+                raise
+    else:
+        raise Exception(f"Cannot connect to {url}!")
+
+
+def get_sorted_releases(product, branch):
+    """Returns a list of release names from Balrog.
+    :param product: product name, AKA appName
+    :param branch: branch name, e.g. mozilla-central
+    :return: a sorted list of release names, most recent first.
+    """
+    url = f"{_get_balrog_api_root(branch)}/releases"
+    params = {
+        "product": product,
+        # Adding -nightly-2 (2 stands for the beginning of build ID
+        # based on date) should filter out release and latest blobs.
+        # This should be changed to -nightly-3 in 3000 ;)
+        "name_prefix": f"{product}-{branch}-nightly-2",
+        "names_only": True,
+    }
+    req = _retry_on_http_errors(url=url, verify=True, params=params, errors=[500])
+    releases = req.json()["names"]
+    releases = sorted(releases, reverse=True)
+    return releases
+
+
+def get_release_builds(release, branch):
+    url = f"{_get_balrog_api_root(branch)}/releases/{release}"
+    req = _retry_on_http_errors(url=url, verify=True, params=None, errors=[500])
+    return req.json()
+
+
+def _get_balrog_api_root(branch):
+    # Query into the scopes scriptworker uses to make sure we check against the same balrog server
+    # That our jobs would use.
+    scope = None
+    for alias, projects in BALROG_SCOPE_ALIAS_TO_PROJECT:
+        if branch in projects and alias in BALROG_SERVER_SCOPES:
+            scope = BALROG_SERVER_SCOPES[alias]
+            break
+    else:
+        scope = BALROG_SERVER_SCOPES["default"]
+
+    if scope == "balrog:server:dep":
+        return "https://stage.balrog.nonprod.cloudops.mozgcp.net/api/v1"
+    return "https://aus5.mozilla.org/api/v1"
+
+
+def find_localtest(fileUrls):
+    for channel in fileUrls:
+        if "-localtest" in channel:
+            return channel
+
+
+def populate_release_history(
+    product, branch, maxbuilds=4, maxsearch=10, partial_updates=None
+):
+    # Assuming we are using release branches when we know the list of previous
+    # releases in advance
+    if partial_updates is not None:
+        return _populate_release_history(
+            product, branch, partial_updates=partial_updates
+        )
+    return _populate_nightly_history(
+        product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch
+    )
+
+
+def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10):
+    """Find relevant releases in Balrog
+    Not all releases have all platforms and locales, due
+    to Taskcluster migration.
+
+        Args:
+            product (str): capitalized product name, AKA appName, e.g. Firefox
+            branch (str): branch name (mozilla-central)
+            maxbuilds (int): Maximum number of historical releases to populate
+            maxsearch(int): Traverse at most this many releases, to avoid
+                working through the entire history.
+        Returns:
+            json object based on data from balrog api
+
+            results = {
+                'platform1': {
+                    'locale1': {
+                        'buildid1': mar_url,
+                        'buildid2': mar_url,
+                        'buildid3': mar_url,
+                    },
+                    'locale2': {
+                        'target.partial-1.mar': {'buildid1': 'mar_url'},
+                    }
+                },
+                'platform2': {
+                }
+            }
+    """
+    last_releases = get_sorted_releases(product, branch)
+
+    partial_mar_tmpl = "target.partial-{}.mar"
+
+    builds = dict()
+    for release in last_releases[:maxsearch]:
+        # maxbuilds in all categories, don't make any more queries
+        full = len(builds) > 0 and all(
+            len(builds[platform][locale]) >= maxbuilds
+            for platform in builds
+            for locale in builds[platform]
+        )
+        if full:
+            break
+        history = get_release_builds(release, branch)
+
+        for platform in history["platforms"]:
+            if "alias" in history["platforms"][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history["platforms"][platform]["locales"]:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                if len(builds[platform][locale]) >= maxbuilds:
+                    continue
+                buildid = history["platforms"][platform]["locales"][locale]["buildID"]
+                url = history["platforms"][platform]["locales"][locale]["completes"][0][
+                    "fileUrl"
+                ]
+                nextkey = len(builds[platform][locale]) + 1
+                builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
+                    "buildid": buildid,
+                    "mar_url": url,
+                }
+    return builds
+
+
+def _populate_release_history(product, branch, partial_updates):
+    builds = dict()
+    for version, release in partial_updates.items():
+        prev_release_blob = "{product}-{version}-build{build_number}".format(
+            product=product, version=version, build_number=release["buildNumber"]
+        )
+        partial_mar_key = f"target-{version}.partial.mar"
+        history = get_release_builds(prev_release_blob, branch)
+        # use one of the localtest channels to avoid relying on bouncer
+        localtest = find_localtest(history["fileUrls"])
+        url_pattern = history["fileUrls"][localtest]["completes"]["*"]
+
+        for platform in history["platforms"]:
+            if "alias" in history["platforms"][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history["platforms"][platform]["locales"]:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                buildid = history["platforms"][platform]["locales"][locale]["buildID"]
+                url = url_pattern.replace(
+                    "%OS_FTP%", FTP_PLATFORM_MAP[platform]
+                ).replace("%LOCALE%", locale)
+                builds[platform][locale][partial_mar_key] = {
+                    "buildid": buildid,
+                    "mar_url": url,
+                    "previousVersion": version,
+                    "previousBuildNumber": release["buildNumber"],
+                    "product": product,
+                }
+    return builds