# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import logging import redo import requests from gecko_taskgraph.util.scriptworker import ( BALROG_SCOPE_ALIAS_TO_PROJECT, BALROG_SERVER_SCOPES, ) logger = logging.getLogger(__name__) PLATFORM_RENAMES = { "windows2012-32": "win32", "windows2012-64": "win64", "windows2012-aarch64": "win64-aarch64", "osx-cross": "macosx64", "osx": "macosx64", } BALROG_PLATFORM_MAP = { "linux": ["Linux_x86-gcc3"], "linux32": ["Linux_x86-gcc3"], "linux64": ["Linux_x86_64-gcc3"], "linux64-asan-reporter": ["Linux_x86_64-gcc3-asan"], "macosx64": [ "Darwin_x86_64-gcc3-u-i386-x86_64", "Darwin_x86-gcc3-u-i386-x86_64", "Darwin_aarch64-gcc3", "Darwin_x86-gcc3", "Darwin_x86_64-gcc3", ], "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"], "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"], "win64-asan-reporter": ["WINNT_x86_64-msvc-x64-asan"], "win64-aarch64": [ "WINNT_aarch64-msvc-aarch64", ], } FTP_PLATFORM_MAP = { "Darwin_x86-gcc3": "mac", "Darwin_x86-gcc3-u-i386-x86_64": "mac", "Darwin_x86_64-gcc3": "mac", "Darwin_x86_64-gcc3-u-i386-x86_64": "mac", "Darwin_aarch64-gcc3": "mac", "Linux_x86-gcc3": "linux-i686", "Linux_x86_64-gcc3": "linux-x86_64", "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter", "WINNT_x86_64-msvc-x64-asan": "win64-asan-reporter", "WINNT_x86-msvc": "win32", "WINNT_x86-msvc-x64": "win32", "WINNT_x86-msvc-x86": "win32", "WINNT_x86_64-msvc": "win64", "WINNT_x86_64-msvc-x64": "win64", "WINNT_aarch64-msvc-aarch64": "win64-aarch64", } def get_balrog_platform_name(platform): """Convert build platform names into balrog platform names. Remove known values instead to catch aarch64 and other platforms that may be added. """ removals = ["-devedition", "-shippable"] for remove in removals: platform = platform.replace(remove, "") return PLATFORM_RENAMES.get(platform, platform) def _sanitize_platform(platform): platform = get_balrog_platform_name(platform) if platform not in BALROG_PLATFORM_MAP: return platform return BALROG_PLATFORM_MAP[platform][0] def get_builds(release_history, platform, locale): """Examine cached balrog release history and return the list of builds we need to generate diffs from""" platform = _sanitize_platform(platform) return release_history.get(platform, {}).get(locale, {}) def get_partials_artifacts_from_params(release_history, platform, locale): platform = _sanitize_platform(platform) return [ (artifact, details.get("previousVersion", None)) for artifact, details in release_history.get(platform, {}) .get(locale, {}) .items() ] def get_partials_info_from_params(release_history, platform, locale): platform = _sanitize_platform(platform) artifact_map = {} for k in release_history.get(platform, {}).get(locale, {}): details = release_history[platform][locale][k] attributes = ("buildid", "previousBuildNumber", "previousVersion") artifact_map[k] = { attr: details[attr] for attr in attributes if attr in details } return artifact_map def _retry_on_http_errors(url, verify, params, errors): if params: params_str = "&".join("=".join([k, str(v)]) for k, v in params.items()) else: params_str = "" logger.info("Connecting to %s?%s", url, params_str) for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): try: req = requests.get(url, verify=verify, params=params, timeout=10) req.raise_for_status() return req except requests.HTTPError as e: if e.response.status_code in errors: logger.exception( "Got HTTP %s trying to reach %s", e.response.status_code, url ) else: raise else: raise Exception(f"Cannot connect to {url}!") def get_sorted_releases(product, branch): """Returns a list of release names from Balrog. :param product: product name, AKA appName :param branch: branch name, e.g. mozilla-central :return: a sorted list of release names, most recent first. """ url = f"{_get_balrog_api_root(branch)}/releases" params = { "product": product, # Adding -nightly-2 (2 stands for the beginning of build ID # based on date) should filter out release and latest blobs. # This should be changed to -nightly-3 in 3000 ;) "name_prefix": f"{product}-{branch}-nightly-2", "names_only": True, } req = _retry_on_http_errors(url=url, verify=True, params=params, errors=[500]) releases = req.json()["names"] releases = sorted(releases, reverse=True) return releases def get_release_builds(release, branch): url = f"{_get_balrog_api_root(branch)}/releases/{release}" req = _retry_on_http_errors(url=url, verify=True, params=None, errors=[500]) return req.json() def _get_balrog_api_root(branch): # Query into the scopes scriptworker uses to make sure we check against the same balrog server # That our jobs would use. scope = None for alias, projects in BALROG_SCOPE_ALIAS_TO_PROJECT: if branch in projects and alias in BALROG_SERVER_SCOPES: scope = BALROG_SERVER_SCOPES[alias] break else: scope = BALROG_SERVER_SCOPES["default"] if scope == "balrog:server:dep": return "https://stage.balrog.nonprod.cloudops.mozgcp.net/api/v1" return "https://aus5.mozilla.org/api/v1" def find_localtest(fileUrls): for channel in fileUrls: if "-localtest" in channel: return channel def populate_release_history( product, branch, maxbuilds=4, maxsearch=10, partial_updates=None ): # Assuming we are using release branches when we know the list of previous # releases in advance if partial_updates is not None: return _populate_release_history( product, branch, partial_updates=partial_updates ) return _populate_nightly_history( product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch ) def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10): """Find relevant releases in Balrog Not all releases have all platforms and locales, due to Taskcluster migration. Args: product (str): capitalized product name, AKA appName, e.g. Firefox branch (str): branch name (mozilla-central) maxbuilds (int): Maximum number of historical releases to populate maxsearch(int): Traverse at most this many releases, to avoid working through the entire history. Returns: json object based on data from balrog api results = { 'platform1': { 'locale1': { 'buildid1': mar_url, 'buildid2': mar_url, 'buildid3': mar_url, }, 'locale2': { 'target.partial-1.mar': {'buildid1': 'mar_url'}, } }, 'platform2': { } } """ last_releases = get_sorted_releases(product, branch) partial_mar_tmpl = "target.partial-{}.mar" builds = dict() for release in last_releases[:maxsearch]: # maxbuilds in all categories, don't make any more queries full = len(builds) > 0 and all( len(builds[platform][locale]) >= maxbuilds for platform in builds for locale in builds[platform] ) if full: break history = get_release_builds(release, branch) for platform in history["platforms"]: if "alias" in history["platforms"][platform]: continue if platform not in builds: builds[platform] = dict() for locale in history["platforms"][platform]["locales"]: if locale not in builds[platform]: builds[platform][locale] = dict() if len(builds[platform][locale]) >= maxbuilds: continue if "buildID" not in history["platforms"][platform]["locales"][locale]: continue buildid = history["platforms"][platform]["locales"][locale]["buildID"] if ( "completes" not in history["platforms"][platform]["locales"][locale] or len( history["platforms"][platform]["locales"][locale]["completes"] ) == 0 ): continue url = history["platforms"][platform]["locales"][locale]["completes"][0][ "fileUrl" ] nextkey = len(builds[platform][locale]) + 1 builds[platform][locale][partial_mar_tmpl.format(nextkey)] = { "buildid": buildid, "mar_url": url, } return builds def _populate_release_history(product, branch, partial_updates): builds = dict() for version, release in partial_updates.items(): prev_release_blob = "{product}-{version}-build{build_number}".format( product=product, version=version, build_number=release["buildNumber"] ) partial_mar_key = f"target-{version}.partial.mar" history = get_release_builds(prev_release_blob, branch) # use one of the localtest channels to avoid relying on bouncer localtest = find_localtest(history["fileUrls"]) url_pattern = history["fileUrls"][localtest]["completes"]["*"] for platform in history["platforms"]: if "alias" in history["platforms"][platform]: continue if platform not in builds: builds[platform] = dict() for locale in history["platforms"][platform]["locales"]: if locale not in builds[platform]: builds[platform][locale] = dict() buildid = history["platforms"][platform]["locales"][locale]["buildID"] url = url_pattern.replace( "%OS_FTP%", FTP_PLATFORM_MAP[platform] ).replace("%LOCALE%", locale) builds[platform][locale][partial_mar_key] = { "buildid": buildid, "mar_url": url, "previousVersion": version, "previousBuildNumber": release["buildNumber"], "product": product, } return builds