diff options
Diffstat (limited to 'testing/mozharness/scripts/release')
3 files changed, 1107 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/release/bouncer_check.py b/testing/mozharness/scripts/release/bouncer_check.py new file mode 100644 index 0000000000..7a7e39b274 --- /dev/null +++ b/testing/mozharness/scripts/release/bouncer_check.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" bouncer_check.py + +A script to check HTTP statuses of Bouncer products to be shipped. +""" + +import os +import sys + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.script import BaseScript +from mozharness.mozilla.automation import EXIT_STATUS_DICT, TBPL_FAILURE + +BOUNCER_URL_PATTERN = "{bouncer_prefix}?product={product}&os={os}&lang={lang}" + + +class BouncerCheck(BaseScript): + config_options = [ + [ + ["--version"], + { + "dest": "version", + "help": "Version of release, eg: 39.0b5", + }, + ], + [ + ["--product-field"], + { + "dest": "product_field", + "help": "Version field of release from product details, eg: LATEST_FIREFOX_VERSION", # NOQA: E501 + }, + ], + [ + ["--products-url"], + { + "dest": "products_url", + "help": "The URL of the current Firefox product versions", + "type": str, + "default": "https://product-details.mozilla.org/1.0/firefox_versions.json", + }, + ], + [ + ["--previous-version"], + { + "dest": "prev_versions", + "action": "extend", + "help": "Previous version(s)", + }, + ], + [ + ["--locale"], + { + "dest": "locales", + # Intentionally limited for several reasons: + # 1) faster to check + # 2) do not need to deal with situation when a new locale + # introduced and we do not have partials for it yet + # 3) it mimics the old Sentry behaviour that worked for ages + # 4) no need to handle ja-JP-mac + "default": ["en-US", "de", "it", "zh-TW"], + "action": "append", + "help": "List of locales to check.", + }, + ], + [ + ["-j", "--parallelization"], + { + "dest": "parallelization", + "default": 20, + "type": int, + "help": "Number of HTTP sessions running in parallel", + }, + ], + ] + + def __init__(self, require_config_file=True): + super(BouncerCheck, self).__init__( + config_options=self.config_options, + require_config_file=require_config_file, + config={ + "cdn_urls": [ + "download-installer.cdn.mozilla.net", + "download.cdn.mozilla.net", + "download.mozilla.org", + "archive.mozilla.org", + ], + }, + all_actions=[ + "check-bouncer", + ], + default_actions=[ + "check-bouncer", + ], + ) + + def _pre_config_lock(self, rw_config): + super(BouncerCheck, self)._pre_config_lock(rw_config) + + if "product_field" not in self.config: + return + + firefox_versions = self.load_json_url(self.config["products_url"]) + + if self.config["product_field"] not in firefox_versions: + self.fatal("Unknown Firefox label: {}".format(self.config["product_field"])) + self.config["version"] = firefox_versions[self.config["product_field"]] + self.log("Set Firefox version {}".format(self.config["version"])) + + def check_url(self, session, url): + from redo import retry + from requests.exceptions import HTTPError + + try: + from urllib.parse import urlparse + except ImportError: + # Python 2 + from urlparse import urlparse + + def do_check_url(): + self.log("Checking {}".format(url)) + r = session.head(url, verify=True, timeout=10, allow_redirects=True) + try: + r.raise_for_status() + except HTTPError: + self.error("FAIL: {}, status: {}".format(url, r.status_code)) + raise + + final_url = urlparse(r.url) + if final_url.scheme != "https": + self.error("FAIL: URL scheme is not https: {}".format(r.url)) + self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] + + if final_url.netloc not in self.config["cdn_urls"]: + self.error("FAIL: host not in allowed locations: {}".format(r.url)) + self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] + + try: + retry(do_check_url, sleeptime=3, max_sleeptime=10, attempts=3) + except HTTPError: + # The error was already logged above. + self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] + return + + def get_urls(self): + for product in self.config["products"].values(): + product_name = product["product-name"] % {"version": self.config["version"]} + for bouncer_platform in product["platforms"]: + for locale in self.config["locales"]: + url = BOUNCER_URL_PATTERN.format( + bouncer_prefix=self.config["bouncer_prefix"], + product=product_name, + os=bouncer_platform, + lang=locale, + ) + yield url + + for product in self.config.get("partials", {}).values(): + for prev_version in self.config.get("prev_versions", []): + product_name = product["product-name"] % { + "version": self.config["version"], + "prev_version": prev_version, + } + for bouncer_platform in product["platforms"]: + for locale in self.config["locales"]: + url = BOUNCER_URL_PATTERN.format( + bouncer_prefix=self.config["bouncer_prefix"], + product=product_name, + os=bouncer_platform, + lang=locale, + ) + yield url + + def check_bouncer(self): + import concurrent.futures as futures + + import requests + + session = requests.Session() + http_adapter = requests.adapters.HTTPAdapter( + pool_connections=self.config["parallelization"], + pool_maxsize=self.config["parallelization"], + ) + session.mount("https://", http_adapter) + session.mount("http://", http_adapter) + + with futures.ThreadPoolExecutor(self.config["parallelization"]) as e: + fs = [] + for url in self.get_urls(): + fs.append(e.submit(self.check_url, session, url)) + for f in futures.as_completed(fs): + f.result() + + +if __name__ == "__main__": + BouncerCheck().run_and_exit() diff --git a/testing/mozharness/scripts/release/generate-checksums.py b/testing/mozharness/scripts/release/generate-checksums.py new file mode 100644 index 0000000000..ae092ae4de --- /dev/null +++ b/testing/mozharness/scripts/release/generate-checksums.py @@ -0,0 +1,263 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import binascii +import hashlib +import os +import re +import sys +from multiprocessing.pool import ThreadPool + +import six + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.mozilla.checksums import parse_checksums_file +from mozharness.mozilla.merkle import MerkleTree + + +class ChecksumsGenerator(BaseScript, VirtualenvMixin): + config_options = [ + [ + ["--stage-product"], + { + "dest": "stage_product", + "help": "Name of product used in file server's directory structure, " + "e.g.: firefox, mobile", + }, + ], + [ + ["--version"], + { + "dest": "version", + "help": "Version of release, e.g.: 59.0b5", + }, + ], + [ + ["--build-number"], + { + "dest": "build_number", + "help": "Build number of release, e.g.: 2", + }, + ], + [ + ["--bucket-name"], + { + "dest": "bucket_name", + "help": "Full bucket name e.g.: moz-fx-productdelivery-pr-38b5-productdelivery.", + }, + ], + [ + ["-j", "--parallelization"], + { + "dest": "parallelization", + "default": 20, + "type": int, + "help": "Number of checksums file to download concurrently", + }, + ], + [ + ["--branch"], + { + "dest": "branch", + "help": "dummy option", + }, + ], + [ + ["--build-pool"], + { + "dest": "build_pool", + "help": "dummy option", + }, + ], + ] + virtualenv_config_options + + def __init__(self): + BaseScript.__init__( + self, + config_options=self.config_options, + require_config_file=False, + config={ + "virtualenv_modules": [ + "boto", + ], + "virtualenv_path": "venv", + }, + all_actions=[ + "create-virtualenv", + "collect-individual-checksums", + "create-big-checksums", + "create-summary", + ], + default_actions=[ + "create-virtualenv", + "collect-individual-checksums", + "create-big-checksums", + "create-summary", + ], + ) + + self.checksums = {} + self.file_prefix = self._get_file_prefix() + + def _pre_config_lock(self, rw_config): + super(ChecksumsGenerator, self)._pre_config_lock(rw_config) + + # These defaults are set here rather in the config because default + # lists cannot be completely overidden, only appended to. + if not self.config.get("formats"): + self.config["formats"] = ["sha512", "sha256"] + + if not self.config.get("includes"): + self.config["includes"] = [ + r"^.*\.tar\.bz2$", + r"^.*\.tar\.xz$", + r"^.*\.snap$", + r"^.*\.dmg$", + r"^.*\.pkg$", + r"^.*\.bundle$", + r"^.*\.mar$", + r"^.*Setup.*\.exe$", + r"^.*Installer\.exe$", + r"^.*\.msi$", + r"^.*\.xpi$", + r"^.*fennec.*\.apk$", + r"^.*/jsshell.*$", + ] + + def _get_file_prefix(self): + return "pub/{}/candidates/{}-candidates/build{}/".format( + self.config["stage_product"], + self.config["version"], + self.config["build_number"], + ) + + def _get_sums_filename(self, format_): + return "{}SUMS".format(format_.upper()) + + def _get_summary_filename(self, format_): + return "{}SUMMARY".format(format_.upper()) + + def _get_hash_function(self, format_): + if format_ in ("sha256", "sha384", "sha512"): + return getattr(hashlib, format_) + else: + self.fatal("Unsupported format {}".format(format_)) + + def _get_bucket(self): + self.activate_virtualenv() + from boto import connect_s3 + + self.info("Connecting to S3") + conn = connect_s3(anon=True, host="storage.googleapis.com") + self.info("Connecting to bucket {}".format(self.config["bucket_name"])) + self.bucket = conn.get_bucket(self.config["bucket_name"]) + return self.bucket + + def collect_individual_checksums(self): + """This step grabs all of the small checksums files for the release, + filters out any unwanted files from within them, and adds the remainder + to self.checksums for subsequent steps to use.""" + bucket = self._get_bucket() + self.info("File prefix is: {}".format(self.file_prefix)) + + # temporary holding place for checksums + raw_checksums = [] + + def worker(item): + self.debug("Downloading {}".format(item)) + sums = bucket.get_key(item).get_contents_as_string() + raw_checksums.append(sums) + + def find_checksums_files(): + self.info("Getting key names from bucket") + checksum_files = {"beets": [], "checksums": []} + for key in bucket.list(prefix=self.file_prefix): + if key.key.endswith(".checksums"): + self.debug("Found checksums file: {}".format(key.key)) + checksum_files["checksums"].append(key.key) + elif key.key.endswith(".beet"): + self.debug("Found beet file: {}".format(key.key)) + checksum_files["beets"].append(key.key) + else: + self.debug("Ignoring non-checksums file: {}".format(key.key)) + if checksum_files["beets"]: + self.log("Using beet format") + return checksum_files["beets"] + else: + self.log("Using checksums format") + return checksum_files["checksums"] + + pool = ThreadPool(self.config["parallelization"]) + pool.map(worker, find_checksums_files()) + + for c in raw_checksums: + for f, info in six.iteritems(parse_checksums_file(c)): + for pattern in self.config["includes"]: + if re.search(pattern, f): + if f in self.checksums: + if info == self.checksums[f]: + self.debug( + "Duplicate checksum for file {}" + " but the data matches;" + " continuing...".format(f) + ) + continue + self.fatal( + "Found duplicate checksum entry for {}, " + "don't know which one to pick.".format(f) + ) + if not set(self.config["formats"]) <= set(info["hashes"]): + self.fatal("Missing necessary format for file {}".format(f)) + self.debug("Adding checksums for file: {}".format(f)) + self.checksums[f] = info + break + else: + self.debug("Ignoring checksums for file: {}".format(f)) + + def create_summary(self): + """ + This step computes a Merkle tree over the checksums for each format + and writes a file containing the head of the tree and inclusion proofs + for each file. + """ + for fmt in self.config["formats"]: + hash_fn = self._get_hash_function(fmt) + files = [fn for fn in sorted(self.checksums)] + data = [self.checksums[fn]["hashes"][fmt] for fn in files] + + tree = MerkleTree(hash_fn, data) + head = binascii.hexlify(tree.head()) + proofs = [ + binascii.hexlify(tree.inclusion_proof(i).to_rfc6962_bis()) + for i in range(len(files)) + ] + + summary = self._get_summary_filename(fmt) + self.info("Creating summary file: {}".format(summary)) + + content = "{} TREE_HEAD\n".format(head.decode("ascii")) + for i in range(len(files)): + content += "{} {}\n".format(proofs[i].decode("ascii"), files[i]) + + self.write_to_file(summary, content) + + def create_big_checksums(self): + for fmt in self.config["formats"]: + sums = self._get_sums_filename(fmt) + self.info("Creating big checksums file: {}".format(sums)) + with open(sums, "w+") as output_file: + for fn in sorted(self.checksums): + output_file.write( + "{} {}\n".format( + self.checksums[fn]["hashes"][fmt].decode("ascii"), fn + ) + ) + + +if __name__ == "__main__": + myScript = ChecksumsGenerator() + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/release/update-verify-config-creator.py b/testing/mozharness/scripts/release/update-verify-config-creator.py new file mode 100644 index 0000000000..9de0175577 --- /dev/null +++ b/testing/mozharness/scripts/release/update-verify-config-creator.py @@ -0,0 +1,642 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import math +import os +import pprint +import re +import sys + +from looseversion import LooseVersion +from mozilla_version.gecko import GeckoVersion +from mozilla_version.version import VersionType +from six.moves.urllib.parse import urljoin + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.log import DEBUG, FATAL, INFO, WARNING +from mozharness.base.script import BaseScript + + +# ensure all versions are 3 part (i.e. 99.1.0) +# ensure all text (i.e. 'esr') is in the last part +class CompareVersion(LooseVersion): + version = "" + + def __init__(self, versionMap): + parts = versionMap.split(".") + # assume version is 99.9.0, look for 99.0 + if len(parts) == 2: + intre = re.compile("([0-9.]+)(.*)") + match = intre.match(parts[-1]) + if match: + parts[-1] = match.group(1) + parts.append("0%s" % match.group(2)) + else: + parts.append("0") + self.version = ".".join(parts) + LooseVersion(versionMap) + + +def is_triangualar(x): + """Check if a number is triangular (0, 1, 3, 6, 10, 15, ...) + see: https://en.wikipedia.org/wiki/Triangular_number#Triangular_roots_and_tests_for_triangular_numbers # noqa + + >>> is_triangualar(0) + True + >>> is_triangualar(1) + True + >>> is_triangualar(2) + False + >>> is_triangualar(3) + True + >>> is_triangualar(4) + False + >>> all(is_triangualar(x) for x in [0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105]) + True + >>> all(not is_triangualar(x) for x in [4, 5, 8, 9, 11, 17, 25, 29, 39, 44, 59, 61, 72, 98, 112]) + True + """ + # pylint --py3k W1619 + n = (math.sqrt(8 * x + 1) - 1) / 2 + return n == int(n) + + +class UpdateVerifyConfigCreator(BaseScript): + config_options = [ + [ + ["--product"], + { + "dest": "product", + "help": "Product being tested, as used in the update URL and filenames. Eg: firefox", # NOQA: E501 + }, + ], + [ + ["--stage-product"], + { + "dest": "stage_product", + "help": "Product being tested, as used in stage directories and ship it" + "If not passed this is assumed to be the same as product.", + }, + ], + [ + ["--app-name"], + { + "dest": "app_name", + "help": "App name being tested. Eg: browser", + }, + ], + [ + ["--branch-prefix"], + { + "dest": "branch_prefix", + "help": "Prefix of release branch names. Eg: mozilla, comm", + }, + ], + [ + ["--channel"], + { + "dest": "channel", + "help": "Channel to run update verify against", + }, + ], + [ + ["--aus-server"], + { + "dest": "aus_server", + "default": "https://aus5.mozilla.org", + "help": "AUS server to run update verify against", + }, + ], + [ + ["--to-version"], + { + "dest": "to_version", + "help": "The version of the release being updated to. Eg: 59.0b5", + }, + ], + [ + ["--to-app-version"], + { + "dest": "to_app_version", + "help": "The in-app version of the release being updated to. Eg: 59.0", + }, + ], + [ + ["--to-display-version"], + { + "dest": "to_display_version", + "help": "The human-readable version of the release being updated to. Eg: 59.0 Beta 9", # NOQA: E501 + }, + ], + [ + ["--to-build-number"], + { + "dest": "to_build_number", + "help": "The build number of the release being updated to", + }, + ], + [ + ["--to-buildid"], + { + "dest": "to_buildid", + "help": "The buildid of the release being updated to", + }, + ], + [ + ["--to-revision"], + { + "dest": "to_revision", + "help": "The revision that the release being updated to was built against", + }, + ], + [ + ["--partial-version"], + { + "dest": "partial_versions", + "default": [], + "action": "append", + "help": "A previous release version that is expected to receive a partial update. " + "Eg: 59.0b4. May be specified multiple times.", + }, + ], + [ + ["--last-watershed"], + { + "dest": "last_watershed", + "help": "The earliest version to include in the update verify config. Eg: 57.0b10", + }, + ], + [ + ["--include-version"], + { + "dest": "include_versions", + "default": [], + "action": "append", + "help": "Only include versions that match one of these regexes. " + "May be passed multiple times", + }, + ], + [ + ["--mar-channel-id-override"], + { + "dest": "mar_channel_id_options", + "default": [], + "action": "append", + "help": "A version regex and channel id string to override those versions with." + "Eg: ^\\d+\\.\\d+(\\.\\d+)?$,firefox-mozilla-beta,firefox-mozilla-release " + "will set accepted mar channel ids to 'firefox-mozilla-beta' and " + "'firefox-mozilla-release for x.y and x.y.z versions. " + "May be passed multiple times", + }, + ], + [ + ["--override-certs"], + { + "dest": "override_certs", + "default": None, + "help": "Certs to override the updater with prior to running update verify." + "If passed, should be one of: dep, nightly, release" + "If not passed, no certificate overriding will be configured", + }, + ], + [ + ["--platform"], + { + "dest": "platform", + "help": "The platform to generate the update verify config for, in FTP-style", + }, + ], + [ + ["--updater-platform"], + { + "dest": "updater_platform", + "help": "The platform to run the updater on, in FTP-style." + "If not specified, this is assumed to be the same as platform", + }, + ], + [ + ["--archive-prefix"], + { + "dest": "archive_prefix", + "help": "The server/path to pull the current release from. " + "Eg: https://archive.mozilla.org/pub", + }, + ], + [ + ["--previous-archive-prefix"], + { + "dest": "previous_archive_prefix", + "help": "The server/path to pull the previous releases from" + "If not specified, this is assumed to be the same as --archive-prefix", + }, + ], + [ + ["--repo-path"], + { + "dest": "repo_path", + "help": ( + "The repository (relative to the hg server root) that the current " + "release was built from Eg: releases/mozilla-beta" + ), + }, + ], + [ + ["--output-file"], + { + "dest": "output_file", + "help": "Where to write the update verify config to", + }, + ], + [ + ["--product-details-server"], + { + "dest": "product_details_server", + "default": "https://product-details.mozilla.org", + "help": "Product Details server to pull previous release info from. " + "Using anything other than the production server is likely to " + "cause issues with update verify.", + }, + ], + [ + ["--hg-server"], + { + "dest": "hg_server", + "default": "https://hg.mozilla.org", + "help": "Mercurial server to pull various previous and current version info from", + }, + ], + [ + ["--full-check-locale"], + { + "dest": "full_check_locales", + "default": ["de", "en-US", "ru"], + "action": "append", + "help": "A list of locales to generate full update verify checks for", + }, + ], + ] + + def __init__(self): + BaseScript.__init__( + self, + config_options=self.config_options, + config={}, + all_actions=[ + "gather-info", + "create-config", + "write-config", + ], + default_actions=[ + "gather-info", + "create-config", + "write-config", + ], + ) + + def _pre_config_lock(self, rw_config): + super(UpdateVerifyConfigCreator, self)._pre_config_lock(rw_config) + + if "updater_platform" not in self.config: + self.config["updater_platform"] = self.config["platform"] + if "stage_product" not in self.config: + self.config["stage_product"] = self.config["product"] + if "previous_archive_prefix" not in self.config: + self.config["previous_archive_prefix"] = self.config["archive_prefix"] + self.config["archive_prefix"].rstrip("/") + self.config["previous_archive_prefix"].rstrip("/") + self.config["mar_channel_id_overrides"] = {} + for override in self.config["mar_channel_id_options"]: + pattern, override_str = override.split(",", 1) + self.config["mar_channel_id_overrides"][pattern] = override_str + + def _get_branch_url(self, branch_prefix, version): + version = GeckoVersion.parse(version) + branch = None + if version.version_type == VersionType.BETA: + branch = "releases/{}-beta".format(branch_prefix) + elif version.version_type == VersionType.ESR: + branch = "releases/{}-esr{}".format(branch_prefix, version.major_number) + elif version.version_type == VersionType.RELEASE: + if branch_prefix == "comm": + # Thunderbird does not have ESR releases, regular releases + # go in an ESR branch + branch = "releases/{}-esr{}".format(branch_prefix, version.major_number) + else: + branch = "releases/{}-release".format(branch_prefix) + if not branch: + raise Exception("Cannot determine branch, cannot continue!") + + return branch + + def _get_update_paths(self): + from mozrelease.l10n import getPlatformLocales + from mozrelease.paths import getCandidatesDir + from mozrelease.platforms import ftp2infoFile + from mozrelease.versions import MozillaVersion + + self.update_paths = {} + + ret = self._retry_download( + "{}/1.0/{}.json".format( + self.config["product_details_server"], + self.config["stage_product"], + ), + "WARNING", + ) + releases = json.load(ret)["releases"] + for release_name, release_info in reversed( + sorted(releases.items(), key=lambda x: MozillaVersion(x[1]["version"])) + ): + # we need to use releases_name instead of release_info since esr + # string is included in the name. later we rely on this. + product, version = release_name.split("-", 1) + tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_")) + + # Exclude any releases that don't match one of our include version + # regexes. This is generally to avoid including versions from other + # channels. Eg: including betas when testing releases + for v in self.config["include_versions"]: + if re.match(v, version): + break + else: + self.log( + "Skipping release whose version doesn't match any " + "include_version pattern: %s" % release_name, + level=INFO, + ) + continue + + # We also have to trim out previous releases that aren't in the same + # product line, too old, etc. + if self.config["stage_product"] != product: + self.log( + "Skipping release that doesn't match product name: %s" + % release_name, + level=INFO, + ) + continue + if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]): + self.log( + "Skipping release that's behind the last watershed: %s" + % release_name, + level=INFO, + ) + continue + if version == self.config["to_version"]: + self.log( + "Skipping release that is the same as to version: %s" + % release_name, + level=INFO, + ) + continue + if MozillaVersion(version) > MozillaVersion(self.config["to_version"]): + self.log( + "Skipping release that's newer than to version: %s" % release_name, + level=INFO, + ) + continue + + if version in self.update_paths: + raise Exception("Found duplicate release for version: %s", version) + + # This is a crappy place to get buildids from, but we don't have a better one. + # This will start to fail if old info files are deleted. + info_file_url = "{}{}/{}_info.txt".format( + self.config["previous_archive_prefix"], + getCandidatesDir( + self.config["stage_product"], + version, + release_info["build_number"], + ), + ftp2infoFile(self.config["platform"]), + ) + self.log( + "Retrieving buildid from info file: %s" % info_file_url, level=DEBUG + ) + ret = self._retry_download(info_file_url, "WARNING") + buildID = ret.read().split(b"=")[1].strip().decode("utf-8") + + branch = self._get_branch_url(self.config["branch_prefix"], version) + + shipped_locales_url = urljoin( + self.config["hg_server"], + "{}/raw-file/{}/{}/locales/shipped-locales".format( + branch, + tag, + self.config["app_name"], + ), + ) + ret = self._retry_download(shipped_locales_url, "WARNING") + shipped_locales = ret.read().strip().decode("utf-8") + + app_version_url = urljoin( + self.config["hg_server"], + "{}/raw-file/{}/{}/config/version.txt".format( + branch, + tag, + self.config["app_name"], + ), + ) + app_version = ( + self._retry_download(app_version_url, "WARNING") + .read() + .strip() + .decode("utf-8") + ) + + self.log("Adding {} to update paths".format(version), level=INFO) + self.update_paths[version] = { + "appVersion": app_version, + "locales": getPlatformLocales(shipped_locales, self.config["platform"]), + "buildID": buildID, + } + for pattern, mar_channel_ids in self.config[ + "mar_channel_id_overrides" + ].items(): + if re.match(pattern, version): + self.update_paths[version]["marChannelIds"] = mar_channel_ids + + def gather_info(self): + from mozilla_version.gecko import GeckoVersion + + self._get_update_paths() + if self.update_paths: + self.log("Found update paths:", level=DEBUG) + self.log(pprint.pformat(self.update_paths), level=DEBUG) + elif GeckoVersion.parse(self.config["to_version"]) <= GeckoVersion.parse( + self.config["last_watershed"] + ): + self.log( + "Didn't find any update paths, but to_version {} is before the last_" + "watershed {}, generating empty config".format( + self.config["to_version"], + self.config["last_watershed"], + ), + level=WARNING, + ) + else: + self.log("Didn't find any update paths, cannot continue", level=FATAL) + + def create_config(self): + from mozrelease.l10n import getPlatformLocales + from mozrelease.paths import ( + getCandidatesDir, + getReleaseInstallerPath, + getReleasesDir, + ) + from mozrelease.platforms import ftp2updatePlatforms + from mozrelease.update_verify import UpdateVerifyConfig + from mozrelease.versions import getPrettyVersion + + candidates_dir = getCandidatesDir( + self.config["stage_product"], + self.config["to_version"], + self.config["to_build_number"], + ) + to_ = getReleaseInstallerPath( + self.config["product"], + self.config["product"].title(), + self.config["to_version"], + self.config["platform"], + locale="%locale%", + ) + to_path = "{}/{}".format(candidates_dir, to_) + + to_display_version = self.config.get("to_display_version") + if not to_display_version: + to_display_version = getPrettyVersion(self.config["to_version"]) + + self.update_verify_config = UpdateVerifyConfig( + product=self.config["product"].title(), + channel=self.config["channel"], + aus_server=self.config["aus_server"], + to=to_path, + to_build_id=self.config["to_buildid"], + to_app_version=self.config["to_app_version"], + to_display_version=to_display_version, + override_certs=self.config.get("override_certs"), + ) + + to_shipped_locales_url = urljoin( + self.config["hg_server"], + "{}/raw-file/{}/{}/locales/shipped-locales".format( + self.config["repo_path"], + self.config["to_revision"], + self.config["app_name"], + ), + ) + to_shipped_locales = ( + self._retry_download(to_shipped_locales_url, "WARNING") + .read() + .strip() + .decode("utf-8") + ) + to_locales = set( + getPlatformLocales(to_shipped_locales, self.config["platform"]) + ) + + completes_only_index = 0 + for fromVersion in reversed(sorted(self.update_paths, key=CompareVersion)): + from_ = self.update_paths[fromVersion] + locales = sorted(list(set(from_["locales"]).intersection(to_locales))) + appVersion = from_["appVersion"] + build_id = from_["buildID"] + mar_channel_IDs = from_.get("marChannelIds") + + # Use new build targets for Windows, but only on compatible + # versions (42+). See bug 1185456 for additional context. + if self.config["platform"] not in ("win32", "win64") or LooseVersion( + fromVersion + ) < LooseVersion("42.0"): + update_platform = ftp2updatePlatforms(self.config["platform"])[0] + else: + update_platform = ftp2updatePlatforms(self.config["platform"])[1] + + release_dir = getReleasesDir(self.config["stage_product"], fromVersion) + path_ = getReleaseInstallerPath( + self.config["product"], + self.config["product"].title(), + fromVersion, + self.config["platform"], + locale="%locale%", + ) + from_path = "{}/{}".format(release_dir, path_) + + updater_package = "{}/{}".format( + release_dir, + getReleaseInstallerPath( + self.config["product"], + self.config["product"].title(), + fromVersion, + self.config["updater_platform"], + locale="%locale%", + ), + ) + + # Exclude locales being full checked + quick_check_locales = [ + l for l in locales if l not in self.config["full_check_locales"] + ] + # Get the intersection of from and to full_check_locales + this_full_check_locales = [ + l for l in self.config["full_check_locales"] if l in locales + ] + + if fromVersion in self.config["partial_versions"]: + self.info( + "Generating configs for partial update checks for %s" % fromVersion + ) + self.update_verify_config.addRelease( + release=appVersion, + build_id=build_id, + locales=locales, + patch_types=["complete", "partial"], + from_path=from_path, + ftp_server_from=self.config["previous_archive_prefix"], + ftp_server_to=self.config["archive_prefix"], + mar_channel_IDs=mar_channel_IDs, + platform=update_platform, + updater_package=updater_package, + ) + else: + if this_full_check_locales and is_triangualar(completes_only_index): + self.info("Generating full check configs for %s" % fromVersion) + self.update_verify_config.addRelease( + release=appVersion, + build_id=build_id, + locales=this_full_check_locales, + from_path=from_path, + ftp_server_from=self.config["previous_archive_prefix"], + ftp_server_to=self.config["archive_prefix"], + mar_channel_IDs=mar_channel_IDs, + platform=update_platform, + updater_package=updater_package, + ) + # Quick test for other locales, no download + if len(quick_check_locales) > 0: + self.info("Generating quick check configs for %s" % fromVersion) + if not is_triangualar(completes_only_index): + # Assuming we skipped full check locales, using all locales + _locales = locales + else: + # Excluding full check locales from the quick check + _locales = quick_check_locales + self.update_verify_config.addRelease( + release=appVersion, + build_id=build_id, + locales=_locales, + platform=update_platform, + ) + completes_only_index += 1 + + def write_config(self): + # Needs to be opened in "bytes" mode because we perform relative seeks on it + with open(self.config["output_file"], "wb+") as fh: + self.update_verify_config.write(fh) + + +if __name__ == "__main__": + UpdateVerifyConfigCreator().run_and_exit() |