diff options
Diffstat (limited to 'python/mozrelease')
28 files changed, 3467 insertions, 0 deletions
diff --git a/python/mozrelease/.ruff.toml b/python/mozrelease/.ruff.toml new file mode 100644 index 0000000000..6459b1ce4a --- /dev/null +++ b/python/mozrelease/.ruff.toml @@ -0,0 +1,4 @@ +extend = "../../pyproject.toml" + +[isort] +known-first-party = ["mozrelease"] diff --git a/python/mozrelease/mozrelease/__init__.py b/python/mozrelease/mozrelease/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozrelease/mozrelease/__init__.py diff --git a/python/mozrelease/mozrelease/attribute_builds.py b/python/mozrelease/mozrelease/attribute_builds.py new file mode 100644 index 0000000000..094c70e1bf --- /dev/null +++ b/python/mozrelease/mozrelease/attribute_builds.py @@ -0,0 +1,214 @@ +#! /usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import json +import logging +import mmap +import os +import shutil +import struct +import sys +import tempfile +import urllib.parse +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") +log = logging.getLogger() + + +def write_attribution_data(filepath, data): + """Insert data into a prepared certificate in a signed PE file. + + Returns False if the file isn't a valid PE file, or if the necessary + certificate was not found. + + This function assumes that somewhere in the given file's certificate table + there exists a 1024-byte space which begins with the tag "__MOZCUSTOM__:". + The given data will be inserted into the file following this tag. + + We don't bother updating the optional header checksum. + Windows doesn't check it for executables, only drivers and certain DLL's. + """ + with open(filepath, "r+b") as file: + mapped = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_WRITE) + + # Get the location of the PE header and the optional header + pe_header_offset = struct.unpack("<I", mapped[0x3C:0x40])[0] + optional_header_offset = pe_header_offset + 24 + + # Look up the magic number in the optional header, + # so we know if we have a 32 or 64-bit executable. + # We need to know that so that we can find the data directories. + pe_magic_number = struct.unpack( + "<H", mapped[optional_header_offset : optional_header_offset + 2] + )[0] + if pe_magic_number == 0x10B: + # 32-bit + cert_dir_entry_offset = optional_header_offset + 128 + elif pe_magic_number == 0x20B: + # 64-bit. Certain header fields are wider. + cert_dir_entry_offset = optional_header_offset + 144 + else: + # Not any known PE format + mapped.close() + return False + + # The certificate table offset and length give us the valid range + # to search through for where we should put our data. + cert_table_offset = struct.unpack( + "<I", mapped[cert_dir_entry_offset : cert_dir_entry_offset + 4] + )[0] + cert_table_size = struct.unpack( + "<I", mapped[cert_dir_entry_offset + 4 : cert_dir_entry_offset + 8] + )[0] + + if cert_table_offset == 0 or cert_table_size == 0: + # The file isn't signed + mapped.close() + return False + + tag = b"__MOZCUSTOM__:" + tag_index = mapped.find( + tag, cert_table_offset, cert_table_offset + cert_table_size + ) + if tag_index == -1: + mapped.close() + return False + + # convert to quoted-url byte-string for insertion + data = urllib.parse.quote(data).encode("utf-8") + mapped[tag_index + len(tag) : tag_index + len(tag) + len(data)] = data + + return True + + +def validate_attribution_code(attribution): + log.info("Checking attribution %s" % attribution) + return_code = True + + if len(attribution) == 0: + log.error("Attribution code has 0 length") + return False + + # Set to match https://searchfox.org/mozilla-central/rev/a92ed79b0bc746159fc31af1586adbfa9e45e264/browser/components/attribution/AttributionCode.jsm#24 # noqa + MAX_LENGTH = 1010 + if len(attribution) > MAX_LENGTH: + log.error("Attribution code longer than %s chars" % MAX_LENGTH) + return_code = False + + # this leaves out empty values like 'foo=' + params = urllib.parse.parse_qsl(attribution) + used_keys = set() + for key, value in params: + # check for invalid keys + if key not in ( + "source", + "medium", + "campaign", + "content", + "experiment", + "variation", + "ua", + "dlsource", + ): + log.error("Invalid key %s" % key) + return_code = False + + # avoid ambiguity from repeated keys + if key in used_keys: + log.error("Repeated key %s" % key) + return_code = False + else: + used_keys.add(key) + + # TODO the service checks for valid source, should we do that here too ? + + # We have two types of attribution with different requirements: + # 1) Partner attribution, which requires a few UTM parameters sets + # 2) Attribution of vanilla builds, which only requires `dlsource` + # + # Perhaps in an ideal world we would check what type of build we're + # attributing to make sure that eg: partner builds don't get `dlsource` + # instead of what they actually want -- but the likelyhood of that + # happening is vanishingly small, so it's probably not worth doing. + if "dlsource" not in used_keys: + for key in ("source", "medium", "campaign", "content"): + if key not in used_keys: + return_code = False + + if return_code is False: + log.error( + "Either 'dlsource' must be provided, or all of: 'source', 'medium', 'campaign', and 'content'. Use '(not set)' if one of the latter is not needed." + ) + return return_code + + +def main(): + parser = argparse.ArgumentParser( + description="Add attribution to Windows installer(s).", + epilog=""" + By default, configuration from envvar ATTRIBUTION_CONFIG is used, with + expected format + [{"input": "in/abc.exe", "output": "out/def.exe", "attribution": "abcdef"}, + {"input": "in/ghi.exe", "output": "out/jkl.exe", "attribution": "ghijkl"}] + for 1 or more attributions. Or the script arguments may be used for a single attribution. + + The attribution code should be a string which is not url-encoded. + + If command line arguments are used instead, one or more `--input` parameters may be provided. + Each will be written to the `--output` directory provided to a file of the same name as the + input filename. All inputs will be attributed with the same `--attribution` code. + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--input", + default=[], + action="append", + help="Source installer to attribute; may be specified multiple times", + ) + parser.add_argument("--output", help="Location to write the attributed installers") + parser.add_argument("--attribution", help="Attribution code") + args = parser.parse_args() + + if os.environ.get("ATTRIBUTION_CONFIG"): + work = json.loads(os.environ["ATTRIBUTION_CONFIG"]) + elif args.input and args.output and args.attribution: + work = [] + for i in args.input: + fn = os.path.basename(i) + work.append( + { + "input": i, + "output": os.path.join(args.output, fn), + "attribution": args.attribution, + } + ) + else: + log.error("No configuration found. Set ATTRIBUTION_CONFIG or pass arguments.") + return 1 + + cached_code_checks = [] + for job in work: + if job["attribution"] not in cached_code_checks: + status = validate_attribution_code(job["attribution"]) + if status: + cached_code_checks.append(job["attribution"]) + else: + log.error("Failed attribution code check") + return 1 + + with tempfile.TemporaryDirectory() as td: + log.info("Attributing installer %s ..." % job["input"]) + tf = shutil.copy(job["input"], td) + if write_attribution_data(tf, job["attribution"]): + Path(job["output"]).parent.mkdir(parents=True, exist_ok=True) + shutil.move(tf, job["output"]) + log.info("Wrote %s" % job["output"]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/mozrelease/mozrelease/balrog.py b/python/mozrelease/mozrelease/balrog.py new file mode 100644 index 0000000000..31418d352e --- /dev/null +++ b/python/mozrelease/mozrelease/balrog.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def _generate_show_url(context, entry): + url = entry["url"] + return { + "actions": "showURL", + "openURL": url.format(**context), + } + + +def _generate_product_details(context, entry): + url = entry["url"] + return { + "detailsURL": url.format(**context), + "type": "minor", + } + + +_FIELD_TYPES = { + "show-url": _generate_show_url, + "product-details": _generate_product_details, +} + + +def _generate_conditions(context, entry): + if ( + "release-types" in entry + and context["release-type"] not in entry["release-types"] + ): + return None + if "blob-types" in entry and context["blob-type"] not in entry["blob-types"]: + return None + if "products" in entry and context["product"] not in entry["products"]: + return None + + conditions = {} + if "locales" in entry: + conditions["locales"] = entry["locales"] + if "versions" in entry: + conditions["versions"] = [ + version.format(**context) for version in entry["versions"] + ] + if "update-channel" in entry: + conditions["channels"] = [ + entry["update-channel"] + suffix + for suffix in ("", "-localtest", "-cdntest") + ] + if "build-ids" in entry: + conditions["buildIDs"] = [ + buildid.format(**context) for buildid in entry["build-ids"] + ] + return conditions + + +def generate_update_properties(context, config): + result = [] + for entry in config: + fields = _FIELD_TYPES[entry["type"]](context, entry) + conditions = _generate_conditions(context, entry.get("conditions", {})) + + if conditions is not None: + result.append( + { + "fields": fields, + "for": conditions, + } + ) + return result diff --git a/python/mozrelease/mozrelease/buglist_creator.py b/python/mozrelease/mozrelease/buglist_creator.py new file mode 100644 index 0000000000..8c7b8d0391 --- /dev/null +++ b/python/mozrelease/mozrelease/buglist_creator.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import re +from operator import itemgetter + +import requests +from mozilla_version.gecko import GeckoVersion +from taskcluster import Notify, optionsFromEnvironment + +BUGLIST_TEMPLATE = "* [Bugs since previous changeset]({url})\n" +BACKOUT_REGEX = re.compile(r"back(\s?)out|backed out|backing out", re.IGNORECASE) +BACKOUT_TEMPLATE = "* [Backouts since previous changeset]({url})\n" +BUGZILLA_BUGLIST_TEMPLATE = "https://bugzilla.mozilla.org/buglist.cgi?bug_id={bugs}" +BUG_NUMBER_REGEX = re.compile(r"bug \d+", re.IGNORECASE) +CHANGELOG_TO_FROM_STRING = "{product}_{version}_RELEASE" +CHANGESET_URL_TEMPLATE = ( + "{repo}/{logtype}" "?rev={to_version}+%25+{from_version}&revcount=1000" +) +FULL_CHANGESET_TEMPLATE = "* [Full Mercurial changelog]({url})\n" +LIST_DESCRIPTION_TEMPLATE = "Comparing Mercurial tag {from_version} to {to_version}:\n" +MAX_BUGS_IN_BUGLIST = 250 +MERCURIAL_TAGS_URL_TEMPLATE = "{repo}/json-tags" +NO_BUGS = "" # Return this when bug list can't be created +URL_SHORTENER_TEMPLATE = "https://bugzilla.mozilla.org/rest/bitly/shorten?url={url}" + +log = logging.getLogger(__name__) + + +def create_bugs_url(product, current_version, current_revision, repo=None): + """ + Creates list of bugs and backout bugs for release-drivers email + + :param release: dict -> containing information about release, from Ship-It + :return: str -> description of compared releases, with Bugzilla links + containing all bugs in changeset + """ + try: + # Extract the important data, ignore if beta1 release + if current_version.beta_number == 1: + # If the version is beta 1, don't make any links + return NO_BUGS + + if repo is None: + repo = get_repo_by_version(current_version) + # Get the tag version, for display purposes + current_version_tag = tag_version(product, current_version) + + # Get all Hg tags for this branch, determine the previous version + tag_url = MERCURIAL_TAGS_URL_TEMPLATE.format(repo=repo) + mercurial_tags_json = requests.get(tag_url).json() + previous_version_tag = get_previous_tag_version( + product, current_version, current_version_tag, mercurial_tags_json + ) + + # Get the changeset between these versions, parse for all unique bugs and backout bugs + resp = requests.get( + CHANGESET_URL_TEMPLATE.format( + repo=repo, + from_version=previous_version_tag, + to_version=current_revision, + logtype="json-log", + ) + ) + changeset_data = resp.json() + unique_bugs, unique_backout_bugs = get_bugs_in_changeset(changeset_data) + + # Return a descriptive string with links if any relevant bugs are found + if unique_bugs or unique_backout_bugs: + description = LIST_DESCRIPTION_TEMPLATE.format( + from_version=previous_version_tag, to_version=current_version_tag + ) + + if unique_bugs: + description += BUGLIST_TEMPLATE.format( + url=create_buglist_url(unique_bugs) + ) + if unique_backout_bugs: + description += BACKOUT_TEMPLATE.format( + url=create_buglist_url(unique_backout_bugs) + ) + + changeset_html = CHANGESET_URL_TEMPLATE.format( + repo=repo, + from_version=previous_version_tag, + to_version=current_revision, + logtype="log", + ) + description += FULL_CHANGESET_TEMPLATE.format(url=changeset_html) + + return description + else: + return NO_BUGS + + except Exception as err: + log.info(err) + return NO_BUGS + + +def get_bugs_in_changeset(changeset_data): + unique_bugs, unique_backout_bugs = set(), set() + for changeset in changeset_data["entries"]: + if is_excluded_change(changeset): + continue + + changeset_desc = changeset["desc"] + bug_re = BUG_NUMBER_REGEX.search(changeset_desc) + + if bug_re: + bug_number = bug_re.group().split(" ")[1] + + if is_backout_bug(changeset_desc): + unique_backout_bugs.add(bug_number) + else: + unique_bugs.add(bug_number) + + return unique_bugs, unique_backout_bugs + + +def is_excluded_change(changeset): + excluded_change_keywords = [ + "a=test-only", + "a=release", + ] + return any(keyword in changeset["desc"] for keyword in excluded_change_keywords) + + +def is_backout_bug(changeset_description): + return bool(BACKOUT_REGEX.search(changeset_description)) + + +def create_buglist_url(buglist): + return BUGZILLA_BUGLIST_TEMPLATE.format(bugs="%2C".join(buglist)) + + +def tag_version(product, version): + underscore_version = str(version).replace(".", "_") + return CHANGELOG_TO_FROM_STRING.format( + product=product.upper(), version=underscore_version + ) + + +def parse_tag_version(tag): + dot_version = ".".join(tag.split("_")[1:-1]) + return GeckoVersion.parse(dot_version) + + +def get_previous_tag_version( + product, + current_version, + current_version_tag, + mercurial_tags_json, +): + """ + Gets the previous hg version tag for the product and branch, given the current version tag + """ + + def _invalid_tag_filter(tag): + """Filters by product and removes incorrect major version + base, end releases""" + prod_major_version_re = r"^{product}_{major_version}".format( + product=product.upper(), major_version=current_version.major_number + ) + + return ( + "BASE" not in tag + and "END" not in tag + and "RELEASE" in tag + and re.match(prod_major_version_re, tag) + ) + + # Get rid of irrelevant tags, sort by date and extract the tag string + tags = { + (parse_tag_version(item["tag"]), item["tag"]) + for item in mercurial_tags_json["tags"] + if _invalid_tag_filter(item["tag"]) + } + # Add the current version to the list + tags.add((current_version, current_version_tag)) + tags = sorted(tags, key=lambda tag: tag[0]) + + # Find where the current version is and go back one to get the previous version + next_version_index = list(map(itemgetter(0), tags)).index(current_version) - 1 + + return tags[next_version_index][1] + + +def get_repo_by_version(version): + """ + Get the repo a given version is found on. + """ + if version.is_beta: + return "https://hg.mozilla.org/releases/mozilla-beta" + elif version.is_release: + return "https://hg.mozilla.org/releases/mozilla-release" + elif version.is_esr: + return "https://hg.mozilla.org/releases/mozilla-esr{}".format( + version.major_number + ) + else: + raise Exception( + "Unsupported version type {}: {}".format(version.version_type.name, version) + ) + + +def email_release_drivers( + addresses, + product, + version, + build_number, + repo, + revision, + task_group_id, +): + # Send an email to the mailing after the build + email_buglist_string = create_bugs_url(product, version, revision, repo=repo) + + content = """\ +A new build has been started: + +Commit: [{revision}]({repo}/rev/{revision}) +Task group: [{task_group_id}]({root_url}/tasks/groups/{task_group_id}) + +{email_buglist_string} +""".format( + repo=repo, + revision=revision, + root_url=os.environ["TASKCLUSTER_ROOT_URL"], + task_group_id=task_group_id, + email_buglist_string=email_buglist_string, + ) + + # On r-d, we prefix the subject of the email in order to simplify filtering + subject_prefix = "" + if product in {"fennec"}: + subject_prefix = "[mobile] " + if product in {"firefox", "devedition"}: + subject_prefix = "[desktop] " + + subject = "{} Build of {} {} build {}".format( + subject_prefix, product, version, build_number + ) + + # use proxy if configured, otherwise local credentials from env vars + if "TASKCLUSTER_PROXY_URL" in os.environ: + notify_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} + else: + notify_options = optionsFromEnvironment() + + notify = Notify(notify_options) + for address in addresses: + notify.email( + { + "address": address, + "subject": subject, + "content": content, + } + ) diff --git a/python/mozrelease/mozrelease/chunking.py b/python/mozrelease/mozrelease/chunking.py new file mode 100644 index 0000000000..8c45f74354 --- /dev/null +++ b/python/mozrelease/mozrelease/chunking.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from copy import copy + + +class ChunkingError(Exception): + pass + + +def getChunk(things, chunks, thisChunk): + if thisChunk > chunks: + raise ChunkingError( + "thisChunk (%d) is greater than total chunks (%d)" % (thisChunk, chunks) + ) + possibleThings = copy(things) + nThings = len(possibleThings) + for c in range(1, chunks + 1): + n = nThings // chunks + # If our things aren't evenly divisible by the number of chunks + # we need to append one more onto some of them + if c <= (nThings % chunks): + n += 1 + if c == thisChunk: + return possibleThings[0:n] + del possibleThings[0:n] diff --git a/python/mozrelease/mozrelease/l10n.py b/python/mozrelease/mozrelease/l10n.py new file mode 100644 index 0000000000..1e2a15878d --- /dev/null +++ b/python/mozrelease/mozrelease/l10n.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def getPlatformLocales(shipped_locales, platform): + platform_locales = [] + for line in shipped_locales.splitlines(): + locale = line.strip().split()[0] + # ja-JP-mac locale is a MacOS only locale + if locale == "ja-JP-mac" and not platform.startswith("mac"): + continue + # Skip the "ja" locale on MacOS + if locale == "ja" and platform.startswith("mac"): + continue + platform_locales.append(locale) + return platform_locales diff --git a/python/mozrelease/mozrelease/mach_commands.py b/python/mozrelease/mozrelease/mach_commands.py new file mode 100644 index 0000000000..e7c8da59fe --- /dev/null +++ b/python/mozrelease/mozrelease/mach_commands.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import sys + +from mach.decorators import Command, CommandArgument, SubCommand +from mozilla_version.gecko import GeckoVersion + + +@Command( + "release", + category="release", + description="Task that are part of the release process.", +) +def release(command_context): + """ + The release subcommands all relate to the release process. + """ + + +@SubCommand( + "release", + "buglist", + description="Generate list of bugs since the last release.", +) +@CommandArgument( + "--version", + required=True, + type=GeckoVersion.parse, + help="The version being built.", +) +@CommandArgument("--product", required=True, help="The product being built.") +@CommandArgument("--repo", help="The repo being built.") +@CommandArgument("--revision", required=True, help="The revision being built.") +def buglist(command_context, version, product, revision, repo): + setup_logging(command_context) + from mozrelease.buglist_creator import create_bugs_url + + print( + create_bugs_url( + product=product, + current_version=version, + current_revision=revision, + repo=repo, + ) + ) + + +@SubCommand( + "release", + "send-buglist-email", + description="Send an email with the bugs since the last release.", +) +@CommandArgument( + "--address", + required=True, + action="append", + dest="addresses", + help="The email address to send the bug list to " + "(may be specified more than once.", +) +@CommandArgument( + "--version", + type=GeckoVersion.parse, + required=True, + help="The version being built.", +) +@CommandArgument("--product", required=True, help="The product being built.") +@CommandArgument("--repo", required=True, help="The repo being built.") +@CommandArgument("--revision", required=True, help="The revision being built.") +@CommandArgument("--build-number", required=True, help="The build number") +@CommandArgument("--task-group-id", help="The task group of the build.") +def buglist_email(command_context, **options): + setup_logging(command_context) + from mozrelease.buglist_creator import email_release_drivers + + email_release_drivers(**options) + + +@SubCommand( + "release", + "push-scriptworker-canary", + description="Push tasks to try, to test new scriptworker deployments.", +) +@CommandArgument( + "--address", + required=True, + action="append", + dest="addresses", + help="The email address to send notifications to " + "(may be specified more than once).", +) +@CommandArgument( + "--scriptworker", + required=True, + action="append", + dest="scriptworkers", + help="Scriptworker to run canary for (may be specified more than once).", +) +@CommandArgument( + "--ssh-key-secret", + required=False, + help="Taskcluster secret with ssh-key to use for hg.mozilla.org", +) +def push_scriptworker_canary(command_context, scriptworkers, addresses, ssh_key_secret): + setup_logging(command_context) + from mozrelease.scriptworker_canary import push_canary + + push_canary( + scriptworkers=scriptworkers, + addresses=addresses, + ssh_key_secret=ssh_key_secret, + ) + + +def setup_logging(command_context, quiet=False, verbose=True): + """ + Set up Python logging for all loggers, sending results to stderr (so + that command output can be redirected easily) and adding the typical + mach timestamp. + """ + # remove the old terminal handler + old = command_context.log_manager.replace_terminal_handler(None) + + # re-add it, with level and fh set appropriately + if not quiet: + level = logging.DEBUG if verbose else logging.INFO + command_context.log_manager.add_terminal_logging( + fh=sys.stderr, + level=level, + write_interval=old.formatter.write_interval, + write_times=old.formatter.write_times, + ) + + # all of the taskgraph logging is unstructured logging + command_context.log_manager.enable_unstructured() diff --git a/python/mozrelease/mozrelease/partner_repack.py b/python/mozrelease/mozrelease/partner_repack.py new file mode 100644 index 0000000000..1d64f43cca --- /dev/null +++ b/python/mozrelease/mozrelease/partner_repack.py @@ -0,0 +1,895 @@ +#!/usr/bin/env python +# Documentation: https://firefox-source-docs.mozilla.org/taskcluster/partner-repacks.html + +import json +import logging +import os +import re +import stat +import sys +import tarfile +import urllib.parse +import urllib.request +import zipfile +from optparse import OptionParser +from pathlib import Path +from shutil import copy, copytree, move, rmtree, which +from subprocess import Popen + +from redo import retry + +logging.basicConfig( + stream=sys.stdout, + level=logging.INFO, + format="%(asctime)-15s - %(levelname)s - %(message)s", +) +log = logging.getLogger(__name__) + + +# Set default values. +PARTNERS_DIR = Path("..") / ".." / "workspace" / "partners" +# No platform in this path because script only supports repacking a single platform at once +DEFAULT_OUTPUT_DIR = "%(partner)s/%(partner_distro)s/%(locale)s" +TASKCLUSTER_ARTIFACTS = ( + os.environ.get("TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com") + + "/api/queue/v1/task/{taskId}/artifacts" +) +UPSTREAM_ENUS_PATH = "public/build/{filename}" +UPSTREAM_L10N_PATH = "public/build/{locale}/{filename}" + +WINDOWS_DEST_DIR = Path("firefox") +MAC_DEST_DIR = Path("Contents/Resources") +LINUX_DEST_DIR = Path("firefox") + +BOUNCER_PRODUCT_TEMPLATE = ( + "partner-firefox-{release_type}-{partner}-{partner_distro}-latest" +) + + +class StrictFancyURLopener(urllib.request.FancyURLopener): + """Unlike FancyURLopener this class raises exceptions for generic HTTP + errors, like 404, 500. It reuses URLopener.http_error_default redefined in + FancyURLopener""" + + def http_error_default(self, url, fp, errcode, errmsg, headers): + urllib.request.URLopener.http_error_default( + self, url, fp, errcode, errmsg, headers + ) + + +def rmdirRecursive(directory: Path): + """ + This is similar to a call of shutil.rmtree(), except that it + should work better on Windows since it will more aggressively + attempt to remove files marked as "read-only". + """ + + def rmdir_including_read_only(func, path: str, exc_info): + """ + Source: https://stackoverflow.com/a/4829285 + path contains the path of the file that couldn't be removed. + Let's just assume that it's read-only and unlink it. + """ + path = Path(path) + + path.chmod(mode=stat.S_IWRITE) + path.unlink() + + rmtree(str(directory), onerror=rmdir_including_read_only) + + +def printSeparator(): + log.info("##################################################") + + +def shellCommand(cmd): + log.debug("Executing %s" % cmd) + log.debug(f"in {Path.cwd()}") + # Shell command output gets dumped immediately to stdout, whereas + # print statements get buffered unless we flush them explicitly. + sys.stdout.flush() + p = Popen(cmd, shell=True) + (_, ret) = os.waitpid(p.pid, 0) + if ret != 0: + ret_real = (ret & 0xFF00) >> 8 + log.error("Error: shellCommand had non-zero exit status: %d" % ret_real) + log.error("Command: %s" % cmd, exc_info=True) + sys.exit(ret_real) + return True + + +def isLinux(platform: str): + return "linux" in platform + + +def isLinux32(platform: str): + return "linux32" in platform or "linux-i686" in platform or platform == "linux" + + +def isLinux64(platform: str): + return "linux64" in platform or "linux-x86_64" in platform + + +def isMac(platform: str): + return "mac" in platform + + +def isWin(platform: str): + return "win" in platform + + +def isWin32(platform: str): + return "win32" in platform + + +def isWin64(platform: str): + return platform == "win64" + + +def isWin64Aarch64(platform: str): + return platform == "win64-aarch64" + + +def isValidPlatform(platform: str): + return ( + isLinux64(platform) + or isLinux32(platform) + or isMac(platform) + or isWin64(platform) + or isWin64Aarch64(platform) + or isWin32(platform) + ) + + +def parseRepackConfig(file: Path, platform: str): + """Did you hear about this cool file format called yaml ? json ? Yeah, me neither""" + config = {} + config["platforms"] = [] + for line in file.open(): + line = line.rstrip("\n") + # Ignore empty lines + if line.strip() == "": + continue + # Ignore comments + if line.startswith("#"): + continue + [key, value] = line.split("=", 2) + value = value.strip('"') + # strings that don't need special handling + if key in ("dist_id", "replacement_setup_exe"): + config[key] = value + continue + # booleans that don't need special handling + if key in ("migrationWizardDisabled", "oem", "repack_stub_installer"): + if value.lower() == "true": + config[key] = True + continue + # special cases + if key == "locales": + config["locales"] = value.split(" ") + continue + if key.startswith("locale."): + config[key] = value + continue + if key == "deb_section": + config["deb_section"] = re.sub("/", "\/", value) + continue + if isValidPlatform(key): + ftp_platform = getFtpPlatform(key) + if ftp_platform == getFtpPlatform(platform) and value.lower() == "true": + config["platforms"].append(ftp_platform) + continue + + # this only works for one locale because setup.exe is localised + if config.get("replacement_setup_exe") and len(config.get("locales", [])) > 1: + log.error( + "Error: replacement_setup_exe is only supported for one locale, got %s" + % config["locales"] + ) + sys.exit(1) + # also only works for one platform because setup.exe is platform-specific + + if config["platforms"]: + return config + + +def getFtpPlatform(platform: str): + """Returns the platform in the format used in building package names. + Note: we rely on this code being idempotent + i.e. getFtpPlatform(getFtpPlatform(foo)) should work + """ + if isLinux64(platform): + return "linux-x86_64" + if isLinux(platform): + return "linux-i686" + if isMac(platform): + return "mac" + if isWin64Aarch64(platform): + return "win64-aarch64" + if isWin64(platform): + return "win64" + if isWin32(platform): + return "win32" + + +def getFileExtension(platform: str): + """The extension for the output file, which may be passed to the internal-signing task""" + if isLinux(platform): + return "tar.bz2" + elif isMac(platform): + return "tar.gz" + elif isWin(platform): + return "zip" + + +def getFilename(platform: str): + """Returns the filename to be repacked for the platform""" + return f"target.{getFileExtension(platform)}" + + +def getAllFilenames(platform: str, repack_stub_installer): + """Returns the full list of filenames we want to downlaod for each platform""" + file_names = [getFilename(platform)] + if isWin(platform): + # we want to copy forward setup.exe from upstream tasks to make it easier to repackage + # windows installers later + file_names.append("setup.exe") + # Same for the stub installer with setup-stub.exe, but only in win32 repack jobs + if isWin32(platform) and repack_stub_installer: + file_names.append("setup-stub.exe") + return tuple(file_names) + + +def getTaskArtifacts(taskId): + try: + retrieveFile( + TASKCLUSTER_ARTIFACTS.format(taskId=taskId), Path("tc_artifacts.json") + ) + tc_index = json.load(open("tc_artifacts.json")) + return tc_index["artifacts"] + except (ValueError, KeyError): + log.error("Failed to get task artifacts from TaskCluster") + raise + + +def getUpstreamArtifacts(upstream_tasks, repack_stub_installer): + useful_artifacts = getAllFilenames(options.platform, repack_stub_installer) + + artifact_ids = {} + for taskId in upstream_tasks: + for artifact in getTaskArtifacts(taskId): + name = artifact["name"] + if not name.endswith(useful_artifacts): + continue + if name in artifact_ids: + log.error( + "Duplicated artifact %s processing tasks %s & %s", + name, + taskId, + artifacts[name], + ) + sys.exit(1) + else: + artifact_ids[name] = taskId + log.debug( + "Found artifacts: %s" % json.dumps(artifact_ids, indent=4, sort_keys=True) + ) + return artifact_ids + + +def getArtifactNames(platform: str, locale, repack_stub_installer): + file_names = getAllFilenames(platform, repack_stub_installer) + if locale == "en-US": + names = [UPSTREAM_ENUS_PATH.format(filename=f) for f in file_names] + else: + names = [ + UPSTREAM_L10N_PATH.format(locale=locale, filename=f) for f in file_names + ] + return names + + +def retrieveFile(url, file_path: Path): + success = True + url = urllib.parse.quote(url, safe=":/") + log.info(f"Downloading from {url}") + log.info(f"To: {file_path}") + log.info(f"CWD: {Path.cwd()}") + try: + # use URLopener, which handles errors properly + retry( + StrictFancyURLopener().retrieve, + kwargs=dict(url=url, filename=str(file_path)), + ) + except IOError: + log.error("Error downloading %s" % url, exc_info=True) + success = False + try: + file_path.unlink() + except OSError: + log.info(f"Cannot remove {file_path}", exc_info=True) + + return success + + +def getBouncerProduct(partner, partner_distro): + if "RELEASE_TYPE" not in os.environ: + log.fatal("RELEASE_TYPE must be set in the environment") + sys.exit(1) + release_type = os.environ["RELEASE_TYPE"] + # For X.0 releases we get 'release-rc' but the alias should use 'release' + if release_type == "release-rc": + release_type = "release" + return BOUNCER_PRODUCT_TEMPLATE.format( + release_type=release_type, + partner=partner, + partner_distro=partner_distro, + ) + + +class RepackBase(object): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + file_mode=0o644, + quiet=False, + source_locale=None, + locale=None, + ): + self.base_dir = Path.cwd() + self.build = build + self.full_build_path = build_dir / build + if not self.full_build_path.is_absolute(): + self.full_build_path = self.base_dir / self.full_build_path + self.full_partner_path = self.base_dir / partner_dir + self.working_dir = final_dir / "working" + self.final_dir = final_dir + self.final_build = final_dir / Path(build).name + self.ftp_platform = ftp_platform + self.repack_info = repack_info + self.file_mode = file_mode + self.quiet = quiet + self.source_locale = source_locale + self.locale = locale + self.working_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + + def announceStart(self): + log.info( + "Repacking %s %s build %s" % (self.ftp_platform, self.locale, self.build) + ) + + def announceSuccess(self): + log.info( + "Done repacking %s %s build %s" + % (self.ftp_platform, self.locale, self.build) + ) + + def unpackBuild(self): + copy(str(self.full_build_path), ".") + + def createOverrideIni(self, partner_path: Path): + """If this is a partner specific locale (like en-HK), set the + distribution.ini to use that locale, not the default locale. + """ + if self.locale != self.source_locale: + file_path = partner_path / "distribution" / "distribution.ini" + with file_path.open(file_path.is_file() and "a" or "w") as open_file: + open_file.write("[Locale]\n") + open_file.write("locale=" + self.locale + "\n") + + """ Some partners need to override the migration wizard. This is done + by adding an override.ini file to the base install dir. + """ + # modify distribution.ini if 44 or later and we have migrationWizardDisabled + if int(options.version.split(".")[0]) >= 44: + file_path = partner_path / "distribution" / "distribution.ini" + with file_path.open() as open_file: + ini = open_file.read() + + if ini.find("EnableProfileMigrator") >= 0: + return + else: + browser_dir = partner_path / "browser" + if not browser_dir.exists(): + browser_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + file_path = browser_dir / "override.ini" + if "migrationWizardDisabled" in self.repack_info: + log.info("Adding EnableProfileMigrator to %r" % (file_path,)) + with file_path.open(file_path.is_file() and "a" or "w") as open_file: + open_file.write("[XRE]\n") + open_file.write("EnableProfileMigrator=0\n") + + def copyFiles(self, platform_dir: Path): + log.info(f"Copying files into {platform_dir}") + # Check whether we've already copied files over for this partner. + if not platform_dir.exists(): + platform_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + for i in ["distribution", "extensions"]: + full_path = self.full_partner_path / i + if full_path.exists(): + copytree(str(full_path), str(platform_dir / i)) + self.createOverrideIni(platform_dir) + + def repackBuild(self): + pass + + def stage(self): + move(self.build, str(self.final_dir)) + self.final_build.chmod(self.file_mode) + + def cleanup(self): + self.final_build.unlink() + + def doRepack(self): + self.announceStart() + os.chdir(self.working_dir) + self.unpackBuild() + self.copyFiles() + self.repackBuild() + self.stage() + os.chdir(self.base_dir) + rmdirRecursive(self.working_dir) + self.announceSuccess() + + +class RepackLinux(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackLinux, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + self.uncompressed_build = build.replace(".bz2", "") + + def unpackBuild(self): + super(RepackLinux, self).unpackBuild() + bunzip2_cmd = "bunzip2 %s" % self.build + shellCommand(bunzip2_cmd) + if not Path(self.uncompressed_build).exists(): + log.error(f"Error: Unable to uncompress build {self.build}") + sys.exit(1) + + def copyFiles(self): + super(RepackLinux, self).copyFiles(LINUX_DEST_DIR) + + def repackBuild(self): + if options.quiet: + tar_flags = "rf" + else: + tar_flags = "rvf" + tar_cmd = "tar %s %s %s" % (tar_flags, self.uncompressed_build, LINUX_DEST_DIR) + shellCommand(tar_cmd) + bzip2_command = "bzip2 %s" % self.uncompressed_build + shellCommand(bzip2_command) + + +class RepackMac(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackMac, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + self.uncompressed_build = build.replace(".gz", "") + + def unpackBuild(self): + super(RepackMac, self).unpackBuild() + gunzip_cmd = "gunzip %s" % self.build + shellCommand(gunzip_cmd) + if not Path(self.uncompressed_build).exists(): + log.error(f"Error: Unable to uncompress build {self.build}") + sys.exit(1) + self.appName = self.getAppName() + + def getAppName(self): + # Cope with Firefox.app vs Firefox Nightly.app by returning the first root object/folder found + t = tarfile.open(self.build.rsplit(".", 1)[0]) + for name in t.getnames(): + root_object = name.split("/")[0] + if root_object.endswith(".app"): + log.info(f"Found app name in tarball: {root_object}") + return root_object + log.error( + f"Error: Unable to determine app name from tarball: {self.build} - Expected .app in root" + ) + sys.exit(1) + + def copyFiles(self): + super(RepackMac, self).copyFiles(Path(self.appName) / MAC_DEST_DIR) + + def repackBuild(self): + if options.quiet: + tar_flags = "rf" + else: + tar_flags = "rvf" + # the final arg is quoted because it may contain a space, eg Firefox Nightly.app/.... + tar_cmd = "tar %s %s '%s'" % ( + tar_flags, + self.uncompressed_build, + Path(self.appName) / MAC_DEST_DIR, + ) + shellCommand(tar_cmd) + gzip_command = "gzip %s" % self.uncompressed_build + shellCommand(gzip_command) + + +class RepackWin(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackWin, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + + def copyFiles(self): + super(RepackWin, self).copyFiles(WINDOWS_DEST_DIR) + + def repackBuild(self): + if options.quiet: + zip_flags = "-rq" + else: + zip_flags = "-r" + zip_cmd = f"zip {zip_flags} {self.build} {WINDOWS_DEST_DIR}" + shellCommand(zip_cmd) + + # we generate the stub installer during the win32 build, so repack it on win32 too + if isWin32(options.platform) and self.repack_info.get("repack_stub_installer"): + log.info("Creating target-stub.zip to hold custom urls") + dest = str(self.final_build).replace("target.zip", "target-stub.zip") + z = zipfile.ZipFile(dest, "w") + # load the partner.ini template and interpolate %LOCALE% to the actual locale + with (self.full_partner_path / "stub" / "partner.ini").open() as open_file: + partner_ini_template = open_file.readlines() + partner_ini = "" + for l in partner_ini_template: + l = l.replace("%LOCALE%", self.locale) + l = l.replace("%BOUNCER_PRODUCT%", self.repack_info["bouncer_product"]) + partner_ini += l + z.writestr("partner.ini", partner_ini) + # we need an empty firefox directory to use the repackage code + d = zipfile.ZipInfo("firefox/") + # https://stackoverflow.com/a/6297838, zip's representation of drwxr-xr-x permissions + # is 040755 << 16L, bitwise OR with 0x10 for the MS-DOS directory flag + d.external_attr = 1106051088 + z.writestr(d, "") + z.close() + + def stage(self): + super(RepackWin, self).stage() + setup_dest = Path(str(self.final_build).replace("target.zip", "setup.exe")) + if "replacement_setup_exe" in self.repack_info: + log.info("Overriding setup.exe with custom copy") + retrieveFile(self.repack_info["replacement_setup_exe"], setup_dest) + else: + # otherwise copy forward the vanilla copy + log.info("Copying vanilla setup.exe forward for installer creation") + setup = str(self.full_build_path).replace("target.zip", "setup.exe") + copy(setup, str(setup_dest)) + setup_dest.chmod(self.file_mode) + + # we generate the stub installer in the win32 build, so repack it on win32 too + if isWin32(options.platform) and self.repack_info.get("repack_stub_installer"): + log.info( + "Copying vanilla setup-stub.exe forward for stub installer creation" + ) + setup_dest = Path( + str(self.final_build).replace("target.zip", "setup-stub.exe") + ) + setup_source = str(self.full_build_path).replace( + "target.zip", "setup-stub.exe" + ) + copy(setup_source, str(setup_dest)) + setup_dest.chmod(self.file_mode) + + +if __name__ == "__main__": + error = False + partner_builds = {} + repack_build = { + "linux-i686": RepackLinux, + "linux-x86_64": RepackLinux, + "mac": RepackMac, + "win32": RepackWin, + "win64": RepackWin, + "win64-aarch64": RepackWin, + } + + parser = OptionParser(usage="usage: %prog [options]") + parser.add_option( + "-d", + "--partners-dir", + dest="partners_dir", + default=str(PARTNERS_DIR), + help="Specify the directory where the partner config files are found", + ) + parser.add_option( + "-p", + "--partner", + dest="partner", + help="Repack for a single partner, specified by name", + ) + parser.add_option( + "-v", "--version", dest="version", help="Set the version number for repacking" + ) + parser.add_option( + "-n", + "--build-number", + dest="build_number", + default=1, + help="Set the build number for repacking", + ) + parser.add_option("--platform", dest="platform", help="Set the platform to repack") + parser.add_option( + "--include-oem", + action="store_true", + dest="include_oem", + default=False, + help="Process partners marked as OEM (these are usually one-offs)", + ) + parser.add_option( + "-q", + "--quiet", + action="store_true", + dest="quiet", + default=False, + help="Suppress standard output from the packaging tools", + ) + parser.add_option( + "--taskid", + action="append", + dest="upstream_tasks", + help="Specify taskIds for upstream artifacts, using 'internal sign' tasks. Multiples " + "expected, e.g. --taskid foo --taskid bar. Alternatively, use a space-separated list " + "stored in UPSTREAM_TASKIDS in the environment.", + ) + parser.add_option( + "-l", + "--limit-locale", + action="append", + dest="limit_locales", + default=[], + ) + + (options, args) = parser.parse_args() + + if not options.quiet: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.WARNING) + + options.partners_dir = Path(options.partners_dir.rstrip("/")) + if not options.partners_dir.is_dir(): + log.error(f"Error: partners dir {options.partners_dir} is not a directory.") + error = True + + if not options.version: + log.error("Error: you must specify a version number.") + error = True + + if not options.platform: + log.error("No platform specified.") + error = True + + if not isValidPlatform(options.platform): + log.error("Invalid platform %s." % options.platform) + error = True + + upstream_tasks = options.upstream_tasks or os.getenv("UPSTREAM_TASKIDS") + if not upstream_tasks: + log.error( + "upstream tasks should be defined using --taskid args or " + "UPSTREAM_TASKIDS in env." + ) + error = True + + for tool in ("tar", "bunzip2", "bzip2", "gunzip", "gzip", "zip"): + if not which(tool): + log.error(f"Error: couldn't find the {tool} executable in PATH.") + error = True + + if error: + sys.exit(1) + + base_workdir = Path.cwd() + + # Look up the artifacts available on our upstreams, but only if we need to + artifact_ids = {} + + # Local directories for builds + script_directory = Path.cwd() + original_builds_dir = ( + script_directory + / "original_builds" + / options.version + / f"build{options.build_number}" + ) + repack_version = f"{options.version}-{options.build_number}" + if os.getenv("MOZ_AUTOMATION"): + # running in production + repacked_builds_dir = Path("/builds/worker/artifacts") + else: + # local development + repacked_builds_dir = script_directory / "artifacts" + original_builds_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + repacked_builds_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + printSeparator() + + # For each partner in the partners dir + # Read/check the config file + # Download required builds (if not already on disk) + # Perform repacks + + # walk the partner dirs, find valid repack.cfg configs, and load them + partner_dirs = [] + need_stub_installers = False + for root, _, all_files in os.walk(options.partners_dir): + root = root.lstrip("/") + partner = root[len(str(options.partners_dir)) + 1 :].split("/")[0] + partner_distro = os.path.split(root)[-1] + if options.partner: + if ( + options.partner != partner + and options.partner != partner_distro[: len(options.partner)] + ): + continue + + for file in all_files: + if file == "repack.cfg": + log.debug( + "Found partner config: {} ['{}'] {}".format( + root, "', '".join(_), file + ) + ) + root = Path(root) + repack_cfg = root / file + repack_info = parseRepackConfig(repack_cfg, options.platform) + if not repack_info: + log.debug( + "no repack_info for platform %s in %s, skipping" + % (options.platform, repack_cfg) + ) + continue + if repack_info.get("repack_stub_installer"): + need_stub_installers = True + repack_info["bouncer_product"] = getBouncerProduct( + partner, partner_distro + ) + partner_dirs.append((partner, partner_distro, root, repack_info)) + + log.info("Retrieving artifact lists from upstream tasks") + artifact_ids = getUpstreamArtifacts(upstream_tasks, need_stub_installers) + if not artifact_ids: + log.fatal("No upstream artifacts were found") + sys.exit(1) + + for partner, partner_distro, full_partner_dir, repack_info in partner_dirs: + log.info( + "Starting repack process for partner: %s/%s" % (partner, partner_distro) + ) + if "oem" in repack_info and options.include_oem is False: + log.info( + "Skipping partner: %s - marked as OEM and --include-oem was not set" + % partner + ) + continue + + repack_stub_installer = repack_info.get("repack_stub_installer") + # where everything ends up + partner_repack_dir = repacked_builds_dir / DEFAULT_OUTPUT_DIR + + # Figure out which base builds we need to repack. + for locale in repack_info["locales"]: + if options.limit_locales and locale not in options.limit_locales: + log.info("Skipping %s because it is not in limit_locales list", locale) + continue + source_locale = locale + # Partner has specified a different locale to + # use as the base for their custom locale. + if "locale." + locale in repack_info: + source_locale = repack_info["locale." + locale] + for platform in repack_info["platforms"]: + # ja-JP-mac only exists for Mac, so skip non-existent + # platform/locale combos. + if (source_locale == "ja" and isMac(platform)) or ( + source_locale == "ja-JP-mac" and not isMac(platform) + ): + continue + ftp_platform = getFtpPlatform(platform) + + local_filepath = original_builds_dir / ftp_platform / locale + local_filepath.mkdir(mode=0o755, exist_ok=True, parents=True) + final_dir = Path( + str(partner_repack_dir) + % dict( + partner=partner, + partner_distro=partner_distro, + locale=locale, + ) + ) + if final_dir.exists(): + rmdirRecursive(final_dir) + final_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + + # for the main repacking artifact + file_name = getFilename(ftp_platform) + local_filename = local_filepath / file_name + + # Check to see if this build is already on disk, i.e. + # has already been downloaded. + artifacts = getArtifactNames(platform, locale, repack_stub_installer) + for artifact in artifacts: + local_artifact = local_filepath / Path(artifact).name + if local_artifact.exists(): + log.info(f"Found {local_artifact} on disk, not downloading") + continue + + if artifact not in artifact_ids: + log.fatal( + "Can't determine what taskID to retrieve %s from", artifact + ) + sys.exit(1) + original_build_url = "%s/%s" % ( + TASKCLUSTER_ARTIFACTS.format(taskId=artifact_ids[artifact]), + artifact, + ) + retrieveFile(original_build_url, local_artifact) + + # Make sure we have the local file now + if not local_filename.exists(): + log.info(f"Error: Unable to retrieve {file_name}\n") + sys.exit(1) + + repackObj = repack_build[ftp_platform]( + file_name, + full_partner_dir, + local_filepath, + final_dir, + ftp_platform, + repack_info, + locale=locale, + source_locale=source_locale, + ) + repackObj.doRepack() diff --git a/python/mozrelease/mozrelease/paths.py b/python/mozrelease/mozrelease/paths.py new file mode 100644 index 0000000000..b3d48c4ac7 --- /dev/null +++ b/python/mozrelease/mozrelease/paths.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from six.moves.urllib.parse import urlunsplit + +product_ftp_map = { + "fennec": "mobile", +} + + +def product2ftp(product): + return product_ftp_map.get(product, product) + + +def getCandidatesDir(product, version, buildNumber, protocol=None, server=None): + if protocol: + assert server is not None, "server is required with protocol" + + product = product2ftp(product) + directory = "/{}/candidates/{}-candidates/build{}".format( + product, + str(version), + str(buildNumber), + ) + + if protocol: + return urlunsplit((protocol, server, directory, None, None)) + else: + return directory + + +def getReleasesDir(product, version=None, protocol=None, server=None): + if protocol: + assert server is not None, "server is required with protocol" + + directory = "/{}/releases".format(product) + if version: + directory = "{}/{}".format(directory, version) + + if protocol: + return urlunsplit((protocol, server, directory, None, None)) + else: + return directory + + +def getReleaseInstallerPath(productName, brandName, version, platform, locale="en-US"): + if productName not in ("fennec",): + if platform.startswith("linux"): + return "/".join( + [ + p.strip("/") + for p in [ + platform, + locale, + "%s-%s.tar.bz2" % (productName, version), + ] + ] + ) + elif "mac" in platform: + return "/".join( + [ + p.strip("/") + for p in [platform, locale, "%s %s.dmg" % (brandName, version)] + ] + ) + elif platform.startswith("win"): + return "/".join( + [ + p.strip("/") + for p in [ + platform, + locale, + "%s Setup %s.exe" % (brandName, version), + ] + ] + ) + else: + raise "Unsupported platform" + else: + if platform.startswith("android"): + filename = "%s-%s.%s.android-arm.apk" % (productName, version, locale) + return "/".join([p.strip("/") for p in [platform, locale, filename]]) + else: + raise "Unsupported platform" diff --git a/python/mozrelease/mozrelease/platforms.py b/python/mozrelease/mozrelease/platforms.py new file mode 100644 index 0000000000..2970725a73 --- /dev/null +++ b/python/mozrelease/mozrelease/platforms.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +update_platform_map = { + "android": ["Android_arm-eabi-gcc3"], + "android-arm": ["Android_arm-eabi-gcc3"], + "android-x86": ["Android_x86-gcc3"], + "android-x86_64": ["Android_x86-64-gcc3"], + "android-aarch64": ["Android_aarch64-gcc3"], + "linux-i686": ["Linux_x86-gcc3"], + "linux-x86_64": ["Linux_x86_64-gcc3"], + "mac": [ + "Darwin_x86_64-gcc3-u-i386-x86_64", + "Darwin_x86-gcc3-u-i386-x86_64", + "Darwin_aarch64-gcc3", + "Darwin_x86-gcc3", + "Darwin_x86_64-gcc3", + ], + "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"], + "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"], + "win64-aarch64": ["WINNT_aarch64-msvc-aarch64"], +} + +# ftp -> shipped locales map +sl_platform_map = { + "linux-i686": "linux", + "linux-x86_64": "linux", + "mac": "osx", + "win32": "win32", + "win64": "win64", +} + +# ftp -> info file platform map +info_file_platform_map = { + "linux-i686": "linux", + "linux-x86_64": "linux64", + "mac": "macosx64", + "win32": "win32", + "win64": "win64", + "win64-aarch64": "win64_aarch64", +} + + +def ftp2updatePlatforms(platform): + return update_platform_map[platform] + + +def ftp2shippedLocales(platform): + return sl_platform_map.get(platform, platform) + + +def ftp2infoFile(platform): + return info_file_platform_map.get(platform, platform) diff --git a/python/mozrelease/mozrelease/scriptworker_canary.py b/python/mozrelease/mozrelease/scriptworker_canary.py new file mode 100644 index 0000000000..dabdc6868d --- /dev/null +++ b/python/mozrelease/mozrelease/scriptworker_canary.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import shutil +import subprocess +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import taskcluster +from appdirs import user_config_dir +from gecko_taskgraph import GECKO +from mach.base import FailedCommandError + +logger = logging.getLogger(__name__) + + +TASK_TYPES = { + "signing": ["linux-signing", "linux-signing-partial"], + "beetmover": ["beetmover-candidates"], + "bouncer": ["bouncer-submit"], + "balrog": ["balrog-submit"], + "tree": ["tree"], +} + + +def get_secret(secret): + # use proxy if configured, otherwise use local credentials from env vars + if "TASKCLUSTER_PROXY_URL" in os.environ: + secrets_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} + else: + secrets_options = taskcluster.optionsFromEnvironment() + secrets = taskcluster.Secrets(secrets_options) + return secrets.get(secret)["secret"] + + +@contextmanager +def configure_ssh(ssh_key_secret): + if ssh_key_secret is None: + yield + + # If we get here, we are running in automation. + # We use a user hgrc, so that we also get the system-wide hgrc settings. + hgrc = Path(user_config_dir("hg")) / "hgrc" + if hgrc.exists(): + raise FailedCommandError(f"Not overwriting `{hgrc}`; cannot configure ssh.") + + try: + ssh_key_dir = Path(tempfile.mkdtemp()) + + ssh_key = get_secret(ssh_key_secret) + ssh_key_file = ssh_key_dir / "id_rsa" + ssh_key_file.write_text(ssh_key["ssh_privkey"]) + ssh_key_file.chmod(0o600) + + hgrc_content = ( + "[ui]\n" + "username = trybld\n" + "ssh = ssh -i {path} -l {user}\n".format( + path=ssh_key_file, user=ssh_key["user"] + ) + ) + hgrc.write_text(hgrc_content) + + yield + finally: + shutil.rmtree(str(ssh_key_dir)) + hgrc.unlink() + + +def push_canary(scriptworkers, addresses, ssh_key_secret): + if ssh_key_secret and os.environ.get("MOZ_AUTOMATION", "0") != "1": + # We make assumptions about the layout of the docker image + # for creating the hgrc that we use for the key. + raise FailedCommandError("Cannot use ssh-key-secret outside of automation.") + + # Collect the set of `mach try scriptworker` task sets to run. + tasks = [] + for scriptworker in scriptworkers: + worker_tasks = TASK_TYPES.get(scriptworker) + if worker_tasks: + logger.info("Running tasks for {}: {}".format(scriptworker, worker_tasks)) + tasks.extend(worker_tasks) + else: + logger.info("No tasks for {}.".format(scriptworker)) + + mach = Path(GECKO) / "mach" + base_command = [str(mach), "try", "scriptworker"] + for address in addresses: + base_command.extend( + [ + "--route", + "notify.email.{}.on-failed".format(address), + "--route", + "notify.email.{}.on-exception".format(address), + ] + ) + + with configure_ssh(ssh_key_secret): + env = os.environ.copy() + for task in tasks: + subprocess.check_call(base_command + [task], env=env) diff --git a/python/mozrelease/mozrelease/update_verify.py b/python/mozrelease/mozrelease/update_verify.py new file mode 100644 index 0000000000..49fe21db15 --- /dev/null +++ b/python/mozrelease/mozrelease/update_verify.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re + +from six import string_types + +from .chunking import getChunk + + +class UpdateVerifyError(Exception): + pass + + +class UpdateVerifyConfig(object): + comment_regex = re.compile("^#") + key_write_order = ( + "release", + "product", + "platform", + "build_id", + "locales", + "channel", + "patch_types", + "from", + "aus_server", + "ftp_server_from", + "ftp_server_to", + "to", + "mar_channel_IDs", + "override_certs", + "to_build_id", + "to_display_version", + "to_app_version", + "updater_package", + ) + global_keys = ( + "product", + "channel", + "aus_server", + "to", + "to_build_id", + "to_display_version", + "to_app_version", + "override_certs", + ) + release_keys = ( + "release", + "build_id", + "locales", + "patch_types", + "from", + "ftp_server_from", + "ftp_server_to", + "mar_channel_IDs", + "platform", + "updater_package", + ) + first_only_keys = ( + "from", + "aus_server", + "to", + "to_build_id", + "to_display_version", + "to_app_version", + "override_certs", + ) + compare_attrs = global_keys + ("releases",) + + def __init__( + self, + product=None, + channel=None, + aus_server=None, + to=None, + to_build_id=None, + to_display_version=None, + to_app_version=None, + override_certs=None, + ): + self.product = product + self.channel = channel + self.aus_server = aus_server + self.to = to + self.to_build_id = to_build_id + self.to_display_version = to_display_version + self.to_app_version = to_app_version + self.override_certs = override_certs + self.releases = [] + + def __eq__(self, other): + self_list = [getattr(self, attr) for attr in self.compare_attrs] + other_list = [getattr(other, attr) for attr in self.compare_attrs] + return self_list == other_list + + def __ne__(self, other): + return not self.__eq__(other) + + def _parseLine(self, line): + entry = {} + items = re.findall(r"\w+=[\"'][^\"']*[\"']", line) + for i in items: + m = re.search(r"(?P<key>\w+)=[\"'](?P<value>.+)[\"']", i).groupdict() + if m["key"] not in self.global_keys and m["key"] not in self.release_keys: + raise UpdateVerifyError( + "Unknown key '%s' found on line:\n%s" % (m["key"], line) + ) + if m["key"] in entry: + raise UpdateVerifyError( + "Multiple values found for key '%s' on line:\n%s" % (m["key"], line) + ) + entry[m["key"]] = m["value"] + if not entry: + raise UpdateVerifyError("No parseable data in line '%s'" % line) + return entry + + def _addEntry(self, entry, first): + releaseKeys = {} + for k, v in entry.items(): + if k in self.global_keys: + setattr(self, k, entry[k]) + elif k in self.release_keys: + # "from" is reserved in Python + if k == "from": + releaseKeys["from_path"] = v + else: + releaseKeys[k] = v + self.addRelease(**releaseKeys) + + def read(self, config): + f = open(config) + # Only the first non-comment line of an update verify config should + # have a "from" and"ausServer". Ignore any subsequent lines with them. + first = True + for line in f.readlines(): + # Skip comment lines + if self.comment_regex.search(line): + continue + self._addEntry(self._parseLine(line), first) + first = False + + def write(self, fh): + first = True + for releaseInfo in self.releases: + for key in self.key_write_order: + if key in self.global_keys and ( + first or key not in self.first_only_keys + ): + value = getattr(self, key) + elif key in self.release_keys: + value = releaseInfo[key] + else: + value = None + if value is not None: + fh.write(key.encode("utf-8")) + fh.write(b"=") + if isinstance(value, (list, tuple)): + fh.write(('"%s" ' % " ".join(value)).encode("utf-8")) + else: + fh.write(('"%s" ' % value).encode("utf-8")) + # Rewind one character to avoid having a trailing space + fh.seek(-1, os.SEEK_CUR) + fh.write(b"\n") + first = False + + def addRelease( + self, + release=None, + build_id=None, + locales=[], + patch_types=["complete"], + from_path=None, + ftp_server_from=None, + ftp_server_to=None, + mar_channel_IDs=None, + platform=None, + updater_package=None, + ): + """Locales and patch_types can be passed as either a string or a list. + If a string is passed, they will be converted to a list for internal + storage""" + if self.getRelease(build_id, from_path): + raise UpdateVerifyError( + "Couldn't add release identified by build_id '%s' and from_path '%s': " + "already exists in config" % (build_id, from_path) + ) + if isinstance(locales, string_types): + locales = sorted(list(locales.split())) + if isinstance(patch_types, string_types): + patch_types = list(patch_types.split()) + self.releases.append( + { + "release": release, + "build_id": build_id, + "locales": locales, + "patch_types": patch_types, + "from": from_path, + "ftp_server_from": ftp_server_from, + "ftp_server_to": ftp_server_to, + "mar_channel_IDs": mar_channel_IDs, + "platform": platform, + "updater_package": updater_package, + } + ) + + def addLocaleToRelease(self, build_id, locale, from_path=None): + r = self.getRelease(build_id, from_path) + if not r: + raise UpdateVerifyError( + "Couldn't add '%s' to release identified by build_id '%s' and from_path '%s': " + "'%s' doesn't exist in this config." + % (locale, build_id, from_path, build_id) + ) + r["locales"].append(locale) + r["locales"] = sorted(r["locales"]) + + def getRelease(self, build_id, from_path): + for r in self.releases: + if r["build_id"] == build_id and r["from"] == from_path: + return r + return {} + + def getFullReleaseTests(self): + return [r for r in self.releases if r["from"] is not None] + + def getQuickReleaseTests(self): + return [r for r in self.releases if r["from"] is None] + + def getChunk(self, chunks, thisChunk): + fullTests = [] + quickTests = [] + for test in self.getFullReleaseTests(): + for locale in test["locales"]: + fullTests.append([test["build_id"], locale, test["from"]]) + for test in self.getQuickReleaseTests(): + for locale in test["locales"]: + quickTests.append([test["build_id"], locale, test["from"]]) + allTests = getChunk(fullTests, chunks, thisChunk) + allTests.extend(getChunk(quickTests, chunks, thisChunk)) + + newConfig = UpdateVerifyConfig( + self.product, + self.channel, + self.aus_server, + self.to, + self.to_build_id, + self.to_display_version, + self.to_app_version, + self.override_certs, + ) + for t in allTests: + build_id, locale, from_path = t + if from_path == "None": + from_path = None + r = self.getRelease(build_id, from_path) + try: + newConfig.addRelease( + r["release"], + build_id, + locales=[], + ftp_server_from=r["ftp_server_from"], + ftp_server_to=r["ftp_server_to"], + patch_types=r["patch_types"], + from_path=from_path, + mar_channel_IDs=r["mar_channel_IDs"], + platform=r["platform"], + updater_package=r["updater_package"], + ) + except UpdateVerifyError: + pass + newConfig.addLocaleToRelease(build_id, locale, from_path) + return newConfig diff --git a/python/mozrelease/mozrelease/util.py b/python/mozrelease/mozrelease/util.py new file mode 100644 index 0000000000..3858c40514 --- /dev/null +++ b/python/mozrelease/mozrelease/util.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from yaml.loader import SafeLoader + + +class UnicodeLoader(SafeLoader): + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + +UnicodeLoader.add_constructor("tag:yaml.org,2002:str", UnicodeLoader.construct_yaml_str) + + +def load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + loader = UnicodeLoader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() diff --git a/python/mozrelease/mozrelease/versions.py b/python/mozrelease/mozrelease/versions.py new file mode 100644 index 0000000000..e3e47d4e4a --- /dev/null +++ b/python/mozrelease/mozrelease/versions.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +from distutils.version import StrictVersion + +from looseversion import LooseVersion + + +class MozillaVersionCompareMixin: + def __cmp__(self, other): + # We expect this function to never be called. + raise AssertionError() + + def _cmp(self, other): + has_esr = set() + if isinstance(other, LooseModernMozillaVersion) and str(other).endswith("esr"): + # If other version ends with esr, coerce through MozillaVersion ending up with + # a StrictVersion if possible + has_esr.add("other") + other = MozillaVersion(str(other)[:-3]) # strip ESR from end of string + if isinstance(self, LooseModernMozillaVersion) and str(self).endswith("esr"): + # If our version ends with esr, coerce through MozillaVersion ending up with + # a StrictVersion if possible + has_esr.add("self") + self = MozillaVersion(str(self)[:-3]) # strip ESR from end of string + if isinstance(other, LooseModernMozillaVersion) or isinstance( + self, LooseModernMozillaVersion + ): + # If we're still LooseVersion for self or other, run LooseVersion compare + # Being sure to pass through Loose Version type first + val = LooseVersion._cmp( + LooseModernMozillaVersion(str(self)), + LooseModernMozillaVersion(str(other)), + ) + else: + # No versions are loose, therefore we can use StrictVersion + val = StrictVersion._cmp(self, other) + if has_esr.isdisjoint(set(["other", "self"])) or has_esr.issuperset( + set(["other", "self"]) + ): + # If both had esr string or neither, then _cmp() was accurate + return val + elif val != 0: + # cmp is accurate here even if esr is present in only 1 compare, since + # versions are not equal + return val + elif "other" in has_esr: + return -1 # esr is not greater than non esr + return 1 # non esr is greater than esr + + +class ModernMozillaVersion(MozillaVersionCompareMixin, StrictVersion): + """A version class that is slightly less restrictive than StrictVersion. + Instead of just allowing "a" or "b" as prerelease tags, it allows any + alpha. This allows us to support the once-shipped "3.6.3plugin1" and + similar versions.""" + + version_re = re.compile( + r"""^(\d+) \. (\d+) (\. (\d+))? + ([a-zA-Z]+(\d+))?$""", + re.VERBOSE, + ) + + +class AncientMozillaVersion(MozillaVersionCompareMixin, StrictVersion): + """A version class that is slightly less restrictive than StrictVersion. + Instead of just allowing "a" or "b" as prerelease tags, it allows any + alpha. This allows us to support the once-shipped "3.6.3plugin1" and + similar versions. + It also supports versions w.x.y.z by transmuting to w.x.z, which + is useful for versions like 1.5.0.x and 2.0.0.y""" + + version_re = re.compile( + r"""^(\d+) \. (\d+) \. \d (\. (\d+)) + ([a-zA-Z]+(\d+))?$""", + re.VERBOSE, + ) + + +class LooseModernMozillaVersion(MozillaVersionCompareMixin, LooseVersion): + """A version class that is more restrictive than LooseVersion. + This class reduces the valid strings to "esr", "a", "b" and "rc" in order + to support esr. StrictVersion requires a trailing number after all strings.""" + + component_re = re.compile(r"(\d+ | a | b | rc | esr | \.)", re.VERBOSE) + + def __repr__(self): + return "LooseModernMozillaVersion ('%s')" % str(self) + + +def MozillaVersion(version): + try: + return ModernMozillaVersion(version) + except ValueError: + pass + try: + if version.count(".") == 3: + return AncientMozillaVersion(version) + except ValueError: + pass + try: + return LooseModernMozillaVersion(version) + except ValueError: + pass + raise ValueError("Version number %s is invalid." % version) + + +def getPrettyVersion(version): + version = re.sub(r"a([0-9]+)$", r" Alpha \1", version) + version = re.sub(r"b([0-9]+)$", r" Beta \1", version) + version = re.sub(r"rc([0-9]+)$", r" RC \1", version) + return version diff --git a/python/mozrelease/setup.py b/python/mozrelease/setup.py new file mode 100644 index 0000000000..d831cb14c5 --- /dev/null +++ b/python/mozrelease/setup.py @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import find_packages, setup + +VERSION = "0.2" + +setup( + author="Mozilla Foundation", + author_email="Mozilla Release Engineering", + name="mozrelease", + description="Common functionality used by Mozilla Release Automation", + license="MPL 2.0", + packages=find_packages(), + version=VERSION, + classifiers=[ + "Development Status :: 3 - Alpha", + "Topic :: Software Development :: Build Tools", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: Implementation :: CPython", + ], + keywords="mozilla", +) diff --git a/python/mozrelease/test/data/Firefox-62.0.3.update.json b/python/mozrelease/test/data/Firefox-62.0.3.update.json new file mode 100644 index 0000000000..c22cf42753 --- /dev/null +++ b/python/mozrelease/test/data/Firefox-62.0.3.update.json @@ -0,0 +1,74 @@ +[ + { + "fields": { + "detailsURL": "https://www.mozilla.org/%LOCALE%/firefox/62.0.3/releasenotes/", + "type": "minor" + }, + "for": {} + }, + { + "fields": { + "actions": "showURL", + "openURL": "https://www.mozilla.org/%LOCALE%/firefox/62.0.3/whatsnew/?oldversion=%OLD_VERSION%" + }, + "for": { + "channels": [ + "release", + "release-localtest", + "release-cdntest" + ], + "locales": [ + "cak", + "cy", + "da", + "de", + "dsb", + "en-CA", + "en-US", + "es-AR", + "es-CL", + "es-ES", + "es-MX", + "et", + "fa", + "fi", + "fr", + "fy-NL", + "gn", + "gu-IN", + "hsb", + "hu", + "ia", + "id", + "it", + "ja", + "ja-JP-mac", + "ka", + "kab", + "ko", + "lij", + "lt", + "ms", + "nb-NO", + "nl", + "nn-NO", + "pl", + "pt-BR", + "pt-PT", + "sk", + "sl", + "sq", + "sr", + "sv-SE", + "tr", + "uk", + "vi", + "zh-CN", + "zh-TW" + ], + "versions": [ + "<62.0" + ] + } + } +] diff --git a/python/mozrelease/test/data/Firefox-62.0b11-update.json b/python/mozrelease/test/data/Firefox-62.0b11-update.json new file mode 100644 index 0000000000..c22cf42753 --- /dev/null +++ b/python/mozrelease/test/data/Firefox-62.0b11-update.json @@ -0,0 +1,74 @@ +[ + { + "fields": { + "detailsURL": "https://www.mozilla.org/%LOCALE%/firefox/62.0.3/releasenotes/", + "type": "minor" + }, + "for": {} + }, + { + "fields": { + "actions": "showURL", + "openURL": "https://www.mozilla.org/%LOCALE%/firefox/62.0.3/whatsnew/?oldversion=%OLD_VERSION%" + }, + "for": { + "channels": [ + "release", + "release-localtest", + "release-cdntest" + ], + "locales": [ + "cak", + "cy", + "da", + "de", + "dsb", + "en-CA", + "en-US", + "es-AR", + "es-CL", + "es-ES", + "es-MX", + "et", + "fa", + "fi", + "fr", + "fy-NL", + "gn", + "gu-IN", + "hsb", + "hu", + "ia", + "id", + "it", + "ja", + "ja-JP-mac", + "ka", + "kab", + "ko", + "lij", + "lt", + "ms", + "nb-NO", + "nl", + "nn-NO", + "pl", + "pt-BR", + "pt-PT", + "sk", + "sl", + "sq", + "sr", + "sv-SE", + "tr", + "uk", + "vi", + "zh-CN", + "zh-TW" + ], + "versions": [ + "<62.0" + ] + } + } +] diff --git a/python/mozrelease/test/data/Firefox-64.0b13.update.json b/python/mozrelease/test/data/Firefox-64.0b13.update.json new file mode 100644 index 0000000000..0d9a4405e5 --- /dev/null +++ b/python/mozrelease/test/data/Firefox-64.0b13.update.json @@ -0,0 +1,9 @@ +[ + { + "fields": { + "detailsURL": "https://www.mozilla.org/%LOCALE%/firefox/64.0/releasenotes/", + "type": "minor" + }, + "for": {} + } +] diff --git a/python/mozrelease/test/data/buglist_changesets.json b/python/mozrelease/test/data/buglist_changesets.json new file mode 100644 index 0000000000..dedcd8a810 --- /dev/null +++ b/python/mozrelease/test/data/buglist_changesets.json @@ -0,0 +1,94 @@ +{ + "entries": [ + { + "desc": "Bug 1354038 - [push-apk] taskgraph: Use rollout and deactivate dry-run on release p=jlorenzo r=aki a=release DONTBUILD" + }, + { + "desc": "Bug 1356563 - Only set global ready state on native widget loading; r=snorp a=sylvestre\n\nOur \"chrome-document-loaded\" observer may detect several different types\nof widgets that can exist in the parent process, including the Android\nnsWindow, PuppetWidget, etc. We should only set the global state to\nready when the first top-level nsWindow has loaded, and not just any\nwindow." + }, + { + "desc": "No bug, Automated blocklist update from host bld-linux64-spot-305 - a=blocklist-update" + }, + { + "desc": "Automatic version bump. CLOSED TREE NO BUG a=release" + }, + { + "desc": "No bug - Tagging d345b657d381ade5195f1521313ac651618f54a2 with FIREFOX_53_0_BUILD6, FIREFOX_53_0_RELEASE a=release CLOSED TREE" + }, + { + "desc": "No bug, Automated blocklist update from host bld-linux64-spot-305 - a=blocklist-update" + }, + { + "desc": "Bug 1344529 - Remove unused variable in widget/gtk/gtk2drawing.c. r=frg a=release DONOTBUILD in a CLOSED TREE" + }, + { + "desc": "Bug 1306543 - Avoid using g_unicode_script_from_iso15924 directly. r=jfkthame a=release in a CLOSED TREE DONTBUILD" + }, + { + "desc": "Bug 1320072 - Backout intent change - broke partner Google test. r=snorp, a=lizzard" + }, + { + "desc": "Bug 1328762 - Cherry-pick ANGLE a4aaa2de57dc51243da35ea147d289a21a9f0c49. a=lizzard\n\nMozReview-Commit-ID: WVK0smAfAW" + }, + { + "desc": "Bug 1341190 - Remove .popup-anchor visibility rule. r=mconley, a=lizzard\n\nMozReview-Commit-ID: DFMIKMMnLx5" + }, + { + "desc": "Bug 1348409 - Stop supporting the showDialog argument for window.find. r=mrbkap, a=lizzard\n\nThe dialog functionality of the non-standard window.find API has been broken\nwith e10s since it shipped, and bug 1182569 or bug 1232432 (or both) have\nbroken it for non-e10s.\n\nThis patch remove showDialog support entirely, for both e10s and non-e10s,\nin a more deliberate way. We now ignore the argument.\n\nMozReview-Commit-ID: 1CTzgEkDhHW" + }, + { + "desc": "Bug 1358089 - [RTL] Separate xml drawable into v17 folder. r=ahunt, a=lizzard\n\nMozReview-Commit-ID: LaOwxXwhsHA" + }, + { + "desc": "Bug 1360626 - Create a blacklist for adaptive playback support. r=jolin, a=lizzard\n\nOn some devices / os combinations, enabling adaptive playback causes decoded frame unusable.\nIt may cause the decode frame to be black and white or return tiled frames.\nSo we should do the blacklist according to the report.\n\nMozReview-Commit-ID: j3PZXTtkXG" + }, + { + "desc": "Bug 1354038 - part2: [push-apk] taskgraph: Use rollout and deactivate dry-run on release r=aki a=bustage DONTBUILD\n\nMozReview-Commit-ID: 1f22BcAZkvp" + }, + { + "desc": "bug 1354038 - empty commit to force builds. a=release" + }, + { + "desc": "Bug 1337861 - [Fennec-Relpro] Enforce the presence of $MOZ_BUILD_DATE r=jlund a=release\n\nMozReview-Commit-ID: DzEeeYQjwLW" + }, + { + "desc": "Bug 1332731 - Follow-up to fix accessibility breakage. r=sebastian, a=lizzard\n\nFollow-up to fix breakage in accessibility caused by the bundle\nconversion. In particular, optString(foo) should have been converted to\ngetString(foo, \"\") because optString returns \"\" by default.\n\nAlso fix a small bug in Presentation.jsm where an array or null should\nbe used instead of a string." + }, + { + "desc": "Bug 1355870 - Allow a system preference to determine distribution dir. r=nalexander, a=lizzard" + }, + { + "desc": "Bug 1354911 - Guard against null menu item names. r=sebastian, a=lizzard\n\nAddons may give us invalid menu item names; bail instead of crashing in\nsuch cases." + }, + { + "desc": "Bug 1356563 - Remove chrome-document-loaded observer only after handling it. r=me, a=gchang\n\nOnly remove the \"chrome-document-loaded\" observer after handling it in\nnsAppShell. Otherwise we may never end up handling it." + }, + { + "desc": "Bug 1352333 - remove autophone webrtc test manifests, r=dminor, a=test-only." + }, + { + "desc": "Bug 1352333 - sync autophone webrtc test manifests with normal webrtc manifests, r=jmaher,dminor, a=test-only." + }, + { + "desc": "No bug - Tagging f239279b709072490993b099832fa8c18f07713a with FENNEC_53_0_BUILD1, FENNEC_53_0_RELEASE a=release CLOSED TREE" + }, + { + "desc": "Automated checkin: version bump for fennec 53.0.1 release. DONTBUILD CLOSED TREE a=release" + }, + { + "desc": "Added FENNEC_53_0_1_RELEASE FENNEC_53_0_1_BUILD1 tag(s) for changeset f029d1a1324b. DONTBUILD CLOSED TREE a=release" + }, + { + "desc": "Backout Bug 1337861 (Enforce MOZ_BUILD_DATE) due to Bug 1360550. r=catlee a=catlee\n\nBug 1360550 resulted in the buildid the Linux builds had being different than the directory they were uploaded to. This had fallout affects for QA's firefox-ui tests and presumably anything using mozdownload.\n\nMozReview-Commit-ID: 8lMvLU0vGiS" + }, + { + "desc": "No bug, Automated blocklist update from host bld-linux64-spot-303 - a=blocklist-update" + }, + { + "desc": "Automatic version bump. CLOSED TREE NO BUG a=release" + }, + { + "desc": "No bug - Tagging 5cbf464688a47129c0ea36fe38f42f59926e4b2c with FENNEC_53_0_1_BUILD2, FENNEC_53_0_1_RELEASE a=release CLOSED TREE" + } + ] +} diff --git a/python/mozrelease/test/data/sample-update-verify.cfg b/python/mozrelease/test/data/sample-update-verify.cfg new file mode 100644 index 0000000000..b8c87457b5 --- /dev/null +++ b/python/mozrelease/test/data/sample-update-verify.cfg @@ -0,0 +1,4 @@ +release="4.0" product="Firefox" platform="Linux_x86-gcc3" build_id="888" locales="af de en-US ja zh-TW" channel="betatest" patch_types="partial complete" from="/firefox/4.0rc1.tar.bz2" aus_server="https://aus4.mozilla.org" ftp_server_from="stage.mozilla.org/firefox" ftp_server_to="stage.mozilla.org/firefox" to="/firefox/4.0rc2.tar.bz2" mar_channel_IDs="firefox-mozilla-beta" to_build_id="999" to_display_version="99.0 Zeta 9" to_app_version="99.0" +release="4.0b12" product="Firefox" platform="Linux_x86-gcc3" build_id="777" locales="af en-US" channel="betatest" patch_types="complete" from="/firefox/4.0b12.tar.bz2" ftp_server_from="stage.mozilla.org/firefox" ftp_server_to="stage.mozilla.org/firefox" +release="4.0b12" product="Firefox" platform="Linux_x86-gcc3" build_id="777" locales="de ja zh-TW" channel="betatest" patch_types="complete" ftp_server_from="stage.mozilla.org/firefox" ftp_server_to="stage.mozilla.org/firefox" +release="3.7a1" product="Firefox" platform="Linux_x86-gcc3" build_id="666" locales="en-US" channel="betatest" patch_types="complete" ftp_server_from="stage.mozilla.org/firefox" ftp_server_to="stage.mozilla.org/firefox" diff --git a/python/mozrelease/test/data/whatsnew-62.0.3.yml b/python/mozrelease/test/data/whatsnew-62.0.3.yml new file mode 100644 index 0000000000..5a9e4c9a1f --- /dev/null +++ b/python/mozrelease/test/data/whatsnew-62.0.3.yml @@ -0,0 +1,65 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +--- +- type: product-details + url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/releasenotes/" +# %LOCALE% is automatically replaced by Balrog. +- type: show-url + # yamllint disable-line rule:line-length + url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/whatsnew/?oldversion=%OLD_VERSION%" + conditions: + release-types: [release] + products: [firefox] + update-channel: release + # e.g.: ["<61.0"]. {version.major_number} reflects the current version. + # This is done by taskgraph. + versions: ["<{version.major_number}.0"] + locales: + - cak + - cy + - da + - de + - dsb + - en-CA + - en-US + - es-AR + - es-CL + - es-ES + - es-MX + - et + - fa + - fi + - fr + - fy-NL + - gn + - gu-IN + - hsb + - hu + - ia + - id + - it + - ja + - ja-JP-mac + - ka + - kab + - ko + - lij + - lt + - ms + - nb-NO + - nl + - nn-NO + - pl + - pt-BR + - pt-PT + - sk + - sl + - sq + - sr + - sv-SE + - tr + - uk + - vi + - zh-CN + - zh-TW diff --git a/python/mozrelease/test/data/whatsnew-release.yml b/python/mozrelease/test/data/whatsnew-release.yml new file mode 100644 index 0000000000..5a9e4c9a1f --- /dev/null +++ b/python/mozrelease/test/data/whatsnew-release.yml @@ -0,0 +1,65 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +--- +- type: product-details + url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/releasenotes/" +# %LOCALE% is automatically replaced by Balrog. +- type: show-url + # yamllint disable-line rule:line-length + url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/whatsnew/?oldversion=%OLD_VERSION%" + conditions: + release-types: [release] + products: [firefox] + update-channel: release + # e.g.: ["<61.0"]. {version.major_number} reflects the current version. + # This is done by taskgraph. + versions: ["<{version.major_number}.0"] + locales: + - cak + - cy + - da + - de + - dsb + - en-CA + - en-US + - es-AR + - es-CL + - es-ES + - es-MX + - et + - fa + - fi + - fr + - fy-NL + - gn + - gu-IN + - hsb + - hu + - ia + - id + - it + - ja + - ja-JP-mac + - ka + - kab + - ko + - lij + - lt + - ms + - nb-NO + - nl + - nn-NO + - pl + - pt-BR + - pt-PT + - sk + - sl + - sq + - sr + - sv-SE + - tr + - uk + - vi + - zh-CN + - zh-TW diff --git a/python/mozrelease/test/python.ini b/python/mozrelease/test/python.ini new file mode 100644 index 0000000000..5854d57850 --- /dev/null +++ b/python/mozrelease/test/python.ini @@ -0,0 +1,7 @@ +[DEFAULT] +subsuite=mozrelease + +[test_versions.py] +[test_update_verify.py] +[test_balrog.py] +[test_buglist_creator.py] diff --git a/python/mozrelease/test/test_balrog.py b/python/mozrelease/test/test_balrog.py new file mode 100644 index 0000000000..0c3adfacac --- /dev/null +++ b/python/mozrelease/test/test_balrog.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +from pathlib import Path + +import mozunit +import pytest +from mozilla_version.gecko import GeckoVersion + +from mozrelease.balrog import generate_update_properties +from mozrelease.util import load as yaml_load + +DATA_PATH = Path(__file__).parent.joinpath("data") + + +@pytest.mark.parametrize( + "context,config_file,output_file", + [ + ( + { + "release-type": "release", + "product": "firefox", + "version": GeckoVersion.parse("62.0.3"), + }, + "whatsnew-62.0.3.yml", + "Firefox-62.0.3.update.json", + ), + ( + { + "release-type": "beta", + "product": "firefox", + "version": GeckoVersion.parse("64.0"), + }, + "whatsnew-62.0.3.yml", + "Firefox-64.0b13.update.json", + ), + ], +) +def test_update_properties(context, config_file, output_file): + with DATA_PATH.joinpath(config_file).open("r", encoding="utf-8") as f: + config = yaml_load(f) + + update_line = generate_update_properties(context, config) + + assert update_line == json.load( + DATA_PATH.joinpath(output_file).open("r", encoding="utf-8") + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozrelease/test/test_buglist_creator.py b/python/mozrelease/test/test_buglist_creator.py new file mode 100644 index 0000000000..13a530bb97 --- /dev/null +++ b/python/mozrelease/test/test_buglist_creator.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +from pathlib import Path + +import mozunit +import pytest +from mozilla_version.gecko import GeckoVersion + +from mozrelease.buglist_creator import ( + create_bugs_url, + get_bugs_in_changeset, + get_previous_tag_version, + is_backout_bug, + is_excluded_change, + parse_tag_version, + tag_version, +) + +DATA_PATH = Path(__file__).with_name("data") + + +def test_beta_1_release(): + buglist_str_54_0b1 = create_bugs_url( + product="firefox", + current_version=GeckoVersion.parse("54.0b1"), + current_revision="cf76e00dcd6f", + ) + assert buglist_str_54_0b1 == "", "There should be no bugs to compare for beta 1." + + +@pytest.mark.parametrize( + "description,is_excluded", + ( + ( + "something something something a=test-only something something something", + True, + ), + ("this is a a=release change!", True), + ), +) +def test_is_excluded_change(description, is_excluded): + assert is_excluded_change({"desc": description}) == is_excluded + + +@pytest.mark.parametrize( + "description,is_backout", + ( + ("I backed out this bug because", True), + ("Backing out this bug due to", True), + ("Backout bug xyz", True), + ("Back out bug xyz", True), + ("this is a regular bug description", False), + ), +) +def test_is_backout_bug(description, is_backout): + assert is_backout_bug(description) == is_backout + + +@pytest.mark.parametrize( + "product,version,tag", + ( + ("firefox", GeckoVersion.parse("53.0b10"), "FIREFOX_53_0b10_RELEASE"), + ("firefox", GeckoVersion.parse("52.0"), "FIREFOX_52_0_RELEASE"), + ("fennec", GeckoVersion.parse("52.0.2"), "FENNEC_52_0_2_RELEASE"), + ), +) +def test_tag_version(product, version, tag): + assert tag_version(product, version) == tag + + +@pytest.mark.parametrize( + "tag,version", + ( + ("FIREFOX_53_0b10_RELEASE", GeckoVersion.parse("53.0b10")), + ("FIREFOX_52_0_RELEASE", GeckoVersion.parse("52.0")), + ("FENNEC_52_0_2_RELEASE", GeckoVersion.parse("52.0.2")), + ), +) +def test_parse_tag_version(tag, version): + assert parse_tag_version(tag) == version + + +@pytest.mark.parametrize( + "version,tag,previous_tag", + ( + ( + GeckoVersion.parse("48.0b4"), + "FIREFOX_48_0b4_RELEASE", + "FIREFOX_48_0b3_RELEASE", + ), + ( + GeckoVersion.parse("48.0b9"), + "FIREFOX_48_0b9_RELEASE", + "FIREFOX_48_0b7_RELEASE", + ), + ( + GeckoVersion.parse("48.0.2"), + "FIREFOX_48_0_2_RELEASE", + "FIREFOX_48_0_1_RELEASE", + ), + ( + GeckoVersion.parse("48.0.1"), + "FIREFOX_48_0_1_RELEASE", + "FIREFOX_48_0_RELEASE", + ), + ), +) +def test_get_previous_tag_version(version, tag, previous_tag): + product = "firefox" + ff_48_tags = [ + u"FIREFOX_BETA_48_END", + u"FIREFOX_RELEASE_48_END", + u"FIREFOX_48_0_2_RELEASE", + u"FIREFOX_48_0_2_BUILD1", + u"FIREFOX_48_0_1_RELEASE", + u"FIREFOX_48_0_1_BUILD3", + u"FIREFOX_48_0_RELEASE", + u"FIREFOX_48_0_BUILD2", + u"FIREFOX_RELEASE_48_BASE", + u"FIREFOX_48_0b10_RELEASE", + u"FIREFOX_48_0b10_BUILD1", + u"FIREFOX_48_0b9_RELEASE", + u"FIREFOX_48_0b9_BUILD1", + u"FIREFOX_48_0b7_RELEASE", + u"FIREFOX_48_0b7_BUILD1", + u"FIREFOX_48_0b6_RELEASE", + u"FIREFOX_48_0b6_BUILD1", + u"FIREFOX_48_0b5_RELEASE", + u"FIREFOX_48_0b5_BUILD1", + u"FIREFOX_48_0b4_RELEASE", + u"FIREFOX_48_0b4_BUILD1", + u"FIREFOX_48_0b3_RELEASE", + u"FIREFOX_48_0b3_BUILD1", + u"FIREFOX_48_0b2_RELEASE", + u"FIREFOX_48_0b2_BUILD2", + u"FIREFOX_48_0b1_RELEASE", + u"FIREFOX_48_0b1_BUILD2", + u"FIREFOX_AURORA_48_END", + u"FIREFOX_BETA_48_BASE", + u"FIREFOX_AURORA_48_BASE", + ] + + mock_hg_json = {"tags": [{"tag": ff_48_tag} for ff_48_tag in ff_48_tags]} + + assert get_previous_tag_version(product, version, tag, mock_hg_json) == previous_tag + + +def test_get_bugs_in_changeset(): + with DATA_PATH.joinpath("buglist_changesets.json").open("r") as fp: + changeset_data = json.load(fp) + bugs, backouts = get_bugs_in_changeset(changeset_data) + + assert bugs == { + u"1356563", + u"1348409", + u"1341190", + u"1360626", + u"1332731", + u"1328762", + u"1355870", + u"1358089", + u"1354911", + u"1354038", + } + assert backouts == {u"1337861", u"1320072"} + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozrelease/test/test_update_verify.py b/python/mozrelease/test/test_update_verify.py new file mode 100644 index 0000000000..6f1cb197f7 --- /dev/null +++ b/python/mozrelease/test/test_update_verify.py @@ -0,0 +1,425 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest +from pathlib import Path +from tempfile import mkstemp + +import mozunit + +from mozrelease.update_verify import UpdateVerifyConfig, UpdateVerifyError + +DATA_PATH = Path(__file__).with_name("data") + + +class TestUpdateVerifyConfig(unittest.TestCase): + config = str(DATA_PATH.joinpath("sample-update-verify.cfg")) + + def setUp(self): + self.uvc = UpdateVerifyConfig() + fd, self.tmpfilename = mkstemp() + self.tmpfile = os.fdopen(fd, "wb") + + def tearDown(self): + self.tmpfile.close() + os.unlink(self.tmpfilename) + + def testEq(self): + self.uvc.product = "foo" + self.uvc.channel = "betatest" + self.uvc.aus_server = "aus" + self.uvc.ftp_server_from = "ftp" + self.uvc.ftp_server_to = "ftp" + self.uvc.to = "/firefox/4.0rc2.tar.bz2" + self.uvc.mar_channel_IDs = "baz" + self.uvc.to_build_id = "999" + self.uvc.to_display_version = "99.0 Zeta 9" + self.uvc.to_app_version = "99.0" + uvc2 = UpdateVerifyConfig() + uvc2.product = "foo" + uvc2.channel = "betatest" + uvc2.aus_server = "aus" + uvc2.ftp_server_form = "ftp" + uvc2.ftp_server_to = "ftp" + uvc2.to = "/firefox/4.0rc2.tar.bz2" + uvc2.mar_channel_IDs = "baz" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + self.assertEqual(self.uvc, uvc2) + + def testNe(self): + self.uvc.product = "foo" + uvc2 = UpdateVerifyConfig() + # assertNotEqual doesn't test the __ne__ function, so we do this + self.assertTrue(self.uvc != uvc2) + + def testAddRelease(self): + releases = [ + { + "release": "4.0", + "platform": "bar", + "build_id": 555, + "locales": ["af", "de"], + "patch_types": ["partial", "complete"], + "from": "/pub/firefox/foo.bz2", + "ftp_server_from": "from", + "ftp_server_to": "to", + "mar_channel_IDs": "firefox-mozilla-booyah", + "updater_package": None, + } + ] + self.uvc.addRelease( + "4.0", + build_id=555, + locales=["af", "de"], + patch_types=["partial", "complete"], + from_path="/pub/firefox/foo.bz2", + ftp_server_from="from", + ftp_server_to="to", + mar_channel_IDs="firefox-mozilla-booyah", + platform="bar", + ) + self.assertEqual(self.uvc.releases, releases) + + def testAddReleasesWithDifferentPlatforms(self): + releases = [ + { + "release": "4.0", + "platform": "WINNT_x86-msvc", + "build_id": 555, + "locales": ["af", "de"], + "patch_types": ["partial", "complete"], + "from": "/pub/firefox/foo.bz2", + "ftp_server_from": "from", + "ftp_server_to": "to", + "mar_channel_IDs": "firefox-mozilla-booyah", + "updater_package": None, + }, + { + "release": "5.0", + "platform": "WINNT_x86-msvc-x86", + "build_id": 666, + "locales": ["af", "de"], + "patch_types": ["partial", "complete"], + "from": "/pub/firefox/foo2.bz2", + "ftp_server_from": "from", + "ftp_server_to": "to", + "mar_channel_IDs": "firefox-mozilla-booyah", + "updater_package": None, + }, + ] + self.uvc.addRelease( + "4.0", + build_id=555, + locales=["af", "de"], + patch_types=["partial", "complete"], + from_path="/pub/firefox/foo.bz2", + ftp_server_from="from", + ftp_server_to="to", + mar_channel_IDs="firefox-mozilla-booyah", + platform="WINNT_x86-msvc", + ) + self.uvc.addRelease( + "5.0", + build_id=666, + locales=["af", "de"], + patch_types=["partial", "complete"], + from_path="/pub/firefox/foo2.bz2", + ftp_server_from="from", + ftp_server_to="to", + mar_channel_IDs="firefox-mozilla-booyah", + platform="WINNT_x86-msvc-x86", + ) + self.assertEqual(self.uvc.releases, releases) + + def testRead(self): + ftp_server_from = "stage.mozilla.org/firefox" + ftp_server_to = "stage.mozilla.org/firefox" + uvc2 = UpdateVerifyConfig() + uvc2.product = "Firefox" + uvc2.channel = "betatest" + uvc2.aus_server = "https://aus4.mozilla.org" + uvc2.to = "/firefox/4.0rc2.tar.bz2" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + uvc2.addRelease( + "4.0", + build_id="888", + platform="Linux_x86-gcc3", + locales=["af", "de", "en-US", "ja", "zh-TW"], + patch_types=["partial", "complete"], + from_path="/firefox/4.0rc1.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + mar_channel_IDs="firefox-mozilla-beta", + ) + uvc2.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["af", "en-US"], + from_path="/firefox/4.0b12.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + uvc2.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["de", "ja", "zh-TW"], + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + uvc2.addRelease( + "3.7a1", + build_id="666", + locales=["en-US"], + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + platform="Linux_x86-gcc3", + ) + + self.uvc.read(self.config) + self.assertEqual(self.uvc, uvc2) + + def testWrite(self): + ftp_server_from = "stage.mozilla.org/firefox" + ftp_server_to = "stage.mozilla.org/firefox" + self.uvc.product = "Firefox" + self.uvc.channel = "betatest" + self.uvc.aus_server = "https://aus4.mozilla.org" + self.uvc.to = "/firefox/4.0rc2.tar.bz2" + self.uvc.to_build_id = "999" + self.uvc.to_display_version = "99.0 Zeta 9" + self.uvc.to_app_version = "99.0" + self.uvc.addRelease( + "4.0", + build_id="888", + platform="Linux_x86-gcc3", + locales=("af", "de", "en-US", "ja", "zh-TW"), + patch_types=("partial", "complete"), + from_path="/firefox/4.0rc1.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + mar_channel_IDs="firefox-mozilla-beta", + ) + self.uvc.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["af", "en-US"], + from_path="/firefox/4.0b12.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + self.uvc.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=("de", "ja", "zh-TW"), + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + self.uvc.addRelease( + "3.7a1", + build_id="666", + locales=("en-US",), + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + platform="Linux_x86-gcc3", + ) + + self.uvc.write(self.tmpfile) + self.tmpfile.close() + self.assertEqual(open(self.config).read(), open(self.tmpfilename).read()) + + def testReadInvalidKey(self): + invalidLine = 'foo="bar"' + self.assertRaises(UpdateVerifyError, self.uvc._parseLine, invalidLine) + + def testReadDuplicateKey(self): + invalidLine = 'release="bar" release="blah"' + self.assertRaises(UpdateVerifyError, self.uvc._parseLine, invalidLine) + + def testParseLineBad(self): + invalidLine = "abh nthntuehonhuh nhhueont hntueoh nthouo" + self.assertRaises(UpdateVerifyError, self.uvc._parseLine, invalidLine) + + def testGetChunk(self): + ftp_server_from = "stage.mozilla.org/firefox" + ftp_server_to = "stage.mozilla.org/firefox" + self.uvc.read(self.config) + uvc2 = UpdateVerifyConfig() + uvc2.product = "Firefox" + uvc2.channel = "betatest" + uvc2.aus_server = "https://aus4.mozilla.org" + uvc2.to = "/firefox/4.0rc2.tar.bz2" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + uvc2.addRelease( + "4.0", + build_id="888", + platform="Linux_x86-gcc3", + locales=["af", "de", "en-US"], + patch_types=["partial", "complete"], + from_path="/firefox/4.0rc1.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + mar_channel_IDs="firefox-mozilla-beta", + ) + uvc2.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["de", "ja"], + patch_types=["complete"], + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + from_path=None, + ) + chunkedConfig = self.uvc.getChunk(chunks=3, thisChunk=1) + self.assertEqual(chunkedConfig, uvc2) + + def testGetChunkWithPathWithSpaces(self): + self.uvc.product = "Firefox" + self.uvc.channel = "betatest" + self.uvc.aus_server = "https://aus4.mozilla.org" + self.uvc.ftp_server_from = "stage.mozilla.org/firefox" + self.uvc.ftp_server_to = "stage.mozilla.org/firefox" + self.uvc.to = "/firefox/Firefox 4.0 Beta 2.exe" + self.uvc.to_build_id = "999" + self.uvc.to_display_version = "99.0 Zeta 9" + self.uvc.to_app_version = "99.0" + self.uvc.addRelease( + "4.0b1", + build_id="222", + platform="Linux_x86-gcc3", + locales=["en-US", "ja", "zh-TW"], + patch_types=["complete"], + from_path="/firefox/Firefox 4.0 Beta 1.exe", + ) + uvc2 = UpdateVerifyConfig() + uvc2.product = "Firefox" + uvc2.channel = "betatest" + uvc2.aus_server = "https://aus4.mozilla.org" + uvc2.ftp_server_from = "stage.mozilla.org/firefox" + uvc2.ftp_server_to = "stage.mozilla.org/firefox" + uvc2.to = "/firefox/Firefox 4.0 Beta 2.exe" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + uvc2.addRelease( + "4.0b1", + build_id="222", + platform="Linux_x86-gcc3", + locales=["en-US", "ja"], + patch_types=["complete"], + from_path="/firefox/Firefox 4.0 Beta 1.exe", + ) + chunkedConfig = self.uvc.getChunk(chunks=2, thisChunk=1) + self.assertEqual(chunkedConfig, uvc2) + + def testAddLocaleToRelease(self): + from_path = "/firefox/4.0rc1.tar.bz2" + self.uvc.read(self.config) + self.uvc.addLocaleToRelease("888", "he", from_path) + self.assertEqual( + self.uvc.getRelease("888", from_path)["locales"], + ["af", "de", "en-US", "he", "ja", "zh-TW"], + ) + + def testAddLocaleToReleaseMultipleBuildIDs(self): + from_path = None + self.uvc.read(self.config) + self.uvc.addLocaleToRelease("777", "he", from_path) + self.assertEqual( + self.uvc.getRelease("777", from_path)["locales"], + ["de", "he", "ja", "zh-TW"], + ) + + def testAddLocaleToNonexistentRelease(self): + self.uvc.read(self.config) + self.assertRaises(UpdateVerifyError, self.uvc.addLocaleToRelease, "123", "he") + + def testGetReleaseNonexistenceRelease(self): + self.uvc.read(self.config) + self.assertEqual(self.uvc.getRelease("123", None), {}) + + def testGetFullReleaseTests(self): + ftp_server_from = "stage.mozilla.org/firefox" + ftp_server_to = "stage.mozilla.org/firefox" + self.uvc.read(self.config) + uvc2 = UpdateVerifyConfig() + uvc2.product = "Firefox" + uvc2.channel = "betatest" + uvc2.aus_server = "https://aus4.mozilla.org" + uvc2.to = "/firefox/4.0rc2.tar.bz2" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + uvc2.addRelease( + "4.0", + build_id="888", + platform="Linux_x86-gcc3", + locales=["af", "de", "en-US", "ja", "zh-TW"], + patch_types=["partial", "complete"], + from_path="/firefox/4.0rc1.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + mar_channel_IDs="firefox-mozilla-beta", + ) + uvc2.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["af", "en-US"], + patch_types=["complete"], + from_path="/firefox/4.0b12.tar.bz2", + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + self.assertEqual(self.uvc.getFullReleaseTests(), uvc2.releases) + + def testGetQuickReleaseTests(self): + ftp_server_from = "stage.mozilla.org/firefox" + ftp_server_to = "stage.mozilla.org/firefox" + self.uvc.read(self.config) + uvc2 = UpdateVerifyConfig() + uvc2.product = "Firefox" + uvc2.channel = "betatest" + uvc2.aus_server = "https://aus4.mozilla.org" + uvc2.to = "/firefox/4.0rc2.tar.bz2" + uvc2.to_build_id = "999" + uvc2.to_display_version = "99.0 Zeta 9" + uvc2.to_app_version = "99.0" + uvc2.addRelease( + "4.0b12", + build_id="777", + platform="Linux_x86-gcc3", + locales=["de", "ja", "zh-TW"], + patch_types=["complete"], + from_path=None, + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + uvc2.addRelease( + "3.7a1", + build_id="666", + platform="Linux_x86-gcc3", + locales=["en-US"], + patch_types=["complete"], + from_path=None, + ftp_server_from=ftp_server_from, + ftp_server_to=ftp_server_to, + ) + self.assertEqual(self.uvc.getQuickReleaseTests(), uvc2.releases) + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozrelease/test/test_versions.py b/python/mozrelease/test/test_versions.py new file mode 100644 index 0000000000..f3bca91f1f --- /dev/null +++ b/python/mozrelease/test/test_versions.py @@ -0,0 +1,101 @@ +import mozunit +import pytest + +from mozrelease.versions import MozillaVersion + +ALL_VERSIONS = [ # Keep this sorted + "3.0", + "3.0.1", + "3.0.2", + "3.0.3", + "3.0.4", + "3.0.5", + "3.0.6", + "3.0.7", + "3.0.8", + "3.0.9", + "3.0.10", + "3.0.11", + "3.0.12", + "3.0.13", + "3.0.14", + "3.0.15", + "3.0.16", + "3.0.17", + "3.0.18", + "3.0.19", + "3.1b1", + "3.1b2", + "3.1b3", + "3.5b4", + "3.5b99", + "3.5rc1", + "3.5rc2", + "3.5rc3", + "3.5", + "3.5.1", + "3.5.2", + "3.5.3", + "3.5.4", + "3.5.5", + "3.5.6", + "3.5.7", + "3.5.8", + "3.5.9", + "3.5.10", + # ... Start skipping around... + "4.0b9", + "10.0.2esr", + "10.0.3esr", + "32.0", + "49.0a1", + "49.0a2", + "59.0", + "60.0", + "60.0esr", + "60.0.1esr", + "60.1", + "60.1esr", + "61.0", +] + + +@pytest.fixture( + scope="function", + params=range(len(ALL_VERSIONS) - 1), + ids=lambda x: "{}, {}".format(ALL_VERSIONS[x], ALL_VERSIONS[x + 1]), +) +def comparable_versions(request): + index = request.param + return ALL_VERSIONS[index], ALL_VERSIONS[index + 1] + + +@pytest.mark.parametrize("version", ALL_VERSIONS) +def test_versions_parseable(version): + """Test that we can parse previously shipped versions. + + We only test 3.0 and up, since we never generate updates against + versions that old.""" + assert MozillaVersion(version) is not None + + +def test_versions_compare_less(comparable_versions): + """Test that versions properly compare in order.""" + smaller_version, larger_version = comparable_versions + assert MozillaVersion(smaller_version) < MozillaVersion(larger_version) + + +def test_versions_compare_greater(comparable_versions): + """Test that versions properly compare in order.""" + smaller_version, larger_version = comparable_versions + assert MozillaVersion(larger_version) > MozillaVersion(smaller_version) + + +@pytest.mark.parametrize("version", ALL_VERSIONS) +def test_versions_compare_equal(version): + """Test that versions properly compare as equal through multiple passes.""" + assert MozillaVersion(version) == MozillaVersion(version) + + +if __name__ == "__main__": + mozunit.main() |