diff options
Diffstat (limited to 'python/mozrelease/mozrelease')
| -rw-r--r-- | python/mozrelease/mozrelease/__init__.py | 0 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/attribute_builds.py | 214 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/balrog.py | 72 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/buglist_creator.py | 261 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/chunking.py | 27 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/l10n.py | 17 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/mach_commands.py | 141 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/partner_repack.py | 895 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/paths.py | 85 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/platforms.py | 54 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/scriptworker_canary.py | 107 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/update_verify.py | 275 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/util.py | 26 | ||||
| -rw-r--r-- | python/mozrelease/mozrelease/versions.py | 114 |
14 files changed, 2288 insertions, 0 deletions
diff --git a/python/mozrelease/mozrelease/__init__.py b/python/mozrelease/mozrelease/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozrelease/mozrelease/__init__.py diff --git a/python/mozrelease/mozrelease/attribute_builds.py b/python/mozrelease/mozrelease/attribute_builds.py new file mode 100644 index 0000000000..094c70e1bf --- /dev/null +++ b/python/mozrelease/mozrelease/attribute_builds.py @@ -0,0 +1,214 @@ +#! /usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import json +import logging +import mmap +import os +import shutil +import struct +import sys +import tempfile +import urllib.parse +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") +log = logging.getLogger() + + +def write_attribution_data(filepath, data): + """Insert data into a prepared certificate in a signed PE file. + + Returns False if the file isn't a valid PE file, or if the necessary + certificate was not found. + + This function assumes that somewhere in the given file's certificate table + there exists a 1024-byte space which begins with the tag "__MOZCUSTOM__:". + The given data will be inserted into the file following this tag. + + We don't bother updating the optional header checksum. + Windows doesn't check it for executables, only drivers and certain DLL's. + """ + with open(filepath, "r+b") as file: + mapped = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_WRITE) + + # Get the location of the PE header and the optional header + pe_header_offset = struct.unpack("<I", mapped[0x3C:0x40])[0] + optional_header_offset = pe_header_offset + 24 + + # Look up the magic number in the optional header, + # so we know if we have a 32 or 64-bit executable. + # We need to know that so that we can find the data directories. + pe_magic_number = struct.unpack( + "<H", mapped[optional_header_offset : optional_header_offset + 2] + )[0] + if pe_magic_number == 0x10B: + # 32-bit + cert_dir_entry_offset = optional_header_offset + 128 + elif pe_magic_number == 0x20B: + # 64-bit. Certain header fields are wider. + cert_dir_entry_offset = optional_header_offset + 144 + else: + # Not any known PE format + mapped.close() + return False + + # The certificate table offset and length give us the valid range + # to search through for where we should put our data. + cert_table_offset = struct.unpack( + "<I", mapped[cert_dir_entry_offset : cert_dir_entry_offset + 4] + )[0] + cert_table_size = struct.unpack( + "<I", mapped[cert_dir_entry_offset + 4 : cert_dir_entry_offset + 8] + )[0] + + if cert_table_offset == 0 or cert_table_size == 0: + # The file isn't signed + mapped.close() + return False + + tag = b"__MOZCUSTOM__:" + tag_index = mapped.find( + tag, cert_table_offset, cert_table_offset + cert_table_size + ) + if tag_index == -1: + mapped.close() + return False + + # convert to quoted-url byte-string for insertion + data = urllib.parse.quote(data).encode("utf-8") + mapped[tag_index + len(tag) : tag_index + len(tag) + len(data)] = data + + return True + + +def validate_attribution_code(attribution): + log.info("Checking attribution %s" % attribution) + return_code = True + + if len(attribution) == 0: + log.error("Attribution code has 0 length") + return False + + # Set to match https://searchfox.org/mozilla-central/rev/a92ed79b0bc746159fc31af1586adbfa9e45e264/browser/components/attribution/AttributionCode.jsm#24 # noqa + MAX_LENGTH = 1010 + if len(attribution) > MAX_LENGTH: + log.error("Attribution code longer than %s chars" % MAX_LENGTH) + return_code = False + + # this leaves out empty values like 'foo=' + params = urllib.parse.parse_qsl(attribution) + used_keys = set() + for key, value in params: + # check for invalid keys + if key not in ( + "source", + "medium", + "campaign", + "content", + "experiment", + "variation", + "ua", + "dlsource", + ): + log.error("Invalid key %s" % key) + return_code = False + + # avoid ambiguity from repeated keys + if key in used_keys: + log.error("Repeated key %s" % key) + return_code = False + else: + used_keys.add(key) + + # TODO the service checks for valid source, should we do that here too ? + + # We have two types of attribution with different requirements: + # 1) Partner attribution, which requires a few UTM parameters sets + # 2) Attribution of vanilla builds, which only requires `dlsource` + # + # Perhaps in an ideal world we would check what type of build we're + # attributing to make sure that eg: partner builds don't get `dlsource` + # instead of what they actually want -- but the likelyhood of that + # happening is vanishingly small, so it's probably not worth doing. + if "dlsource" not in used_keys: + for key in ("source", "medium", "campaign", "content"): + if key not in used_keys: + return_code = False + + if return_code is False: + log.error( + "Either 'dlsource' must be provided, or all of: 'source', 'medium', 'campaign', and 'content'. Use '(not set)' if one of the latter is not needed." + ) + return return_code + + +def main(): + parser = argparse.ArgumentParser( + description="Add attribution to Windows installer(s).", + epilog=""" + By default, configuration from envvar ATTRIBUTION_CONFIG is used, with + expected format + [{"input": "in/abc.exe", "output": "out/def.exe", "attribution": "abcdef"}, + {"input": "in/ghi.exe", "output": "out/jkl.exe", "attribution": "ghijkl"}] + for 1 or more attributions. Or the script arguments may be used for a single attribution. + + The attribution code should be a string which is not url-encoded. + + If command line arguments are used instead, one or more `--input` parameters may be provided. + Each will be written to the `--output` directory provided to a file of the same name as the + input filename. All inputs will be attributed with the same `--attribution` code. + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--input", + default=[], + action="append", + help="Source installer to attribute; may be specified multiple times", + ) + parser.add_argument("--output", help="Location to write the attributed installers") + parser.add_argument("--attribution", help="Attribution code") + args = parser.parse_args() + + if os.environ.get("ATTRIBUTION_CONFIG"): + work = json.loads(os.environ["ATTRIBUTION_CONFIG"]) + elif args.input and args.output and args.attribution: + work = [] + for i in args.input: + fn = os.path.basename(i) + work.append( + { + "input": i, + "output": os.path.join(args.output, fn), + "attribution": args.attribution, + } + ) + else: + log.error("No configuration found. Set ATTRIBUTION_CONFIG or pass arguments.") + return 1 + + cached_code_checks = [] + for job in work: + if job["attribution"] not in cached_code_checks: + status = validate_attribution_code(job["attribution"]) + if status: + cached_code_checks.append(job["attribution"]) + else: + log.error("Failed attribution code check") + return 1 + + with tempfile.TemporaryDirectory() as td: + log.info("Attributing installer %s ..." % job["input"]) + tf = shutil.copy(job["input"], td) + if write_attribution_data(tf, job["attribution"]): + Path(job["output"]).parent.mkdir(parents=True, exist_ok=True) + shutil.move(tf, job["output"]) + log.info("Wrote %s" % job["output"]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/mozrelease/mozrelease/balrog.py b/python/mozrelease/mozrelease/balrog.py new file mode 100644 index 0000000000..31418d352e --- /dev/null +++ b/python/mozrelease/mozrelease/balrog.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def _generate_show_url(context, entry): + url = entry["url"] + return { + "actions": "showURL", + "openURL": url.format(**context), + } + + +def _generate_product_details(context, entry): + url = entry["url"] + return { + "detailsURL": url.format(**context), + "type": "minor", + } + + +_FIELD_TYPES = { + "show-url": _generate_show_url, + "product-details": _generate_product_details, +} + + +def _generate_conditions(context, entry): + if ( + "release-types" in entry + and context["release-type"] not in entry["release-types"] + ): + return None + if "blob-types" in entry and context["blob-type"] not in entry["blob-types"]: + return None + if "products" in entry and context["product"] not in entry["products"]: + return None + + conditions = {} + if "locales" in entry: + conditions["locales"] = entry["locales"] + if "versions" in entry: + conditions["versions"] = [ + version.format(**context) for version in entry["versions"] + ] + if "update-channel" in entry: + conditions["channels"] = [ + entry["update-channel"] + suffix + for suffix in ("", "-localtest", "-cdntest") + ] + if "build-ids" in entry: + conditions["buildIDs"] = [ + buildid.format(**context) for buildid in entry["build-ids"] + ] + return conditions + + +def generate_update_properties(context, config): + result = [] + for entry in config: + fields = _FIELD_TYPES[entry["type"]](context, entry) + conditions = _generate_conditions(context, entry.get("conditions", {})) + + if conditions is not None: + result.append( + { + "fields": fields, + "for": conditions, + } + ) + return result diff --git a/python/mozrelease/mozrelease/buglist_creator.py b/python/mozrelease/mozrelease/buglist_creator.py new file mode 100644 index 0000000000..8c7b8d0391 --- /dev/null +++ b/python/mozrelease/mozrelease/buglist_creator.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import re +from operator import itemgetter + +import requests +from mozilla_version.gecko import GeckoVersion +from taskcluster import Notify, optionsFromEnvironment + +BUGLIST_TEMPLATE = "* [Bugs since previous changeset]({url})\n" +BACKOUT_REGEX = re.compile(r"back(\s?)out|backed out|backing out", re.IGNORECASE) +BACKOUT_TEMPLATE = "* [Backouts since previous changeset]({url})\n" +BUGZILLA_BUGLIST_TEMPLATE = "https://bugzilla.mozilla.org/buglist.cgi?bug_id={bugs}" +BUG_NUMBER_REGEX = re.compile(r"bug \d+", re.IGNORECASE) +CHANGELOG_TO_FROM_STRING = "{product}_{version}_RELEASE" +CHANGESET_URL_TEMPLATE = ( + "{repo}/{logtype}" "?rev={to_version}+%25+{from_version}&revcount=1000" +) +FULL_CHANGESET_TEMPLATE = "* [Full Mercurial changelog]({url})\n" +LIST_DESCRIPTION_TEMPLATE = "Comparing Mercurial tag {from_version} to {to_version}:\n" +MAX_BUGS_IN_BUGLIST = 250 +MERCURIAL_TAGS_URL_TEMPLATE = "{repo}/json-tags" +NO_BUGS = "" # Return this when bug list can't be created +URL_SHORTENER_TEMPLATE = "https://bugzilla.mozilla.org/rest/bitly/shorten?url={url}" + +log = logging.getLogger(__name__) + + +def create_bugs_url(product, current_version, current_revision, repo=None): + """ + Creates list of bugs and backout bugs for release-drivers email + + :param release: dict -> containing information about release, from Ship-It + :return: str -> description of compared releases, with Bugzilla links + containing all bugs in changeset + """ + try: + # Extract the important data, ignore if beta1 release + if current_version.beta_number == 1: + # If the version is beta 1, don't make any links + return NO_BUGS + + if repo is None: + repo = get_repo_by_version(current_version) + # Get the tag version, for display purposes + current_version_tag = tag_version(product, current_version) + + # Get all Hg tags for this branch, determine the previous version + tag_url = MERCURIAL_TAGS_URL_TEMPLATE.format(repo=repo) + mercurial_tags_json = requests.get(tag_url).json() + previous_version_tag = get_previous_tag_version( + product, current_version, current_version_tag, mercurial_tags_json + ) + + # Get the changeset between these versions, parse for all unique bugs and backout bugs + resp = requests.get( + CHANGESET_URL_TEMPLATE.format( + repo=repo, + from_version=previous_version_tag, + to_version=current_revision, + logtype="json-log", + ) + ) + changeset_data = resp.json() + unique_bugs, unique_backout_bugs = get_bugs_in_changeset(changeset_data) + + # Return a descriptive string with links if any relevant bugs are found + if unique_bugs or unique_backout_bugs: + description = LIST_DESCRIPTION_TEMPLATE.format( + from_version=previous_version_tag, to_version=current_version_tag + ) + + if unique_bugs: + description += BUGLIST_TEMPLATE.format( + url=create_buglist_url(unique_bugs) + ) + if unique_backout_bugs: + description += BACKOUT_TEMPLATE.format( + url=create_buglist_url(unique_backout_bugs) + ) + + changeset_html = CHANGESET_URL_TEMPLATE.format( + repo=repo, + from_version=previous_version_tag, + to_version=current_revision, + logtype="log", + ) + description += FULL_CHANGESET_TEMPLATE.format(url=changeset_html) + + return description + else: + return NO_BUGS + + except Exception as err: + log.info(err) + return NO_BUGS + + +def get_bugs_in_changeset(changeset_data): + unique_bugs, unique_backout_bugs = set(), set() + for changeset in changeset_data["entries"]: + if is_excluded_change(changeset): + continue + + changeset_desc = changeset["desc"] + bug_re = BUG_NUMBER_REGEX.search(changeset_desc) + + if bug_re: + bug_number = bug_re.group().split(" ")[1] + + if is_backout_bug(changeset_desc): + unique_backout_bugs.add(bug_number) + else: + unique_bugs.add(bug_number) + + return unique_bugs, unique_backout_bugs + + +def is_excluded_change(changeset): + excluded_change_keywords = [ + "a=test-only", + "a=release", + ] + return any(keyword in changeset["desc"] for keyword in excluded_change_keywords) + + +def is_backout_bug(changeset_description): + return bool(BACKOUT_REGEX.search(changeset_description)) + + +def create_buglist_url(buglist): + return BUGZILLA_BUGLIST_TEMPLATE.format(bugs="%2C".join(buglist)) + + +def tag_version(product, version): + underscore_version = str(version).replace(".", "_") + return CHANGELOG_TO_FROM_STRING.format( + product=product.upper(), version=underscore_version + ) + + +def parse_tag_version(tag): + dot_version = ".".join(tag.split("_")[1:-1]) + return GeckoVersion.parse(dot_version) + + +def get_previous_tag_version( + product, + current_version, + current_version_tag, + mercurial_tags_json, +): + """ + Gets the previous hg version tag for the product and branch, given the current version tag + """ + + def _invalid_tag_filter(tag): + """Filters by product and removes incorrect major version + base, end releases""" + prod_major_version_re = r"^{product}_{major_version}".format( + product=product.upper(), major_version=current_version.major_number + ) + + return ( + "BASE" not in tag + and "END" not in tag + and "RELEASE" in tag + and re.match(prod_major_version_re, tag) + ) + + # Get rid of irrelevant tags, sort by date and extract the tag string + tags = { + (parse_tag_version(item["tag"]), item["tag"]) + for item in mercurial_tags_json["tags"] + if _invalid_tag_filter(item["tag"]) + } + # Add the current version to the list + tags.add((current_version, current_version_tag)) + tags = sorted(tags, key=lambda tag: tag[0]) + + # Find where the current version is and go back one to get the previous version + next_version_index = list(map(itemgetter(0), tags)).index(current_version) - 1 + + return tags[next_version_index][1] + + +def get_repo_by_version(version): + """ + Get the repo a given version is found on. + """ + if version.is_beta: + return "https://hg.mozilla.org/releases/mozilla-beta" + elif version.is_release: + return "https://hg.mozilla.org/releases/mozilla-release" + elif version.is_esr: + return "https://hg.mozilla.org/releases/mozilla-esr{}".format( + version.major_number + ) + else: + raise Exception( + "Unsupported version type {}: {}".format(version.version_type.name, version) + ) + + +def email_release_drivers( + addresses, + product, + version, + build_number, + repo, + revision, + task_group_id, +): + # Send an email to the mailing after the build + email_buglist_string = create_bugs_url(product, version, revision, repo=repo) + + content = """\ +A new build has been started: + +Commit: [{revision}]({repo}/rev/{revision}) +Task group: [{task_group_id}]({root_url}/tasks/groups/{task_group_id}) + +{email_buglist_string} +""".format( + repo=repo, + revision=revision, + root_url=os.environ["TASKCLUSTER_ROOT_URL"], + task_group_id=task_group_id, + email_buglist_string=email_buglist_string, + ) + + # On r-d, we prefix the subject of the email in order to simplify filtering + subject_prefix = "" + if product in {"fennec"}: + subject_prefix = "[mobile] " + if product in {"firefox", "devedition"}: + subject_prefix = "[desktop] " + + subject = "{} Build of {} {} build {}".format( + subject_prefix, product, version, build_number + ) + + # use proxy if configured, otherwise local credentials from env vars + if "TASKCLUSTER_PROXY_URL" in os.environ: + notify_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} + else: + notify_options = optionsFromEnvironment() + + notify = Notify(notify_options) + for address in addresses: + notify.email( + { + "address": address, + "subject": subject, + "content": content, + } + ) diff --git a/python/mozrelease/mozrelease/chunking.py b/python/mozrelease/mozrelease/chunking.py new file mode 100644 index 0000000000..8c45f74354 --- /dev/null +++ b/python/mozrelease/mozrelease/chunking.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from copy import copy + + +class ChunkingError(Exception): + pass + + +def getChunk(things, chunks, thisChunk): + if thisChunk > chunks: + raise ChunkingError( + "thisChunk (%d) is greater than total chunks (%d)" % (thisChunk, chunks) + ) + possibleThings = copy(things) + nThings = len(possibleThings) + for c in range(1, chunks + 1): + n = nThings // chunks + # If our things aren't evenly divisible by the number of chunks + # we need to append one more onto some of them + if c <= (nThings % chunks): + n += 1 + if c == thisChunk: + return possibleThings[0:n] + del possibleThings[0:n] diff --git a/python/mozrelease/mozrelease/l10n.py b/python/mozrelease/mozrelease/l10n.py new file mode 100644 index 0000000000..1e2a15878d --- /dev/null +++ b/python/mozrelease/mozrelease/l10n.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def getPlatformLocales(shipped_locales, platform): + platform_locales = [] + for line in shipped_locales.splitlines(): + locale = line.strip().split()[0] + # ja-JP-mac locale is a MacOS only locale + if locale == "ja-JP-mac" and not platform.startswith("mac"): + continue + # Skip the "ja" locale on MacOS + if locale == "ja" and platform.startswith("mac"): + continue + platform_locales.append(locale) + return platform_locales diff --git a/python/mozrelease/mozrelease/mach_commands.py b/python/mozrelease/mozrelease/mach_commands.py new file mode 100644 index 0000000000..e7c8da59fe --- /dev/null +++ b/python/mozrelease/mozrelease/mach_commands.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import sys + +from mach.decorators import Command, CommandArgument, SubCommand +from mozilla_version.gecko import GeckoVersion + + +@Command( + "release", + category="release", + description="Task that are part of the release process.", +) +def release(command_context): + """ + The release subcommands all relate to the release process. + """ + + +@SubCommand( + "release", + "buglist", + description="Generate list of bugs since the last release.", +) +@CommandArgument( + "--version", + required=True, + type=GeckoVersion.parse, + help="The version being built.", +) +@CommandArgument("--product", required=True, help="The product being built.") +@CommandArgument("--repo", help="The repo being built.") +@CommandArgument("--revision", required=True, help="The revision being built.") +def buglist(command_context, version, product, revision, repo): + setup_logging(command_context) + from mozrelease.buglist_creator import create_bugs_url + + print( + create_bugs_url( + product=product, + current_version=version, + current_revision=revision, + repo=repo, + ) + ) + + +@SubCommand( + "release", + "send-buglist-email", + description="Send an email with the bugs since the last release.", +) +@CommandArgument( + "--address", + required=True, + action="append", + dest="addresses", + help="The email address to send the bug list to " + "(may be specified more than once.", +) +@CommandArgument( + "--version", + type=GeckoVersion.parse, + required=True, + help="The version being built.", +) +@CommandArgument("--product", required=True, help="The product being built.") +@CommandArgument("--repo", required=True, help="The repo being built.") +@CommandArgument("--revision", required=True, help="The revision being built.") +@CommandArgument("--build-number", required=True, help="The build number") +@CommandArgument("--task-group-id", help="The task group of the build.") +def buglist_email(command_context, **options): + setup_logging(command_context) + from mozrelease.buglist_creator import email_release_drivers + + email_release_drivers(**options) + + +@SubCommand( + "release", + "push-scriptworker-canary", + description="Push tasks to try, to test new scriptworker deployments.", +) +@CommandArgument( + "--address", + required=True, + action="append", + dest="addresses", + help="The email address to send notifications to " + "(may be specified more than once).", +) +@CommandArgument( + "--scriptworker", + required=True, + action="append", + dest="scriptworkers", + help="Scriptworker to run canary for (may be specified more than once).", +) +@CommandArgument( + "--ssh-key-secret", + required=False, + help="Taskcluster secret with ssh-key to use for hg.mozilla.org", +) +def push_scriptworker_canary(command_context, scriptworkers, addresses, ssh_key_secret): + setup_logging(command_context) + from mozrelease.scriptworker_canary import push_canary + + push_canary( + scriptworkers=scriptworkers, + addresses=addresses, + ssh_key_secret=ssh_key_secret, + ) + + +def setup_logging(command_context, quiet=False, verbose=True): + """ + Set up Python logging for all loggers, sending results to stderr (so + that command output can be redirected easily) and adding the typical + mach timestamp. + """ + # remove the old terminal handler + old = command_context.log_manager.replace_terminal_handler(None) + + # re-add it, with level and fh set appropriately + if not quiet: + level = logging.DEBUG if verbose else logging.INFO + command_context.log_manager.add_terminal_logging( + fh=sys.stderr, + level=level, + write_interval=old.formatter.write_interval, + write_times=old.formatter.write_times, + ) + + # all of the taskgraph logging is unstructured logging + command_context.log_manager.enable_unstructured() diff --git a/python/mozrelease/mozrelease/partner_repack.py b/python/mozrelease/mozrelease/partner_repack.py new file mode 100644 index 0000000000..1d64f43cca --- /dev/null +++ b/python/mozrelease/mozrelease/partner_repack.py @@ -0,0 +1,895 @@ +#!/usr/bin/env python +# Documentation: https://firefox-source-docs.mozilla.org/taskcluster/partner-repacks.html + +import json +import logging +import os +import re +import stat +import sys +import tarfile +import urllib.parse +import urllib.request +import zipfile +from optparse import OptionParser +from pathlib import Path +from shutil import copy, copytree, move, rmtree, which +from subprocess import Popen + +from redo import retry + +logging.basicConfig( + stream=sys.stdout, + level=logging.INFO, + format="%(asctime)-15s - %(levelname)s - %(message)s", +) +log = logging.getLogger(__name__) + + +# Set default values. +PARTNERS_DIR = Path("..") / ".." / "workspace" / "partners" +# No platform in this path because script only supports repacking a single platform at once +DEFAULT_OUTPUT_DIR = "%(partner)s/%(partner_distro)s/%(locale)s" +TASKCLUSTER_ARTIFACTS = ( + os.environ.get("TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com") + + "/api/queue/v1/task/{taskId}/artifacts" +) +UPSTREAM_ENUS_PATH = "public/build/{filename}" +UPSTREAM_L10N_PATH = "public/build/{locale}/{filename}" + +WINDOWS_DEST_DIR = Path("firefox") +MAC_DEST_DIR = Path("Contents/Resources") +LINUX_DEST_DIR = Path("firefox") + +BOUNCER_PRODUCT_TEMPLATE = ( + "partner-firefox-{release_type}-{partner}-{partner_distro}-latest" +) + + +class StrictFancyURLopener(urllib.request.FancyURLopener): + """Unlike FancyURLopener this class raises exceptions for generic HTTP + errors, like 404, 500. It reuses URLopener.http_error_default redefined in + FancyURLopener""" + + def http_error_default(self, url, fp, errcode, errmsg, headers): + urllib.request.URLopener.http_error_default( + self, url, fp, errcode, errmsg, headers + ) + + +def rmdirRecursive(directory: Path): + """ + This is similar to a call of shutil.rmtree(), except that it + should work better on Windows since it will more aggressively + attempt to remove files marked as "read-only". + """ + + def rmdir_including_read_only(func, path: str, exc_info): + """ + Source: https://stackoverflow.com/a/4829285 + path contains the path of the file that couldn't be removed. + Let's just assume that it's read-only and unlink it. + """ + path = Path(path) + + path.chmod(mode=stat.S_IWRITE) + path.unlink() + + rmtree(str(directory), onerror=rmdir_including_read_only) + + +def printSeparator(): + log.info("##################################################") + + +def shellCommand(cmd): + log.debug("Executing %s" % cmd) + log.debug(f"in {Path.cwd()}") + # Shell command output gets dumped immediately to stdout, whereas + # print statements get buffered unless we flush them explicitly. + sys.stdout.flush() + p = Popen(cmd, shell=True) + (_, ret) = os.waitpid(p.pid, 0) + if ret != 0: + ret_real = (ret & 0xFF00) >> 8 + log.error("Error: shellCommand had non-zero exit status: %d" % ret_real) + log.error("Command: %s" % cmd, exc_info=True) + sys.exit(ret_real) + return True + + +def isLinux(platform: str): + return "linux" in platform + + +def isLinux32(platform: str): + return "linux32" in platform or "linux-i686" in platform or platform == "linux" + + +def isLinux64(platform: str): + return "linux64" in platform or "linux-x86_64" in platform + + +def isMac(platform: str): + return "mac" in platform + + +def isWin(platform: str): + return "win" in platform + + +def isWin32(platform: str): + return "win32" in platform + + +def isWin64(platform: str): + return platform == "win64" + + +def isWin64Aarch64(platform: str): + return platform == "win64-aarch64" + + +def isValidPlatform(platform: str): + return ( + isLinux64(platform) + or isLinux32(platform) + or isMac(platform) + or isWin64(platform) + or isWin64Aarch64(platform) + or isWin32(platform) + ) + + +def parseRepackConfig(file: Path, platform: str): + """Did you hear about this cool file format called yaml ? json ? Yeah, me neither""" + config = {} + config["platforms"] = [] + for line in file.open(): + line = line.rstrip("\n") + # Ignore empty lines + if line.strip() == "": + continue + # Ignore comments + if line.startswith("#"): + continue + [key, value] = line.split("=", 2) + value = value.strip('"') + # strings that don't need special handling + if key in ("dist_id", "replacement_setup_exe"): + config[key] = value + continue + # booleans that don't need special handling + if key in ("migrationWizardDisabled", "oem", "repack_stub_installer"): + if value.lower() == "true": + config[key] = True + continue + # special cases + if key == "locales": + config["locales"] = value.split(" ") + continue + if key.startswith("locale."): + config[key] = value + continue + if key == "deb_section": + config["deb_section"] = re.sub("/", "\/", value) + continue + if isValidPlatform(key): + ftp_platform = getFtpPlatform(key) + if ftp_platform == getFtpPlatform(platform) and value.lower() == "true": + config["platforms"].append(ftp_platform) + continue + + # this only works for one locale because setup.exe is localised + if config.get("replacement_setup_exe") and len(config.get("locales", [])) > 1: + log.error( + "Error: replacement_setup_exe is only supported for one locale, got %s" + % config["locales"] + ) + sys.exit(1) + # also only works for one platform because setup.exe is platform-specific + + if config["platforms"]: + return config + + +def getFtpPlatform(platform: str): + """Returns the platform in the format used in building package names. + Note: we rely on this code being idempotent + i.e. getFtpPlatform(getFtpPlatform(foo)) should work + """ + if isLinux64(platform): + return "linux-x86_64" + if isLinux(platform): + return "linux-i686" + if isMac(platform): + return "mac" + if isWin64Aarch64(platform): + return "win64-aarch64" + if isWin64(platform): + return "win64" + if isWin32(platform): + return "win32" + + +def getFileExtension(platform: str): + """The extension for the output file, which may be passed to the internal-signing task""" + if isLinux(platform): + return "tar.bz2" + elif isMac(platform): + return "tar.gz" + elif isWin(platform): + return "zip" + + +def getFilename(platform: str): + """Returns the filename to be repacked for the platform""" + return f"target.{getFileExtension(platform)}" + + +def getAllFilenames(platform: str, repack_stub_installer): + """Returns the full list of filenames we want to downlaod for each platform""" + file_names = [getFilename(platform)] + if isWin(platform): + # we want to copy forward setup.exe from upstream tasks to make it easier to repackage + # windows installers later + file_names.append("setup.exe") + # Same for the stub installer with setup-stub.exe, but only in win32 repack jobs + if isWin32(platform) and repack_stub_installer: + file_names.append("setup-stub.exe") + return tuple(file_names) + + +def getTaskArtifacts(taskId): + try: + retrieveFile( + TASKCLUSTER_ARTIFACTS.format(taskId=taskId), Path("tc_artifacts.json") + ) + tc_index = json.load(open("tc_artifacts.json")) + return tc_index["artifacts"] + except (ValueError, KeyError): + log.error("Failed to get task artifacts from TaskCluster") + raise + + +def getUpstreamArtifacts(upstream_tasks, repack_stub_installer): + useful_artifacts = getAllFilenames(options.platform, repack_stub_installer) + + artifact_ids = {} + for taskId in upstream_tasks: + for artifact in getTaskArtifacts(taskId): + name = artifact["name"] + if not name.endswith(useful_artifacts): + continue + if name in artifact_ids: + log.error( + "Duplicated artifact %s processing tasks %s & %s", + name, + taskId, + artifacts[name], + ) + sys.exit(1) + else: + artifact_ids[name] = taskId + log.debug( + "Found artifacts: %s" % json.dumps(artifact_ids, indent=4, sort_keys=True) + ) + return artifact_ids + + +def getArtifactNames(platform: str, locale, repack_stub_installer): + file_names = getAllFilenames(platform, repack_stub_installer) + if locale == "en-US": + names = [UPSTREAM_ENUS_PATH.format(filename=f) for f in file_names] + else: + names = [ + UPSTREAM_L10N_PATH.format(locale=locale, filename=f) for f in file_names + ] + return names + + +def retrieveFile(url, file_path: Path): + success = True + url = urllib.parse.quote(url, safe=":/") + log.info(f"Downloading from {url}") + log.info(f"To: {file_path}") + log.info(f"CWD: {Path.cwd()}") + try: + # use URLopener, which handles errors properly + retry( + StrictFancyURLopener().retrieve, + kwargs=dict(url=url, filename=str(file_path)), + ) + except IOError: + log.error("Error downloading %s" % url, exc_info=True) + success = False + try: + file_path.unlink() + except OSError: + log.info(f"Cannot remove {file_path}", exc_info=True) + + return success + + +def getBouncerProduct(partner, partner_distro): + if "RELEASE_TYPE" not in os.environ: + log.fatal("RELEASE_TYPE must be set in the environment") + sys.exit(1) + release_type = os.environ["RELEASE_TYPE"] + # For X.0 releases we get 'release-rc' but the alias should use 'release' + if release_type == "release-rc": + release_type = "release" + return BOUNCER_PRODUCT_TEMPLATE.format( + release_type=release_type, + partner=partner, + partner_distro=partner_distro, + ) + + +class RepackBase(object): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + file_mode=0o644, + quiet=False, + source_locale=None, + locale=None, + ): + self.base_dir = Path.cwd() + self.build = build + self.full_build_path = build_dir / build + if not self.full_build_path.is_absolute(): + self.full_build_path = self.base_dir / self.full_build_path + self.full_partner_path = self.base_dir / partner_dir + self.working_dir = final_dir / "working" + self.final_dir = final_dir + self.final_build = final_dir / Path(build).name + self.ftp_platform = ftp_platform + self.repack_info = repack_info + self.file_mode = file_mode + self.quiet = quiet + self.source_locale = source_locale + self.locale = locale + self.working_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + + def announceStart(self): + log.info( + "Repacking %s %s build %s" % (self.ftp_platform, self.locale, self.build) + ) + + def announceSuccess(self): + log.info( + "Done repacking %s %s build %s" + % (self.ftp_platform, self.locale, self.build) + ) + + def unpackBuild(self): + copy(str(self.full_build_path), ".") + + def createOverrideIni(self, partner_path: Path): + """If this is a partner specific locale (like en-HK), set the + distribution.ini to use that locale, not the default locale. + """ + if self.locale != self.source_locale: + file_path = partner_path / "distribution" / "distribution.ini" + with file_path.open(file_path.is_file() and "a" or "w") as open_file: + open_file.write("[Locale]\n") + open_file.write("locale=" + self.locale + "\n") + + """ Some partners need to override the migration wizard. This is done + by adding an override.ini file to the base install dir. + """ + # modify distribution.ini if 44 or later and we have migrationWizardDisabled + if int(options.version.split(".")[0]) >= 44: + file_path = partner_path / "distribution" / "distribution.ini" + with file_path.open() as open_file: + ini = open_file.read() + + if ini.find("EnableProfileMigrator") >= 0: + return + else: + browser_dir = partner_path / "browser" + if not browser_dir.exists(): + browser_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + file_path = browser_dir / "override.ini" + if "migrationWizardDisabled" in self.repack_info: + log.info("Adding EnableProfileMigrator to %r" % (file_path,)) + with file_path.open(file_path.is_file() and "a" or "w") as open_file: + open_file.write("[XRE]\n") + open_file.write("EnableProfileMigrator=0\n") + + def copyFiles(self, platform_dir: Path): + log.info(f"Copying files into {platform_dir}") + # Check whether we've already copied files over for this partner. + if not platform_dir.exists(): + platform_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + for i in ["distribution", "extensions"]: + full_path = self.full_partner_path / i + if full_path.exists(): + copytree(str(full_path), str(platform_dir / i)) + self.createOverrideIni(platform_dir) + + def repackBuild(self): + pass + + def stage(self): + move(self.build, str(self.final_dir)) + self.final_build.chmod(self.file_mode) + + def cleanup(self): + self.final_build.unlink() + + def doRepack(self): + self.announceStart() + os.chdir(self.working_dir) + self.unpackBuild() + self.copyFiles() + self.repackBuild() + self.stage() + os.chdir(self.base_dir) + rmdirRecursive(self.working_dir) + self.announceSuccess() + + +class RepackLinux(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackLinux, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + self.uncompressed_build = build.replace(".bz2", "") + + def unpackBuild(self): + super(RepackLinux, self).unpackBuild() + bunzip2_cmd = "bunzip2 %s" % self.build + shellCommand(bunzip2_cmd) + if not Path(self.uncompressed_build).exists(): + log.error(f"Error: Unable to uncompress build {self.build}") + sys.exit(1) + + def copyFiles(self): + super(RepackLinux, self).copyFiles(LINUX_DEST_DIR) + + def repackBuild(self): + if options.quiet: + tar_flags = "rf" + else: + tar_flags = "rvf" + tar_cmd = "tar %s %s %s" % (tar_flags, self.uncompressed_build, LINUX_DEST_DIR) + shellCommand(tar_cmd) + bzip2_command = "bzip2 %s" % self.uncompressed_build + shellCommand(bzip2_command) + + +class RepackMac(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackMac, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + self.uncompressed_build = build.replace(".gz", "") + + def unpackBuild(self): + super(RepackMac, self).unpackBuild() + gunzip_cmd = "gunzip %s" % self.build + shellCommand(gunzip_cmd) + if not Path(self.uncompressed_build).exists(): + log.error(f"Error: Unable to uncompress build {self.build}") + sys.exit(1) + self.appName = self.getAppName() + + def getAppName(self): + # Cope with Firefox.app vs Firefox Nightly.app by returning the first root object/folder found + t = tarfile.open(self.build.rsplit(".", 1)[0]) + for name in t.getnames(): + root_object = name.split("/")[0] + if root_object.endswith(".app"): + log.info(f"Found app name in tarball: {root_object}") + return root_object + log.error( + f"Error: Unable to determine app name from tarball: {self.build} - Expected .app in root" + ) + sys.exit(1) + + def copyFiles(self): + super(RepackMac, self).copyFiles(Path(self.appName) / MAC_DEST_DIR) + + def repackBuild(self): + if options.quiet: + tar_flags = "rf" + else: + tar_flags = "rvf" + # the final arg is quoted because it may contain a space, eg Firefox Nightly.app/.... + tar_cmd = "tar %s %s '%s'" % ( + tar_flags, + self.uncompressed_build, + Path(self.appName) / MAC_DEST_DIR, + ) + shellCommand(tar_cmd) + gzip_command = "gzip %s" % self.uncompressed_build + shellCommand(gzip_command) + + +class RepackWin(RepackBase): + def __init__( + self, + build: str, + partner_dir: Path, + build_dir: Path, + final_dir: Path, + ftp_platform: str, + repack_info, + **kwargs, + ): + super(RepackWin, self).__init__( + build, + partner_dir, + build_dir, + final_dir, + ftp_platform, + repack_info, + **kwargs, + ) + + def copyFiles(self): + super(RepackWin, self).copyFiles(WINDOWS_DEST_DIR) + + def repackBuild(self): + if options.quiet: + zip_flags = "-rq" + else: + zip_flags = "-r" + zip_cmd = f"zip {zip_flags} {self.build} {WINDOWS_DEST_DIR}" + shellCommand(zip_cmd) + + # we generate the stub installer during the win32 build, so repack it on win32 too + if isWin32(options.platform) and self.repack_info.get("repack_stub_installer"): + log.info("Creating target-stub.zip to hold custom urls") + dest = str(self.final_build).replace("target.zip", "target-stub.zip") + z = zipfile.ZipFile(dest, "w") + # load the partner.ini template and interpolate %LOCALE% to the actual locale + with (self.full_partner_path / "stub" / "partner.ini").open() as open_file: + partner_ini_template = open_file.readlines() + partner_ini = "" + for l in partner_ini_template: + l = l.replace("%LOCALE%", self.locale) + l = l.replace("%BOUNCER_PRODUCT%", self.repack_info["bouncer_product"]) + partner_ini += l + z.writestr("partner.ini", partner_ini) + # we need an empty firefox directory to use the repackage code + d = zipfile.ZipInfo("firefox/") + # https://stackoverflow.com/a/6297838, zip's representation of drwxr-xr-x permissions + # is 040755 << 16L, bitwise OR with 0x10 for the MS-DOS directory flag + d.external_attr = 1106051088 + z.writestr(d, "") + z.close() + + def stage(self): + super(RepackWin, self).stage() + setup_dest = Path(str(self.final_build).replace("target.zip", "setup.exe")) + if "replacement_setup_exe" in self.repack_info: + log.info("Overriding setup.exe with custom copy") + retrieveFile(self.repack_info["replacement_setup_exe"], setup_dest) + else: + # otherwise copy forward the vanilla copy + log.info("Copying vanilla setup.exe forward for installer creation") + setup = str(self.full_build_path).replace("target.zip", "setup.exe") + copy(setup, str(setup_dest)) + setup_dest.chmod(self.file_mode) + + # we generate the stub installer in the win32 build, so repack it on win32 too + if isWin32(options.platform) and self.repack_info.get("repack_stub_installer"): + log.info( + "Copying vanilla setup-stub.exe forward for stub installer creation" + ) + setup_dest = Path( + str(self.final_build).replace("target.zip", "setup-stub.exe") + ) + setup_source = str(self.full_build_path).replace( + "target.zip", "setup-stub.exe" + ) + copy(setup_source, str(setup_dest)) + setup_dest.chmod(self.file_mode) + + +if __name__ == "__main__": + error = False + partner_builds = {} + repack_build = { + "linux-i686": RepackLinux, + "linux-x86_64": RepackLinux, + "mac": RepackMac, + "win32": RepackWin, + "win64": RepackWin, + "win64-aarch64": RepackWin, + } + + parser = OptionParser(usage="usage: %prog [options]") + parser.add_option( + "-d", + "--partners-dir", + dest="partners_dir", + default=str(PARTNERS_DIR), + help="Specify the directory where the partner config files are found", + ) + parser.add_option( + "-p", + "--partner", + dest="partner", + help="Repack for a single partner, specified by name", + ) + parser.add_option( + "-v", "--version", dest="version", help="Set the version number for repacking" + ) + parser.add_option( + "-n", + "--build-number", + dest="build_number", + default=1, + help="Set the build number for repacking", + ) + parser.add_option("--platform", dest="platform", help="Set the platform to repack") + parser.add_option( + "--include-oem", + action="store_true", + dest="include_oem", + default=False, + help="Process partners marked as OEM (these are usually one-offs)", + ) + parser.add_option( + "-q", + "--quiet", + action="store_true", + dest="quiet", + default=False, + help="Suppress standard output from the packaging tools", + ) + parser.add_option( + "--taskid", + action="append", + dest="upstream_tasks", + help="Specify taskIds for upstream artifacts, using 'internal sign' tasks. Multiples " + "expected, e.g. --taskid foo --taskid bar. Alternatively, use a space-separated list " + "stored in UPSTREAM_TASKIDS in the environment.", + ) + parser.add_option( + "-l", + "--limit-locale", + action="append", + dest="limit_locales", + default=[], + ) + + (options, args) = parser.parse_args() + + if not options.quiet: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.WARNING) + + options.partners_dir = Path(options.partners_dir.rstrip("/")) + if not options.partners_dir.is_dir(): + log.error(f"Error: partners dir {options.partners_dir} is not a directory.") + error = True + + if not options.version: + log.error("Error: you must specify a version number.") + error = True + + if not options.platform: + log.error("No platform specified.") + error = True + + if not isValidPlatform(options.platform): + log.error("Invalid platform %s." % options.platform) + error = True + + upstream_tasks = options.upstream_tasks or os.getenv("UPSTREAM_TASKIDS") + if not upstream_tasks: + log.error( + "upstream tasks should be defined using --taskid args or " + "UPSTREAM_TASKIDS in env." + ) + error = True + + for tool in ("tar", "bunzip2", "bzip2", "gunzip", "gzip", "zip"): + if not which(tool): + log.error(f"Error: couldn't find the {tool} executable in PATH.") + error = True + + if error: + sys.exit(1) + + base_workdir = Path.cwd() + + # Look up the artifacts available on our upstreams, but only if we need to + artifact_ids = {} + + # Local directories for builds + script_directory = Path.cwd() + original_builds_dir = ( + script_directory + / "original_builds" + / options.version + / f"build{options.build_number}" + ) + repack_version = f"{options.version}-{options.build_number}" + if os.getenv("MOZ_AUTOMATION"): + # running in production + repacked_builds_dir = Path("/builds/worker/artifacts") + else: + # local development + repacked_builds_dir = script_directory / "artifacts" + original_builds_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + repacked_builds_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + printSeparator() + + # For each partner in the partners dir + # Read/check the config file + # Download required builds (if not already on disk) + # Perform repacks + + # walk the partner dirs, find valid repack.cfg configs, and load them + partner_dirs = [] + need_stub_installers = False + for root, _, all_files in os.walk(options.partners_dir): + root = root.lstrip("/") + partner = root[len(str(options.partners_dir)) + 1 :].split("/")[0] + partner_distro = os.path.split(root)[-1] + if options.partner: + if ( + options.partner != partner + and options.partner != partner_distro[: len(options.partner)] + ): + continue + + for file in all_files: + if file == "repack.cfg": + log.debug( + "Found partner config: {} ['{}'] {}".format( + root, "', '".join(_), file + ) + ) + root = Path(root) + repack_cfg = root / file + repack_info = parseRepackConfig(repack_cfg, options.platform) + if not repack_info: + log.debug( + "no repack_info for platform %s in %s, skipping" + % (options.platform, repack_cfg) + ) + continue + if repack_info.get("repack_stub_installer"): + need_stub_installers = True + repack_info["bouncer_product"] = getBouncerProduct( + partner, partner_distro + ) + partner_dirs.append((partner, partner_distro, root, repack_info)) + + log.info("Retrieving artifact lists from upstream tasks") + artifact_ids = getUpstreamArtifacts(upstream_tasks, need_stub_installers) + if not artifact_ids: + log.fatal("No upstream artifacts were found") + sys.exit(1) + + for partner, partner_distro, full_partner_dir, repack_info in partner_dirs: + log.info( + "Starting repack process for partner: %s/%s" % (partner, partner_distro) + ) + if "oem" in repack_info and options.include_oem is False: + log.info( + "Skipping partner: %s - marked as OEM and --include-oem was not set" + % partner + ) + continue + + repack_stub_installer = repack_info.get("repack_stub_installer") + # where everything ends up + partner_repack_dir = repacked_builds_dir / DEFAULT_OUTPUT_DIR + + # Figure out which base builds we need to repack. + for locale in repack_info["locales"]: + if options.limit_locales and locale not in options.limit_locales: + log.info("Skipping %s because it is not in limit_locales list", locale) + continue + source_locale = locale + # Partner has specified a different locale to + # use as the base for their custom locale. + if "locale." + locale in repack_info: + source_locale = repack_info["locale." + locale] + for platform in repack_info["platforms"]: + # ja-JP-mac only exists for Mac, so skip non-existent + # platform/locale combos. + if (source_locale == "ja" and isMac(platform)) or ( + source_locale == "ja-JP-mac" and not isMac(platform) + ): + continue + ftp_platform = getFtpPlatform(platform) + + local_filepath = original_builds_dir / ftp_platform / locale + local_filepath.mkdir(mode=0o755, exist_ok=True, parents=True) + final_dir = Path( + str(partner_repack_dir) + % dict( + partner=partner, + partner_distro=partner_distro, + locale=locale, + ) + ) + if final_dir.exists(): + rmdirRecursive(final_dir) + final_dir.mkdir(mode=0o755, exist_ok=True, parents=True) + + # for the main repacking artifact + file_name = getFilename(ftp_platform) + local_filename = local_filepath / file_name + + # Check to see if this build is already on disk, i.e. + # has already been downloaded. + artifacts = getArtifactNames(platform, locale, repack_stub_installer) + for artifact in artifacts: + local_artifact = local_filepath / Path(artifact).name + if local_artifact.exists(): + log.info(f"Found {local_artifact} on disk, not downloading") + continue + + if artifact not in artifact_ids: + log.fatal( + "Can't determine what taskID to retrieve %s from", artifact + ) + sys.exit(1) + original_build_url = "%s/%s" % ( + TASKCLUSTER_ARTIFACTS.format(taskId=artifact_ids[artifact]), + artifact, + ) + retrieveFile(original_build_url, local_artifact) + + # Make sure we have the local file now + if not local_filename.exists(): + log.info(f"Error: Unable to retrieve {file_name}\n") + sys.exit(1) + + repackObj = repack_build[ftp_platform]( + file_name, + full_partner_dir, + local_filepath, + final_dir, + ftp_platform, + repack_info, + locale=locale, + source_locale=source_locale, + ) + repackObj.doRepack() diff --git a/python/mozrelease/mozrelease/paths.py b/python/mozrelease/mozrelease/paths.py new file mode 100644 index 0000000000..b3d48c4ac7 --- /dev/null +++ b/python/mozrelease/mozrelease/paths.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from six.moves.urllib.parse import urlunsplit + +product_ftp_map = { + "fennec": "mobile", +} + + +def product2ftp(product): + return product_ftp_map.get(product, product) + + +def getCandidatesDir(product, version, buildNumber, protocol=None, server=None): + if protocol: + assert server is not None, "server is required with protocol" + + product = product2ftp(product) + directory = "/{}/candidates/{}-candidates/build{}".format( + product, + str(version), + str(buildNumber), + ) + + if protocol: + return urlunsplit((protocol, server, directory, None, None)) + else: + return directory + + +def getReleasesDir(product, version=None, protocol=None, server=None): + if protocol: + assert server is not None, "server is required with protocol" + + directory = "/{}/releases".format(product) + if version: + directory = "{}/{}".format(directory, version) + + if protocol: + return urlunsplit((protocol, server, directory, None, None)) + else: + return directory + + +def getReleaseInstallerPath(productName, brandName, version, platform, locale="en-US"): + if productName not in ("fennec",): + if platform.startswith("linux"): + return "/".join( + [ + p.strip("/") + for p in [ + platform, + locale, + "%s-%s.tar.bz2" % (productName, version), + ] + ] + ) + elif "mac" in platform: + return "/".join( + [ + p.strip("/") + for p in [platform, locale, "%s %s.dmg" % (brandName, version)] + ] + ) + elif platform.startswith("win"): + return "/".join( + [ + p.strip("/") + for p in [ + platform, + locale, + "%s Setup %s.exe" % (brandName, version), + ] + ] + ) + else: + raise "Unsupported platform" + else: + if platform.startswith("android"): + filename = "%s-%s.%s.android-arm.apk" % (productName, version, locale) + return "/".join([p.strip("/") for p in [platform, locale, filename]]) + else: + raise "Unsupported platform" diff --git a/python/mozrelease/mozrelease/platforms.py b/python/mozrelease/mozrelease/platforms.py new file mode 100644 index 0000000000..2970725a73 --- /dev/null +++ b/python/mozrelease/mozrelease/platforms.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +update_platform_map = { + "android": ["Android_arm-eabi-gcc3"], + "android-arm": ["Android_arm-eabi-gcc3"], + "android-x86": ["Android_x86-gcc3"], + "android-x86_64": ["Android_x86-64-gcc3"], + "android-aarch64": ["Android_aarch64-gcc3"], + "linux-i686": ["Linux_x86-gcc3"], + "linux-x86_64": ["Linux_x86_64-gcc3"], + "mac": [ + "Darwin_x86_64-gcc3-u-i386-x86_64", + "Darwin_x86-gcc3-u-i386-x86_64", + "Darwin_aarch64-gcc3", + "Darwin_x86-gcc3", + "Darwin_x86_64-gcc3", + ], + "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"], + "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"], + "win64-aarch64": ["WINNT_aarch64-msvc-aarch64"], +} + +# ftp -> shipped locales map +sl_platform_map = { + "linux-i686": "linux", + "linux-x86_64": "linux", + "mac": "osx", + "win32": "win32", + "win64": "win64", +} + +# ftp -> info file platform map +info_file_platform_map = { + "linux-i686": "linux", + "linux-x86_64": "linux64", + "mac": "macosx64", + "win32": "win32", + "win64": "win64", + "win64-aarch64": "win64_aarch64", +} + + +def ftp2updatePlatforms(platform): + return update_platform_map[platform] + + +def ftp2shippedLocales(platform): + return sl_platform_map.get(platform, platform) + + +def ftp2infoFile(platform): + return info_file_platform_map.get(platform, platform) diff --git a/python/mozrelease/mozrelease/scriptworker_canary.py b/python/mozrelease/mozrelease/scriptworker_canary.py new file mode 100644 index 0000000000..dabdc6868d --- /dev/null +++ b/python/mozrelease/mozrelease/scriptworker_canary.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import shutil +import subprocess +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import taskcluster +from appdirs import user_config_dir +from gecko_taskgraph import GECKO +from mach.base import FailedCommandError + +logger = logging.getLogger(__name__) + + +TASK_TYPES = { + "signing": ["linux-signing", "linux-signing-partial"], + "beetmover": ["beetmover-candidates"], + "bouncer": ["bouncer-submit"], + "balrog": ["balrog-submit"], + "tree": ["tree"], +} + + +def get_secret(secret): + # use proxy if configured, otherwise use local credentials from env vars + if "TASKCLUSTER_PROXY_URL" in os.environ: + secrets_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} + else: + secrets_options = taskcluster.optionsFromEnvironment() + secrets = taskcluster.Secrets(secrets_options) + return secrets.get(secret)["secret"] + + +@contextmanager +def configure_ssh(ssh_key_secret): + if ssh_key_secret is None: + yield + + # If we get here, we are running in automation. + # We use a user hgrc, so that we also get the system-wide hgrc settings. + hgrc = Path(user_config_dir("hg")) / "hgrc" + if hgrc.exists(): + raise FailedCommandError(f"Not overwriting `{hgrc}`; cannot configure ssh.") + + try: + ssh_key_dir = Path(tempfile.mkdtemp()) + + ssh_key = get_secret(ssh_key_secret) + ssh_key_file = ssh_key_dir / "id_rsa" + ssh_key_file.write_text(ssh_key["ssh_privkey"]) + ssh_key_file.chmod(0o600) + + hgrc_content = ( + "[ui]\n" + "username = trybld\n" + "ssh = ssh -i {path} -l {user}\n".format( + path=ssh_key_file, user=ssh_key["user"] + ) + ) + hgrc.write_text(hgrc_content) + + yield + finally: + shutil.rmtree(str(ssh_key_dir)) + hgrc.unlink() + + +def push_canary(scriptworkers, addresses, ssh_key_secret): + if ssh_key_secret and os.environ.get("MOZ_AUTOMATION", "0") != "1": + # We make assumptions about the layout of the docker image + # for creating the hgrc that we use for the key. + raise FailedCommandError("Cannot use ssh-key-secret outside of automation.") + + # Collect the set of `mach try scriptworker` task sets to run. + tasks = [] + for scriptworker in scriptworkers: + worker_tasks = TASK_TYPES.get(scriptworker) + if worker_tasks: + logger.info("Running tasks for {}: {}".format(scriptworker, worker_tasks)) + tasks.extend(worker_tasks) + else: + logger.info("No tasks for {}.".format(scriptworker)) + + mach = Path(GECKO) / "mach" + base_command = [str(mach), "try", "scriptworker"] + for address in addresses: + base_command.extend( + [ + "--route", + "notify.email.{}.on-failed".format(address), + "--route", + "notify.email.{}.on-exception".format(address), + ] + ) + + with configure_ssh(ssh_key_secret): + env = os.environ.copy() + for task in tasks: + subprocess.check_call(base_command + [task], env=env) diff --git a/python/mozrelease/mozrelease/update_verify.py b/python/mozrelease/mozrelease/update_verify.py new file mode 100644 index 0000000000..49fe21db15 --- /dev/null +++ b/python/mozrelease/mozrelease/update_verify.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re + +from six import string_types + +from .chunking import getChunk + + +class UpdateVerifyError(Exception): + pass + + +class UpdateVerifyConfig(object): + comment_regex = re.compile("^#") + key_write_order = ( + "release", + "product", + "platform", + "build_id", + "locales", + "channel", + "patch_types", + "from", + "aus_server", + "ftp_server_from", + "ftp_server_to", + "to", + "mar_channel_IDs", + "override_certs", + "to_build_id", + "to_display_version", + "to_app_version", + "updater_package", + ) + global_keys = ( + "product", + "channel", + "aus_server", + "to", + "to_build_id", + "to_display_version", + "to_app_version", + "override_certs", + ) + release_keys = ( + "release", + "build_id", + "locales", + "patch_types", + "from", + "ftp_server_from", + "ftp_server_to", + "mar_channel_IDs", + "platform", + "updater_package", + ) + first_only_keys = ( + "from", + "aus_server", + "to", + "to_build_id", + "to_display_version", + "to_app_version", + "override_certs", + ) + compare_attrs = global_keys + ("releases",) + + def __init__( + self, + product=None, + channel=None, + aus_server=None, + to=None, + to_build_id=None, + to_display_version=None, + to_app_version=None, + override_certs=None, + ): + self.product = product + self.channel = channel + self.aus_server = aus_server + self.to = to + self.to_build_id = to_build_id + self.to_display_version = to_display_version + self.to_app_version = to_app_version + self.override_certs = override_certs + self.releases = [] + + def __eq__(self, other): + self_list = [getattr(self, attr) for attr in self.compare_attrs] + other_list = [getattr(other, attr) for attr in self.compare_attrs] + return self_list == other_list + + def __ne__(self, other): + return not self.__eq__(other) + + def _parseLine(self, line): + entry = {} + items = re.findall(r"\w+=[\"'][^\"']*[\"']", line) + for i in items: + m = re.search(r"(?P<key>\w+)=[\"'](?P<value>.+)[\"']", i).groupdict() + if m["key"] not in self.global_keys and m["key"] not in self.release_keys: + raise UpdateVerifyError( + "Unknown key '%s' found on line:\n%s" % (m["key"], line) + ) + if m["key"] in entry: + raise UpdateVerifyError( + "Multiple values found for key '%s' on line:\n%s" % (m["key"], line) + ) + entry[m["key"]] = m["value"] + if not entry: + raise UpdateVerifyError("No parseable data in line '%s'" % line) + return entry + + def _addEntry(self, entry, first): + releaseKeys = {} + for k, v in entry.items(): + if k in self.global_keys: + setattr(self, k, entry[k]) + elif k in self.release_keys: + # "from" is reserved in Python + if k == "from": + releaseKeys["from_path"] = v + else: + releaseKeys[k] = v + self.addRelease(**releaseKeys) + + def read(self, config): + f = open(config) + # Only the first non-comment line of an update verify config should + # have a "from" and"ausServer". Ignore any subsequent lines with them. + first = True + for line in f.readlines(): + # Skip comment lines + if self.comment_regex.search(line): + continue + self._addEntry(self._parseLine(line), first) + first = False + + def write(self, fh): + first = True + for releaseInfo in self.releases: + for key in self.key_write_order: + if key in self.global_keys and ( + first or key not in self.first_only_keys + ): + value = getattr(self, key) + elif key in self.release_keys: + value = releaseInfo[key] + else: + value = None + if value is not None: + fh.write(key.encode("utf-8")) + fh.write(b"=") + if isinstance(value, (list, tuple)): + fh.write(('"%s" ' % " ".join(value)).encode("utf-8")) + else: + fh.write(('"%s" ' % value).encode("utf-8")) + # Rewind one character to avoid having a trailing space + fh.seek(-1, os.SEEK_CUR) + fh.write(b"\n") + first = False + + def addRelease( + self, + release=None, + build_id=None, + locales=[], + patch_types=["complete"], + from_path=None, + ftp_server_from=None, + ftp_server_to=None, + mar_channel_IDs=None, + platform=None, + updater_package=None, + ): + """Locales and patch_types can be passed as either a string or a list. + If a string is passed, they will be converted to a list for internal + storage""" + if self.getRelease(build_id, from_path): + raise UpdateVerifyError( + "Couldn't add release identified by build_id '%s' and from_path '%s': " + "already exists in config" % (build_id, from_path) + ) + if isinstance(locales, string_types): + locales = sorted(list(locales.split())) + if isinstance(patch_types, string_types): + patch_types = list(patch_types.split()) + self.releases.append( + { + "release": release, + "build_id": build_id, + "locales": locales, + "patch_types": patch_types, + "from": from_path, + "ftp_server_from": ftp_server_from, + "ftp_server_to": ftp_server_to, + "mar_channel_IDs": mar_channel_IDs, + "platform": platform, + "updater_package": updater_package, + } + ) + + def addLocaleToRelease(self, build_id, locale, from_path=None): + r = self.getRelease(build_id, from_path) + if not r: + raise UpdateVerifyError( + "Couldn't add '%s' to release identified by build_id '%s' and from_path '%s': " + "'%s' doesn't exist in this config." + % (locale, build_id, from_path, build_id) + ) + r["locales"].append(locale) + r["locales"] = sorted(r["locales"]) + + def getRelease(self, build_id, from_path): + for r in self.releases: + if r["build_id"] == build_id and r["from"] == from_path: + return r + return {} + + def getFullReleaseTests(self): + return [r for r in self.releases if r["from"] is not None] + + def getQuickReleaseTests(self): + return [r for r in self.releases if r["from"] is None] + + def getChunk(self, chunks, thisChunk): + fullTests = [] + quickTests = [] + for test in self.getFullReleaseTests(): + for locale in test["locales"]: + fullTests.append([test["build_id"], locale, test["from"]]) + for test in self.getQuickReleaseTests(): + for locale in test["locales"]: + quickTests.append([test["build_id"], locale, test["from"]]) + allTests = getChunk(fullTests, chunks, thisChunk) + allTests.extend(getChunk(quickTests, chunks, thisChunk)) + + newConfig = UpdateVerifyConfig( + self.product, + self.channel, + self.aus_server, + self.to, + self.to_build_id, + self.to_display_version, + self.to_app_version, + self.override_certs, + ) + for t in allTests: + build_id, locale, from_path = t + if from_path == "None": + from_path = None + r = self.getRelease(build_id, from_path) + try: + newConfig.addRelease( + r["release"], + build_id, + locales=[], + ftp_server_from=r["ftp_server_from"], + ftp_server_to=r["ftp_server_to"], + patch_types=r["patch_types"], + from_path=from_path, + mar_channel_IDs=r["mar_channel_IDs"], + platform=r["platform"], + updater_package=r["updater_package"], + ) + except UpdateVerifyError: + pass + newConfig.addLocaleToRelease(build_id, locale, from_path) + return newConfig diff --git a/python/mozrelease/mozrelease/util.py b/python/mozrelease/mozrelease/util.py new file mode 100644 index 0000000000..3858c40514 --- /dev/null +++ b/python/mozrelease/mozrelease/util.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from yaml.loader import SafeLoader + + +class UnicodeLoader(SafeLoader): + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + +UnicodeLoader.add_constructor("tag:yaml.org,2002:str", UnicodeLoader.construct_yaml_str) + + +def load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + loader = UnicodeLoader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() diff --git a/python/mozrelease/mozrelease/versions.py b/python/mozrelease/mozrelease/versions.py new file mode 100644 index 0000000000..e3e47d4e4a --- /dev/null +++ b/python/mozrelease/mozrelease/versions.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +from distutils.version import StrictVersion + +from looseversion import LooseVersion + + +class MozillaVersionCompareMixin: + def __cmp__(self, other): + # We expect this function to never be called. + raise AssertionError() + + def _cmp(self, other): + has_esr = set() + if isinstance(other, LooseModernMozillaVersion) and str(other).endswith("esr"): + # If other version ends with esr, coerce through MozillaVersion ending up with + # a StrictVersion if possible + has_esr.add("other") + other = MozillaVersion(str(other)[:-3]) # strip ESR from end of string + if isinstance(self, LooseModernMozillaVersion) and str(self).endswith("esr"): + # If our version ends with esr, coerce through MozillaVersion ending up with + # a StrictVersion if possible + has_esr.add("self") + self = MozillaVersion(str(self)[:-3]) # strip ESR from end of string + if isinstance(other, LooseModernMozillaVersion) or isinstance( + self, LooseModernMozillaVersion + ): + # If we're still LooseVersion for self or other, run LooseVersion compare + # Being sure to pass through Loose Version type first + val = LooseVersion._cmp( + LooseModernMozillaVersion(str(self)), + LooseModernMozillaVersion(str(other)), + ) + else: + # No versions are loose, therefore we can use StrictVersion + val = StrictVersion._cmp(self, other) + if has_esr.isdisjoint(set(["other", "self"])) or has_esr.issuperset( + set(["other", "self"]) + ): + # If both had esr string or neither, then _cmp() was accurate + return val + elif val != 0: + # cmp is accurate here even if esr is present in only 1 compare, since + # versions are not equal + return val + elif "other" in has_esr: + return -1 # esr is not greater than non esr + return 1 # non esr is greater than esr + + +class ModernMozillaVersion(MozillaVersionCompareMixin, StrictVersion): + """A version class that is slightly less restrictive than StrictVersion. + Instead of just allowing "a" or "b" as prerelease tags, it allows any + alpha. This allows us to support the once-shipped "3.6.3plugin1" and + similar versions.""" + + version_re = re.compile( + r"""^(\d+) \. (\d+) (\. (\d+))? + ([a-zA-Z]+(\d+))?$""", + re.VERBOSE, + ) + + +class AncientMozillaVersion(MozillaVersionCompareMixin, StrictVersion): + """A version class that is slightly less restrictive than StrictVersion. + Instead of just allowing "a" or "b" as prerelease tags, it allows any + alpha. This allows us to support the once-shipped "3.6.3plugin1" and + similar versions. + It also supports versions w.x.y.z by transmuting to w.x.z, which + is useful for versions like 1.5.0.x and 2.0.0.y""" + + version_re = re.compile( + r"""^(\d+) \. (\d+) \. \d (\. (\d+)) + ([a-zA-Z]+(\d+))?$""", + re.VERBOSE, + ) + + +class LooseModernMozillaVersion(MozillaVersionCompareMixin, LooseVersion): + """A version class that is more restrictive than LooseVersion. + This class reduces the valid strings to "esr", "a", "b" and "rc" in order + to support esr. StrictVersion requires a trailing number after all strings.""" + + component_re = re.compile(r"(\d+ | a | b | rc | esr | \.)", re.VERBOSE) + + def __repr__(self): + return "LooseModernMozillaVersion ('%s')" % str(self) + + +def MozillaVersion(version): + try: + return ModernMozillaVersion(version) + except ValueError: + pass + try: + if version.count(".") == 3: + return AncientMozillaVersion(version) + except ValueError: + pass + try: + return LooseModernMozillaVersion(version) + except ValueError: + pass + raise ValueError("Version number %s is invalid." % version) + + +def getPrettyVersion(version): + version = re.sub(r"a([0-9]+)$", r" Alpha \1", version) + version = re.sub(r"b([0-9]+)$", r" Beta \1", version) + version = re.sub(r"rc([0-9]+)$", r" RC \1", version) + return version |