#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """ l10n_bumper.py Updates a gecko repo with up to date changesets from l10n.mozilla.org. Specifically, it updates l10n-changesets.json which is used by mobile releases. This is to allow for `mach taskgraph` to reference specific l10n revisions without having to resort to task.extra or commandline base64 json hacks. """ import codecs import os import pprint import sys import time try: import simplejson as json assert json except ImportError: import json sys.path.insert(1, os.path.dirname(sys.path[0])) from mozharness.base.errors import HgErrorList from mozharness.base.log import FATAL from mozharness.base.vcs.vcsbase import VCSScript class L10nBumper(VCSScript): config_options = [ [ [ "--ignore-closed-tree", ], { "action": "store_true", "dest": "ignore_closed_tree", "default": False, "help": "Bump l10n changesets on a closed tree.", }, ], [ [ "--build", ], { "action": "store_false", "dest": "dontbuild", "default": True, "help": "Trigger new builds on push.", }, ], ] def __init__(self, require_config_file=True): super(L10nBumper, self).__init__( all_actions=[ "clobber", "check-treestatus", "checkout-gecko", "bump-changesets", "push", "push-loop", ], default_actions=[ "push-loop", ], require_config_file=require_config_file, config_options=self.config_options, # Default config options config={ "treestatus_base_url": "https://treestatus.mozilla-releng.net", "log_max_rotate": 99, }, ) # Helper methods {{{1 def query_abs_dirs(self): if self.abs_dirs: return self.abs_dirs abs_dirs = super(L10nBumper, self).query_abs_dirs() abs_dirs.update( { "gecko_local_dir": os.path.join( abs_dirs["abs_work_dir"], self.config.get( "gecko_local_dir", os.path.basename(self.config["gecko_pull_url"]), ), ), } ) self.abs_dirs = abs_dirs return self.abs_dirs def hg_commit(self, path, repo_path, message): """ Commits changes in repo_path, with specified user and commit message """ user = self.config["hg_user"] hg = self.query_exe("hg", return_type="list") env = self.query_env(partial_env={"LANG": "en_US.UTF-8"}) cmd = hg + ["add", path] self.run_command(cmd, cwd=repo_path, env=env) cmd = hg + ["commit", "-u", user, "-m", message] self.run_command(cmd, cwd=repo_path, env=env) def hg_push(self, repo_path): hg = self.query_exe("hg", return_type="list") command = hg + [ "push", "-e", "ssh -oIdentityFile=%s -l %s" % ( self.config["ssh_key"], self.config["ssh_user"], ), "-r", ".", self.config["gecko_push_url"], ] status = self.run_command(command, cwd=repo_path, error_list=HgErrorList) if status != 0: # We failed; get back to a known state so we can either retry # or fail out and continue later. self.run_command( hg + ["--config", "extensions.mq=", "strip", "--no-backup", "outgoing()"], cwd=repo_path, ) self.run_command(hg + ["up", "-C"], cwd=repo_path) self.run_command( hg + ["--config", "extensions.purge=", "purge", "--all"], cwd=repo_path ) return False return True def _read_json(self, path): contents = self.read_from_file(path) try: json_contents = json.loads(contents) return json_contents except ValueError: self.error("%s is invalid json!" % path) def _read_version(self, path): contents = self.read_from_file(path).split("\n")[0] return contents.split(".") def _build_locale_map(self, old_contents, new_contents): locale_map = {} for key in old_contents: if key not in new_contents: locale_map[key] = "removed" for k, v in new_contents.items(): if old_contents.get(k, {}).get("revision") != v["revision"]: locale_map[k] = v["revision"] elif old_contents.get(k, {}).get("platforms") != v["platforms"]: locale_map[k] = v["platforms"] return locale_map def _build_platform_dict(self, bump_config): dirs = self.query_abs_dirs() repo_path = dirs["gecko_local_dir"] platform_dict = {} ignore_config = bump_config.get("ignore_config", {}) for platform_config in bump_config["platform_configs"]: path = os.path.join(repo_path, platform_config["path"]) self.info( "Reading %s for %s locales..." % (path, platform_config["platforms"]) ) contents = self.read_from_file(path) for locale in contents.splitlines(): # locale is 1st word in line in shipped-locales if platform_config.get("format") == "shipped-locales": locale = locale.split(" ")[0] existing_platforms = set( platform_dict.get(locale, {}).get("platforms", []) ) platforms = set(platform_config["platforms"]) ignore_platforms = set(ignore_config.get(locale, [])) platforms = (platforms | existing_platforms) - ignore_platforms platform_dict[locale] = {"platforms": sorted(list(platforms))} self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict)) return platform_dict def _build_revision_dict(self, bump_config, version_list): self.info("Building revision dict...") platform_dict = self._build_platform_dict(bump_config) revision_dict = {} if bump_config.get("revision_url"): repl_dict = { "MAJOR_VERSION": version_list[0], "COMBINED_MAJOR_VERSION": str( int(version_list[0]) + int(version_list[1]) ), } url = bump_config["revision_url"] % repl_dict path = self.download_file(url, error_level=FATAL) revision_info = self.read_from_file(path) self.info("Got %s" % revision_info) for line in revision_info.splitlines(): locale, revision = line.split(" ") if locale in platform_dict: revision_dict[locale] = platform_dict[locale] revision_dict[locale]["revision"] = revision else: for k, v in platform_dict.items(): v["revision"] = "default" revision_dict[k] = v self.info("revision_dict:\n%s" % pprint.pformat(revision_dict)) return revision_dict def build_commit_message(self, name, locale_map): comments = "" approval_str = "r=release a=l10n-bump" for locale, revision in sorted(locale_map.items()): comments += "%s -> %s\n" % (locale, revision) if self.config["dontbuild"]: approval_str += " DONTBUILD" if self.config["ignore_closed_tree"]: approval_str += " CLOSED TREE" message = "no bug - Bumping %s %s\n\n" % (name, approval_str) message += comments message = message.encode("utf-8") return message def query_treestatus(self): "Return True if we can land based on treestatus" c = self.config dirs = self.query_abs_dirs() tree = c.get( "treestatus_tree", os.path.basename(c["gecko_pull_url"].rstrip("/")) ) treestatus_url = "%s/trees/%s" % (c["treestatus_base_url"], tree) treestatus_json = os.path.join(dirs["abs_work_dir"], "treestatus.json") if not os.path.exists(dirs["abs_work_dir"]): self.mkdir_p(dirs["abs_work_dir"]) self.rmtree(treestatus_json) self.run_command( ["curl", "--retry", "4", "-o", treestatus_json, treestatus_url], throw_exception=True, ) treestatus = self._read_json(treestatus_json) if treestatus["result"]["status"] != "closed": self.info( "treestatus is %s - assuming we can land" % repr(treestatus["result"]["status"]) ) return True return False # Actions {{{1 def check_treestatus(self): if not self.config["ignore_closed_tree"] and not self.query_treestatus(): self.info("breaking early since treestatus is closed") sys.exit(0) def checkout_gecko(self): c = self.config dirs = self.query_abs_dirs() dest = dirs["gecko_local_dir"] repos = [ { "repo": c["gecko_pull_url"], "tag": c.get("gecko_tag", "default"), "dest": dest, "vcs": "hg", } ] self.vcs_checkout_repos(repos) def bump_changesets(self): dirs = self.query_abs_dirs() repo_path = dirs["gecko_local_dir"] version_path = os.path.join(repo_path, self.config["version_path"]) changes = False version_list = self._read_version(version_path) for bump_config in self.config["bump_configs"]: path = os.path.join(repo_path, bump_config["path"]) # For now, assume format == 'json'. When we add desktop support, # we may need to add flatfile support if os.path.exists(path): old_contents = self._read_json(path) else: old_contents = {} new_contents = self._build_revision_dict(bump_config, version_list) if new_contents == old_contents: continue # super basic sanity check if not isinstance(new_contents, dict) or len(new_contents) < 5: self.error( "Cowardly refusing to land a broken-seeming changesets file!" ) continue # Write to disk content_string = json.dumps( new_contents, sort_keys=True, indent=4, separators=(",", ": "), ) fh = codecs.open(path, encoding="utf-8", mode="w+") fh.write(content_string + "\n") fh.close() locale_map = self._build_locale_map(old_contents, new_contents) # Commit message = self.build_commit_message(bump_config["name"], locale_map) self.hg_commit(path, repo_path, message) changes = True return changes def push(self): dirs = self.query_abs_dirs() repo_path = dirs["gecko_local_dir"] return self.hg_push(repo_path) def push_loop(self): max_retries = 5 for _ in range(max_retries): changed = False if not self.config["ignore_closed_tree"] and not self.query_treestatus(): # Tree is closed; exit early to avoid a bunch of wasted time self.info("breaking early since treestatus is closed") break self.checkout_gecko() if self.bump_changesets(): changed = True if not changed: # Nothing changed, we're all done self.info("No changes - all done") break if self.push(): # We did it! Hurray! self.info("Great success!") break # If we're here, then the push failed. It also stripped any # outgoing commits, so we should be in a pristine state again # Empty our local cache of manifests so they get loaded again next # time through this loop. This makes sure we get fresh upstream # manifests, and avoids problems like bug 979080 self.device_manifests = {} # Sleep before trying again self.info("Sleeping 60 before trying again") time.sleep(60) else: self.fatal("Didn't complete successfully (hit max_retries)") # touch status file for nagios dirs = self.query_abs_dirs() status_path = os.path.join(dirs["base_work_dir"], self.config["status_path"]) self._touch_file(status_path) # __main__ {{{1 if __name__ == "__main__": bumper = L10nBumper() bumper.run_and_exit()