Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 380 insertions, 0 deletions

diff --git a/testing/mozharness/scripts/l10n_bumper.py b/testing/mozharness/scripts/l10n_bumper.py
new file mode 100755
index 0000000000..e597d5386d
--- /dev/null
+++ b/testing/mozharness/scripts/l10n_bumper.py
@@ -0,0 +1,380 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" l10n_bumper.py
+
+    Updates a gecko repo with up to date changesets from l10n.mozilla.org.
+
+    Specifically, it updates l10n-changesets.json which is used by mobile releases.
+
+    This is to allow for `mach taskgraph` to reference specific l10n revisions
+    without having to resort to task.extra or commandline base64 json hacks.
+"""
+import codecs
+import os
+import pprint
+import sys
+import time
+
+try:
+    import simplejson as json
+
+    assert json
+except ImportError:
+    import json
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import HgErrorList
+from mozharness.base.log import FATAL
+from mozharness.base.vcs.vcsbase import VCSScript
+
+
+class L10nBumper(VCSScript):
+    config_options = [
+        [
+            [
+                "--ignore-closed-tree",
+            ],
+            {
+                "action": "store_true",
+                "dest": "ignore_closed_tree",
+                "default": False,
+                "help": "Bump l10n changesets on a closed tree.",
+            },
+        ],
+        [
+            [
+                "--build",
+            ],
+            {
+                "action": "store_false",
+                "dest": "dontbuild",
+                "default": True,
+                "help": "Trigger new builds on push.",
+            },
+        ],
+    ]
+
+    def __init__(self, require_config_file=True):
+        super(L10nBumper, self).__init__(
+            all_actions=[
+                "clobber",
+                "check-treestatus",
+                "checkout-gecko",
+                "bump-changesets",
+                "push",
+                "push-loop",
+            ],
+            default_actions=[
+                "push-loop",
+            ],
+            require_config_file=require_config_file,
+            config_options=self.config_options,
+            # Default config options
+            config={
+                "treestatus_base_url": "https://treestatus.mozilla-releng.net",
+                "log_max_rotate": 99,
+            },
+        )
+
+    # Helper methods {{{1
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+
+        abs_dirs = super(L10nBumper, self).query_abs_dirs()
+
+        abs_dirs.update(
+            {
+                "gecko_local_dir": os.path.join(
+                    abs_dirs["abs_work_dir"],
+                    self.config.get(
+                        "gecko_local_dir",
+                        os.path.basename(self.config["gecko_pull_url"]),
+                    ),
+                ),
+            }
+        )
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def hg_commit(self, path, repo_path, message):
+        """
+        Commits changes in repo_path, with specified user and commit message
+        """
+        user = self.config["hg_user"]
+        hg = self.query_exe("hg", return_type="list")
+        env = self.query_env(partial_env={"LANG": "en_US.UTF-8"})
+        cmd = hg + ["add", path]
+        self.run_command(cmd, cwd=repo_path, env=env)
+        cmd = hg + ["commit", "-u", user, "-m", message]
+        self.run_command(cmd, cwd=repo_path, env=env)
+
+    def hg_push(self, repo_path):
+        hg = self.query_exe("hg", return_type="list")
+        command = hg + [
+            "push",
+            "-e",
+            "ssh -oIdentityFile=%s -l %s"
+            % (
+                self.config["ssh_key"],
+                self.config["ssh_user"],
+            ),
+            "-r",
+            ".",
+            self.config["gecko_push_url"],
+        ]
+        status = self.run_command(command, cwd=repo_path, error_list=HgErrorList)
+        if status != 0:
+            # We failed; get back to a known state so we can either retry
+            # or fail out and continue later.
+            self.run_command(
+                hg
+                + ["--config", "extensions.mq=", "strip", "--no-backup", "outgoing()"],
+                cwd=repo_path,
+            )
+            self.run_command(hg + ["up", "-C"], cwd=repo_path)
+            self.run_command(
+                hg + ["--config", "extensions.purge=", "purge", "--all"], cwd=repo_path
+            )
+            return False
+        return True
+
+    def _read_json(self, path):
+        contents = self.read_from_file(path)
+        try:
+            json_contents = json.loads(contents)
+            return json_contents
+        except ValueError:
+            self.error("%s is invalid json!" % path)
+
+    def _read_version(self, path):
+        contents = self.read_from_file(path).split("\n")[0]
+        return contents.split(".")
+
+    def _build_locale_map(self, old_contents, new_contents):
+        locale_map = {}
+        for key in old_contents:
+            if key not in new_contents:
+                locale_map[key] = "removed"
+        for k, v in new_contents.items():
+            if old_contents.get(k, {}).get("revision") != v["revision"]:
+                locale_map[k] = v["revision"]
+            elif old_contents.get(k, {}).get("platforms") != v["platforms"]:
+                locale_map[k] = v["platforms"]
+        return locale_map
+
+    def _build_platform_dict(self, bump_config):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs["gecko_local_dir"]
+        platform_dict = {}
+        ignore_config = bump_config.get("ignore_config", {})
+        for platform_config in bump_config["platform_configs"]:
+            path = os.path.join(repo_path, platform_config["path"])
+            self.info(
+                "Reading %s for %s locales..." % (path, platform_config["platforms"])
+            )
+            contents = self.read_from_file(path)
+            for locale in contents.splitlines():
+                # locale is 1st word in line in shipped-locales
+                if platform_config.get("format") == "shipped-locales":
+                    locale = locale.split(" ")[0]
+                existing_platforms = set(
+                    platform_dict.get(locale, {}).get("platforms", [])
+                )
+                platforms = set(platform_config["platforms"])
+                ignore_platforms = set(ignore_config.get(locale, []))
+                platforms = (platforms | existing_platforms) - ignore_platforms
+                platform_dict[locale] = {"platforms": sorted(list(platforms))}
+        self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
+        return platform_dict
+
+    def _build_revision_dict(self, bump_config, version_list):
+        self.info("Building revision dict...")
+        platform_dict = self._build_platform_dict(bump_config)
+        revision_dict = {}
+        if bump_config.get("revision_url"):
+            repl_dict = {
+                "MAJOR_VERSION": version_list[0],
+                "COMBINED_MAJOR_VERSION": str(
+                    int(version_list[0]) + int(version_list[1])
+                ),
+            }
+
+            url = bump_config["revision_url"] % repl_dict
+            path = self.download_file(url, error_level=FATAL)
+            revision_info = self.read_from_file(path)
+            self.info("Got %s" % revision_info)
+            for line in revision_info.splitlines():
+                locale, revision = line.split(" ")
+                if locale in platform_dict:
+                    revision_dict[locale] = platform_dict[locale]
+                    revision_dict[locale]["revision"] = revision
+        else:
+            for k, v in platform_dict.items():
+                v["revision"] = "default"
+                revision_dict[k] = v
+        self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
+        return revision_dict
+
+    def build_commit_message(self, name, locale_map):
+        comments = ""
+        approval_str = "r=release a=l10n-bump"
+        for locale, revision in sorted(locale_map.items()):
+            comments += "%s -> %s\n" % (locale, revision)
+        if self.config["dontbuild"]:
+            approval_str += " DONTBUILD"
+        if self.config["ignore_closed_tree"]:
+            approval_str += " CLOSED TREE"
+        message = "no bug - Bumping %s %s\n\n" % (name, approval_str)
+        message += comments
+        message = message.encode("utf-8")
+        return message
+
+    def query_treestatus(self):
+        "Return True if we can land based on treestatus"
+        c = self.config
+        dirs = self.query_abs_dirs()
+        tree = c.get(
+            "treestatus_tree", os.path.basename(c["gecko_pull_url"].rstrip("/"))
+        )
+        treestatus_url = "%s/trees/%s" % (c["treestatus_base_url"], tree)
+        treestatus_json = os.path.join(dirs["abs_work_dir"], "treestatus.json")
+        if not os.path.exists(dirs["abs_work_dir"]):
+            self.mkdir_p(dirs["abs_work_dir"])
+        self.rmtree(treestatus_json)
+
+        self.run_command(
+            ["curl", "--retry", "4", "-o", treestatus_json, treestatus_url],
+            throw_exception=True,
+        )
+
+        treestatus = self._read_json(treestatus_json)
+        if treestatus["result"]["status"] != "closed":
+            self.info(
+                "treestatus is %s - assuming we can land"
+                % repr(treestatus["result"]["status"])
+            )
+            return True
+
+        return False
+
+    # Actions {{{1
+    def check_treestatus(self):
+        if not self.config["ignore_closed_tree"] and not self.query_treestatus():
+            self.info("breaking early since treestatus is closed")
+            sys.exit(0)
+
+    def checkout_gecko(self):
+        c = self.config
+        dirs = self.query_abs_dirs()
+        dest = dirs["gecko_local_dir"]
+        repos = [
+            {
+                "repo": c["gecko_pull_url"],
+                "tag": c.get("gecko_tag", "default"),
+                "dest": dest,
+                "vcs": "hg",
+            }
+        ]
+        self.vcs_checkout_repos(repos)
+
+    def bump_changesets(self):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs["gecko_local_dir"]
+        version_path = os.path.join(repo_path, self.config["version_path"])
+        changes = False
+        version_list = self._read_version(version_path)
+        for bump_config in self.config["bump_configs"]:
+            path = os.path.join(repo_path, bump_config["path"])
+            # For now, assume format == 'json'.  When we add desktop support,
+            # we may need to add flatfile support
+            if os.path.exists(path):
+                old_contents = self._read_json(path)
+            else:
+                old_contents = {}
+
+            new_contents = self._build_revision_dict(bump_config, version_list)
+
+            if new_contents == old_contents:
+                continue
+            # super basic sanity check
+            if not isinstance(new_contents, dict) or len(new_contents) < 5:
+                self.error(
+                    "Cowardly refusing to land a broken-seeming changesets file!"
+                )
+                continue
+
+            # Write to disk
+            content_string = json.dumps(
+                new_contents,
+                sort_keys=True,
+                indent=4,
+                separators=(",", ": "),
+            )
+            fh = codecs.open(path, encoding="utf-8", mode="w+")
+            fh.write(content_string + "\n")
+            fh.close()
+
+            locale_map = self._build_locale_map(old_contents, new_contents)
+
+            # Commit
+            message = self.build_commit_message(bump_config["name"], locale_map)
+            self.hg_commit(path, repo_path, message)
+            changes = True
+        return changes
+
+    def push(self):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs["gecko_local_dir"]
+        return self.hg_push(repo_path)
+
+    def push_loop(self):
+        max_retries = 5
+        for _ in range(max_retries):
+            changed = False
+            if not self.config["ignore_closed_tree"] and not self.query_treestatus():
+                # Tree is closed; exit early to avoid a bunch of wasted time
+                self.info("breaking early since treestatus is closed")
+                break
+
+            self.checkout_gecko()
+            if self.bump_changesets():
+                changed = True
+
+            if not changed:
+                # Nothing changed, we're all done
+                self.info("No changes - all done")
+                break
+
+            if self.push():
+                # We did it! Hurray!
+                self.info("Great success!")
+                break
+            # If we're here, then the push failed. It also stripped any
+            # outgoing commits, so we should be in a pristine state again
+            # Empty our local cache of manifests so they get loaded again next
+            # time through this loop. This makes sure we get fresh upstream
+            # manifests, and avoids problems like bug 979080
+            self.device_manifests = {}
+
+            # Sleep before trying again
+            self.info("Sleeping 60 before trying again")
+            time.sleep(60)
+        else:
+            self.fatal("Didn't complete successfully (hit max_retries)")
+
+        # touch status file for nagios
+        dirs = self.query_abs_dirs()
+        status_path = os.path.join(dirs["base_work_dir"], self.config["status_path"])
+        self._touch_file(status_path)
+
+
+# __main__ {{{1
+if __name__ == "__main__":
+    bumper = L10nBumper()
+    bumper.run_and_exit()