summaryrefslogtreecommitdiffstats
path: root/tools/compare-locales
diff options
context:
space:
mode:
Diffstat (limited to 'tools/compare-locales')
-rw-r--r--tools/compare-locales/mach_commands.py409
-rw-r--r--tools/compare-locales/requirements.in2
2 files changed, 411 insertions, 0 deletions
diff --git a/tools/compare-locales/mach_commands.py b/tools/compare-locales/mach_commands.py
new file mode 100644
index 0000000000..56d101467b
--- /dev/null
+++ b/tools/compare-locales/mach_commands.py
@@ -0,0 +1,409 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import logging
+import os
+import tempfile
+from pathlib import Path
+
+from appdirs import user_config_dir
+from hglib.error import CommandError
+from mach.base import FailedCommandError
+from mach.decorators import Command, CommandArgument
+from mozrelease.scriptworker_canary import get_secret
+from redo import retry
+
+
+@Command(
+ "compare-locales",
+ category="build",
+ description="Run source checks on a localization.",
+)
+@CommandArgument(
+ "config_paths",
+ metavar="l10n.toml",
+ nargs="+",
+ help="TOML or INI file for the project",
+)
+@CommandArgument(
+ "l10n_base_dir",
+ metavar="l10n-base-dir",
+ help="Parent directory of localizations",
+)
+@CommandArgument(
+ "locales",
+ nargs="*",
+ metavar="locale-code",
+ help="Locale code and top-level directory of each localization",
+)
+@CommandArgument(
+ "-q",
+ "--quiet",
+ action="count",
+ default=0,
+ help="""Show less data.
+Specified once, don't show obsolete entities. Specified twice, also hide
+missing entities. Specify thrice to exclude warnings and four times to
+just show stats""",
+)
+@CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""")
+@CommandArgument(
+ "--validate", action="store_true", help="Run compare-locales against reference"
+)
+@CommandArgument(
+ "--json",
+ help="""Serialize to JSON. Value is the name of
+the output file, pass "-" to serialize to stdout and hide the default output.
+""",
+)
+@CommandArgument(
+ "-D",
+ action="append",
+ metavar="var=value",
+ default=[],
+ dest="defines",
+ help="Overwrite variables in TOML files",
+)
+@CommandArgument(
+ "--full", action="store_true", help="Compare projects that are disabled"
+)
+@CommandArgument(
+ "--return-zero", action="store_true", help="Return 0 regardless of l10n status"
+)
+def compare(command_context, **kwargs):
+ """Run compare-locales."""
+ from compare_locales.commands import CompareLocales
+
+ class ErrorHelper(object):
+ """Dummy ArgumentParser to marshall compare-locales
+ commandline errors to mach exceptions.
+ """
+
+ def error(self, msg):
+ raise FailedCommandError(msg)
+
+ def exit(self, message=None, status=0):
+ raise FailedCommandError(message, exit_code=status)
+
+ cmd = CompareLocales()
+ cmd.parser = ErrorHelper()
+ return cmd.handle(**kwargs)
+
+
+# https://stackoverflow.com/a/14117511
+def _positive_int(value):
+ value = int(value)
+ if value <= 0:
+ raise argparse.ArgumentTypeError(f"{value} must be a positive integer.")
+ return value
+
+
+class RetryError(Exception):
+ ...
+
+
+VCT_PATH = Path(".").resolve() / "vct"
+VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/"
+FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree"
+HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc")
+
+
+@Command(
+ "l10n-cross-channel",
+ category="misc",
+ description="Create cross-channel content.",
+)
+@CommandArgument(
+ "--strings-path",
+ "-s",
+ metavar="en-US",
+ type=Path,
+ default=Path("en-US"),
+ help="Path to mercurial repository for gecko-strings-quarantine",
+)
+@CommandArgument(
+ "--outgoing-path",
+ "-o",
+ type=Path,
+ help="create an outgoing() patch if there are changes",
+)
+@CommandArgument(
+ "--attempts",
+ type=_positive_int,
+ default=1,
+ help="Number of times to try (for automation)",
+)
+@CommandArgument(
+ "--ssh-secret",
+ action="store",
+ help="Taskcluster secret to use to push (for automation)",
+)
+@CommandArgument(
+ "actions",
+ choices=("prep", "create", "push", "clean"),
+ nargs="+",
+ # This help block will be poorly formatted until we fix bug 1714239
+ help="""
+ "prep": clone repos and pull heads.
+ "create": create the en-US strings commit an optionally create an
+ outgoing() patch.
+ "push": push the en-US strings to the quarantine repo.
+ "clean": clean up any sub-repos.
+ """,
+)
+def cross_channel(
+ command_context,
+ strings_path,
+ outgoing_path,
+ actions,
+ attempts,
+ ssh_secret,
+ **kwargs,
+):
+ """Run l10n cross-channel content generation."""
+ # This can be any path, as long as the name of the directory is en-US.
+ # Not entirely sure where this is a requirement; perhaps in l10n
+ # string manipulation logic?
+ if strings_path.name != "en-US":
+ raise FailedCommandError("strings_path needs to be named `en-US`")
+ command_context.activate_virtualenv()
+ # XXX pin python requirements
+ command_context.virtualenv_manager.install_pip_requirements(
+ Path(os.path.dirname(__file__)) / "requirements.in"
+ )
+ strings_path = strings_path.resolve() # abspath
+ if outgoing_path:
+ outgoing_path = outgoing_path.resolve() # abspath
+ get_config = kwargs.get("get_config", None)
+ try:
+ with tempfile.TemporaryDirectory() as ssh_key_dir:
+ retry(
+ _do_create_content,
+ attempts=attempts,
+ retry_exceptions=(RetryError,),
+ args=(
+ command_context,
+ strings_path,
+ outgoing_path,
+ ssh_secret,
+ Path(ssh_key_dir),
+ actions,
+ get_config,
+ ),
+ )
+ except RetryError as exc:
+ raise FailedCommandError(exc) from exc
+
+
+def _do_create_content(
+ command_context,
+ strings_path,
+ outgoing_path,
+ ssh_secret,
+ ssh_key_dir,
+ actions,
+ get_config,
+):
+ from mozxchannel import CrossChannelCreator, get_default_config
+
+ get_config = get_config or get_default_config
+
+ config = get_config(Path(command_context.topsrcdir), strings_path)
+ ccc = CrossChannelCreator(config)
+ status = 0
+ changes = False
+ ssh_key_secret = None
+ ssh_key_file = None
+
+ if "prep" in actions:
+ if ssh_secret:
+ if not os.environ.get("MOZ_AUTOMATION"):
+ raise CommandError(
+ "I don't know how to fetch the ssh secret outside of automation!"
+ )
+ ssh_key_secret = get_secret(ssh_secret)
+ ssh_key_file = ssh_key_dir.joinpath("id_rsa")
+ ssh_key_file.write_text(ssh_key_secret["ssh_privkey"])
+ ssh_key_file.chmod(0o600)
+ # Set up firefoxtree for comm per bug 1659691 comment 22
+ if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists():
+ _clone_hg_repo(command_context, VCT_URL, VCT_PATH)
+ hgrc_content = [
+ "[extensions]",
+ f"firefoxtree = {FXTREE_PATH}",
+ "",
+ "[ui]",
+ "username = trybld",
+ ]
+ if ssh_key_file:
+ hgrc_content.extend(
+ [
+ f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}",
+ ]
+ )
+ HGRC_PATH.write_text("\n".join(hgrc_content))
+ if strings_path.exists() and _check_outgoing(command_context, strings_path):
+ _strip_outgoing(command_context, strings_path)
+ # Clone strings + source repos, pull heads
+ for repo_config in (config["strings"], *config["source"].values()):
+ if not repo_config["path"].exists():
+ _clone_hg_repo(
+ command_context, repo_config["url"], str(repo_config["path"])
+ )
+ for head in repo_config["heads"].keys():
+ command = ["hg", "--cwd", str(repo_config["path"]), "pull"]
+ command.append(head)
+ status = _retry_run_process(
+ command_context, command, ensure_exit_code=False
+ )
+ if status not in (0, 255): # 255 on pull with no changes
+ raise RetryError(f"Failure on pull: status {status}!")
+ if repo_config.get("update_on_pull"):
+ command = [
+ "hg",
+ "--cwd",
+ str(repo_config["path"]),
+ "up",
+ "-C",
+ "-r",
+ head,
+ ]
+ status = _retry_run_process(
+ command_context, command, ensure_exit_code=False
+ )
+ if status not in (0, 255): # 255 on pull with no changes
+ raise RetryError(f"Failure on update: status {status}!")
+ _check_hg_repo(
+ command_context,
+ repo_config["path"],
+ heads=repo_config.get("heads", {}).keys(),
+ )
+ else:
+ _check_hg_repo(command_context, strings_path)
+ for repo_config in config.get("source", {}).values():
+ _check_hg_repo(
+ command_context,
+ repo_config["path"],
+ heads=repo_config.get("heads", {}).keys(),
+ )
+ if _check_outgoing(command_context, strings_path):
+ raise RetryError(f"check: Outgoing changes in {strings_path}!")
+
+ if "create" in actions:
+ try:
+ status = ccc.create_content()
+ changes = True
+ _create_outgoing_patch(command_context, outgoing_path, strings_path)
+ except CommandError as exc:
+ if exc.ret != 1:
+ raise RetryError(exc) from exc
+ command_context.log(logging.INFO, "create", {}, "No new strings.")
+
+ if "push" in actions:
+ if changes:
+ _retry_run_process(
+ command_context,
+ [
+ "hg",
+ "--cwd",
+ str(strings_path),
+ "push",
+ "-r",
+ ".",
+ config["strings"]["push_url"],
+ ],
+ line_handler=print,
+ )
+ else:
+ command_context.log(logging.INFO, "push", {}, "Skipping empty push.")
+
+ if "clean" in actions:
+ for repo_config in config.get("source", {}).values():
+ if repo_config.get("post-clobber", False):
+ _nuke_hg_repo(command_context, str(repo_config["path"]))
+
+ return status
+
+
+def _check_outgoing(command_context, strings_path):
+ status = _retry_run_process(
+ command_context,
+ ["hg", "--cwd", str(strings_path), "out", "-r", "."],
+ ensure_exit_code=False,
+ )
+ if status == 0:
+ return True
+ if status == 1:
+ return False
+ raise RetryError(f"Outgoing check in {strings_path} returned unexpected {status}!")
+
+
+def _strip_outgoing(command_context, strings_path):
+ _retry_run_process(
+ command_context,
+ [
+ "hg",
+ "--config",
+ "extensions.strip=",
+ "--cwd",
+ str(strings_path),
+ "strip",
+ "--no-backup",
+ "outgoing()",
+ ],
+ )
+
+
+def _create_outgoing_patch(command_context, path, strings_path):
+ if not path:
+ return
+ if not path.parent.exists():
+ os.makedirs(path.parent)
+ with open(path, "w") as fh:
+
+ def writeln(line):
+ fh.write(f"{line}\n")
+
+ _retry_run_process(
+ command_context,
+ [
+ "hg",
+ "--cwd",
+ str(strings_path),
+ "log",
+ "--patch",
+ "--verbose",
+ "-r",
+ "outgoing()",
+ ],
+ line_handler=writeln,
+ )
+
+
+def _retry_run_process(command_context, *args, error_msg=None, **kwargs):
+ try:
+ return command_context.run_process(*args, **kwargs)
+ except Exception as exc:
+ raise RetryError(error_msg or str(exc)) from exc
+
+
+def _check_hg_repo(command_context, path, heads=None):
+ if not (path.is_dir() and (path / ".hg").is_dir()):
+ raise RetryError(f"{path} is not a Mercurial repository")
+ if heads:
+ for head in heads:
+ _retry_run_process(
+ command_context,
+ ["hg", "--cwd", str(path), "log", "-r", head],
+ error_msg=f"check: {path} has no head {head}!",
+ )
+
+
+def _clone_hg_repo(command_context, url, path):
+ _retry_run_process(command_context, ["hg", "clone", url, str(path)])
+
+
+def _nuke_hg_repo(command_context, path):
+ _retry_run_process(command_context, ["rm", "-rf", str(path)])
diff --git a/tools/compare-locales/requirements.in b/tools/compare-locales/requirements.in
new file mode 100644
index 0000000000..d6fbb9d25b
--- /dev/null
+++ b/tools/compare-locales/requirements.in
@@ -0,0 +1,2 @@
+redo
+dataclasses; python_version < '3.7'