diff options
Diffstat (limited to 'comm/python/l10n/tbxchannel')
| -rw-r--r-- | comm/python/l10n/tbxchannel/__init__.py | 47 | ||||
| -rw-r--r-- | comm/python/l10n/tbxchannel/l10n_merge.py | 26 | ||||
| -rw-r--r-- | comm/python/l10n/tbxchannel/quarantine_to_strings.py | 194 | ||||
| -rw-r--r-- | comm/python/l10n/tbxchannel/tb_migration_test.py | 172 |
4 files changed, 439 insertions, 0 deletions
diff --git a/comm/python/l10n/tbxchannel/__init__.py b/comm/python/l10n/tbxchannel/__init__.py new file mode 100644 index 0000000000..b6599032bc --- /dev/null +++ b/comm/python/l10n/tbxchannel/__init__.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path + +from .l10n_merge import COMM_STRINGS_QUARANTINE, COMM_STRINGS_QUARANTINE_PUSH + +TB_XC_NOTIFICATION_TMPL = """\ +**Thunderbird L10n Cross Channel** + +Changes pushed to `comm-strings-quarantine`: {rev_url} +""" + + +def get_thunderbird_xc_config(topsrcdir, strings_path): + assert isinstance(topsrcdir, Path) + assert isinstance(strings_path, Path) + return { + "strings": { + "path": strings_path, + "url": COMM_STRINGS_QUARANTINE, + "heads": {"default": "default"}, + "update_on_pull": True, + "push_url": COMM_STRINGS_QUARANTINE_PUSH, + }, + "source": { + "comm-central": { + "path": topsrcdir / "comm", + "url": "https://hg.mozilla.org/comm-central/", + "heads": { + # This list of repositories is ordered, starting with the + # one with the most recent content (central) to the oldest + # (ESR). In case two ESR versions are supported, the oldest + # ESR goes last (e.g. esr102 goes after esr115). + "comm": "comm-central", + "comm-beta": "releases/comm-beta", + "comm-esr102": "releases/comm-esr102", + }, + "config_files": [ + "comm/calendar/locales/l10n.toml", + "comm/mail/locales/l10n.toml", + "comm/suite/locales/l10n.toml", + ], + }, + }, + } diff --git a/comm/python/l10n/tbxchannel/l10n_merge.py b/comm/python/l10n/tbxchannel/l10n_merge.py new file mode 100644 index 0000000000..b202e7dfe0 --- /dev/null +++ b/comm/python/l10n/tbxchannel/l10n_merge.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +L10N_CENTRAL = "https://hg.mozilla.org/l10n-central" +COMM_L10N = "https://hg.mozilla.org/projects/comm-l10n" +COMM_L10N_PUSH = f"ssh{COMM_L10N[5:]}" +COMM_STRINGS_QUARANTINE = "https://hg.mozilla.org/projects/comm-strings-quarantine" +COMM_STRINGS_QUARANTINE_PUSH = f"ssh{COMM_STRINGS_QUARANTINE[5:]}" + + +GECKO_STRINGS_PATTERNS = ( + "{lang}/browser/pdfviewer/**", + "{lang}/devtools/**", + "{lang}/dom/**", + "{lang}/extensions/spellcheck/**", + "{lang}/netwerk/**", + "{lang}/security/**", + "{lang}/toolkit/**", +) + +COMM_STRINGS_PATTERNS = ( + "{lang}/calendar/**", + "{lang}/chat/**", + "{lang}/mail/**", +) diff --git a/comm/python/l10n/tbxchannel/quarantine_to_strings.py b/comm/python/l10n/tbxchannel/quarantine_to_strings.py new file mode 100644 index 0000000000..def3f59e5e --- /dev/null +++ b/comm/python/l10n/tbxchannel/quarantine_to_strings.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +from typing_extensions import Literal + +from mozversioncontrol import HgRepository +from mozversioncontrol.repoupdate import update_mercurial_repo + +from .l10n_merge import COMM_L10N, COMM_L10N_PUSH, COMM_STRINGS_QUARANTINE + +ACTIONS = Literal["clean", "prep", "migrate", "push"] + + +class HgL10nRepository(HgRepository): + log_trans_table = str.maketrans({"{": "{{", "}": "}}"}) + + def __init__(self, path: Path, check_url=None, logger=print): + super(HgL10nRepository, self).__init__(path, hg="hg") + self._logger = logger + if check_url is not None: + self._check_hg_url(check_url) + + def logger(self, *args): + # Escape python-style format string substitutions because Sentry is annoying + self._logger(*args[:-1], args[-1].translate(self.log_trans_table)) + + def _check_hg_url(self, repo_url): + configured_url = self._run("config", "paths.default").strip() + if configured_url != repo_url: + raise Exception(f"Repository does not match {repo_url}.") + + def check_status(self): + if not self.working_directory_clean() or self.get_outgoing_files(): + raise Exception(f"Repository at {self.path} is not clean, run with 'clean'.") + + def last_convert_rev(self): + args = ( + "log", + "-r", + "last(extra('convert_source', 'comm-strings-quarantine'))", + "--template", + "{get(extras,'convert_revision')}\n", + ) + self.logger(logging.INFO, "last_convert_rev", {}, " ".join(args)) + rv = self._run(*args).strip() + self.logger(logging.INFO, "last_convert_rev", {}, rv) + return rv + + def next_convert_rev(self, last_converted): + args = ("log", "-r", f"first(children({last_converted}))", "--template", "{node}\n") + self.logger(logging.INFO, "next_convert_rev", {}, " ".join(args)) + rv = self._run(*args).strip() + self.logger(logging.INFO, "next_convert_rev", {}, rv) + return rv + + def convert_quarantine(self, strings_path, filemap_path, splicemap_path, next_converted_rev): + args = ( + "convert", + "--config", + "convert.hg.saverev=True", + "--config", + "convert.hg.sourcename=comm-strings-quarantine", + "--config", + f"convert.hg.revs={next_converted_rev}:tip", + "--filemap", + filemap_path, + "--splicemap", + splicemap_path, + "--datesort", + str(self.path), + str(strings_path.absolute()), + ) + self.logger(logging.INFO, "convert_quarantine", {}, " ".join(args)) + rv = self._run(*args) + self.logger(logging.INFO, "convert_quarantine", {}, rv) + return rv + + def push(self, push_url): + popen_kwargs = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "cwd": self.path, + "env": self._env, + "universal_newlines": True, + "bufsize": 1, + } + cmd = ("hg", "push", "-r", ".", push_url) + self.logger(logging.INFO, "push", {}, " ".join(cmd)) + # This function doesn't really push to try... + self._push_to_try_with_log_capture(cmd, popen_kwargs) + + +def _nuke_hg_repos(*paths: Path): + failed = {} + for path in paths: + try: + if path.exists(): + shutil.rmtree(str(path)) + except Exception as e: + failed[str(path)] = e + + if failed: + for f in failed: + print(f"Unable to nuke '{f}': {failed[f]}") + raise Exception() + + +def publish_strings( + command_context, + quarantine_path: Path, + comm_l10n_path: Path, + actions: ACTIONS, + **kwargs, +): + if "clean" in actions: + command_context.log(logging.INFO, "clean", {}, "Removing old repository clones.") + _nuke_hg_repos(quarantine_path, comm_l10n_path) + + if "prep" in actions: + # update_mercurial_repo also will clone if a repo is not already there + command_context.log( + logging.INFO, "prep", {}, f"Updating comm-strings-quarantine at {quarantine_path}." + ) + update_mercurial_repo("hg", COMM_STRINGS_QUARANTINE, quarantine_path) + command_context.log(logging.INFO, "prep", {}, f"Updating comm-l10n at {comm_l10n_path}.") + update_mercurial_repo("hg", COMM_L10N, comm_l10n_path) + + local_quarantine = HgL10nRepository( + quarantine_path, COMM_STRINGS_QUARANTINE, command_context.log + ) + local_comm_l10n = HgL10nRepository(comm_l10n_path, COMM_L10N, command_context.log) + + if "prep" not in actions: + local_quarantine.update("tip") + local_comm_l10n.update("tip") + + if "migrate" in actions: + local_quarantine.check_status() + local_comm_l10n.check_status() + + command_context.log( + logging.INFO, "migrate", {}, "Starting string migration from quarantine." + ) + head_rev = local_comm_l10n.head_ref + last_convert_rev = local_comm_l10n.last_convert_rev() + first_convert_rev = local_quarantine.next_convert_rev(last_convert_rev) + command_context.log( + logging.INFO, "migrate", {}, f" Last converted rev: {last_convert_rev}" + ) + command_context.log( + logging.INFO, "migrate", {}, f" First converted rev: {first_convert_rev}" + ) + + with tempfile.NamedTemporaryFile( + prefix="splicemap", suffix=".txt", delete=False + ) as splice_fp: + splicemap = splice_fp.name + command_context.log( + logging.INFO, "migrate", {}, f" Writing splicemap to: {splicemap}" + ) + splice_fp.write(f"{first_convert_rev} {head_rev}\n".encode("utf-8")) + + with tempfile.NamedTemporaryFile(prefix="filemap", suffix=".txt", delete=False) as file_fp: + filemap = file_fp.name + command_context.log(logging.INFO, "migrate", {}, f" Writing filemap to: {filemap}") + file_fp.writelines( + ["exclude _configs\n".encode("utf-8"), "rename . en-US\n".encode("utf-8")] + ) + + command_context.log(logging.INFO, "migrate", {}, " Running hg convert...") + local_quarantine.convert_quarantine(comm_l10n_path, filemap, splicemap, first_convert_rev) + try: + os.unlink(splicemap) + os.unlink(filemap) + except Exception: + pass + + local_comm_l10n.update("tip") + command_context.log(logging.INFO, "migrate", {}, " Finished!") + + if "push" in actions: + if local_comm_l10n.get_outgoing_files(): + command_context.log(logging.INFO, "push", {}, " Pushing to comm-l10n.") + local_comm_l10n.push(COMM_L10N_PUSH) + else: + command_context.log(logging.INFO, "push", {}, "Skipping empty push.") diff --git a/comm/python/l10n/tbxchannel/tb_migration_test.py b/comm/python/l10n/tbxchannel/tb_migration_test.py new file mode 100644 index 0000000000..8e01e1d889 --- /dev/null +++ b/comm/python/l10n/tbxchannel/tb_migration_test.py @@ -0,0 +1,172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Test comm-l10n Fluent migrations +""" + +import logging +import os +import re +import shutil + +import hglib +from compare_locales.merge import merge_channels +from compare_locales.paths.configparser import TOMLParser +from compare_locales.paths.files import ProjectFiles +from fluent.migratetb import validator +from test_fluent_migrations.fmt import diff_resources + +import mozpack.path as mozpath +from mach.util import get_state_dir +from mozversioncontrol.repoupdate import update_mercurial_repo + +from .l10n_merge import COMM_L10N + + +def inspect_migration(path): + """Validate recipe and extract some metadata.""" + return validator.Validator.validate(path) + + +def prepare_object_dir(cmd): + """Prepare object dir to have an up-to-date clone of comm-l10n. + + We run this once per mach invocation, for all tested migrations. + """ + obj_dir = mozpath.join(cmd.topobjdir, "comm", "python", "l10n") + if not os.path.exists(obj_dir): + os.makedirs(obj_dir) + state_dir = get_state_dir() + update_mercurial_repo("hg", COMM_L10N, mozpath.join(state_dir, "comm-strings")) + return obj_dir + + +def test_migration(cmd, obj_dir, to_test, references): + """Test the given recipe. + + This creates a workdir by merging comm-strings-quarantine and the c-c source, + to mimic comm-strings-quarantine after the patch to test landed. + It then runs the recipe with a comm-strings-quarantine clone as localization, both + dry and wet. + It inspects the generated commits, and shows a diff between the merged + reference and the generated content. + The diff is intended to be visually inspected. Some changes might be + expected, in particular when formatting of the en-US strings is different. + """ + rv = 0 + migration_name = os.path.splitext(os.path.split(to_test)[1])[0] + l10n_lib = os.path.abspath(os.path.dirname(os.path.dirname(to_test))) + work_dir = mozpath.join(obj_dir, migration_name) + + paths = os.path.normpath(to_test).split(os.sep) + # Migration modules should be in a sub-folder of l10n. + migration_module = ".".join(paths[paths.index("l10n") + 1 : -1]) + "." + migration_name + + if os.path.exists(work_dir): + shutil.rmtree(work_dir) + os.makedirs(mozpath.join(work_dir, "reference")) + l10n_toml = mozpath.join(cmd.topsrcdir, cmd.substs["MOZ_BUILD_APP"], "locales", "l10n.toml") + pc = TOMLParser().parse(l10n_toml, env={"l10n_base": work_dir}) + pc.set_locales(["reference"]) + files = ProjectFiles("reference", [pc]) + for ref in references: + if ref != mozpath.normpath(ref): + cmd.log( + logging.ERROR, + "tb-fluent-migration-test", + { + "file": to_test, + "ref": ref, + }, + 'Reference path "{ref}" needs to be normalized for {file}', + ) + rv = 1 + continue + full_ref = mozpath.join(work_dir, "reference", ref) + m = files.match(full_ref) + if m is None: + raise ValueError(f"Bad reference path: {ref} - {full_ref}") + m_c_path = m[1] + g_s_path = mozpath.join(work_dir, "comm-strings", ref) + resources = [ + b"" if not os.path.exists(f) else open(f, "rb").read() for f in (g_s_path, m_c_path) + ] + ref_dir = os.path.dirname(full_ref) + if not os.path.exists(ref_dir): + os.makedirs(ref_dir) + open(full_ref, "wb").write(merge_channels(ref, resources)) + client = hglib.clone( + source=mozpath.join(get_state_dir(), "comm-strings"), + dest=mozpath.join(work_dir, "comm-strings"), + ) + client.open() + old_tip = client.tip().node + run_migration = [ + cmd._virtualenv_manager.python_path, + "-m", + "fluent.migratetb.tool", + "--locale", + "en-US", + "--reference-dir", + mozpath.join(work_dir, "reference"), + "--localization-dir", + mozpath.join(work_dir, "comm-strings"), + "--dry-run", + migration_module, + ] + append_env = {"PYTHONPATH": l10n_lib} + cmd.run_process( + run_migration, + append_env=append_env, + cwd=work_dir, + line_handler=print, + ) + # drop --dry-run + run_migration.pop(-2) + cmd.run_process( + run_migration, + append_env=append_env, + cwd=work_dir, + line_handler=print, + ) + tip = client.tip().node + if old_tip == tip: + cmd.log( + logging.WARN, + "tb-fluent-migration-test", + { + "file": to_test, + }, + "No migration applied for {file}", + ) + return rv + for ref in references: + diff_resources( + mozpath.join(work_dir, "reference", ref), + mozpath.join(work_dir, "comm-strings", "en-US", ref), + ) + messages = [l.desc.decode("utf-8") for l in client.log(b"::%s - ::%s" % (tip, old_tip))] + bug = re.search("[0-9]{5,}", migration_name) + # Just check first message for bug number, they're all following the same pattern + if bug is None or bug.group() not in messages[0]: + rv = 1 + cmd.log( + logging.ERROR, + "tb-fluent-migration-test", + { + "file": to_test, + }, + "Missing or wrong bug number for {file}", + ) + if any("part {}".format(n + 1) not in msg for n, msg in enumerate(messages)): + rv = 1 + cmd.log( + logging.ERROR, + "tb-fluent-migration-test", + { + "file": to_test, + }, + 'Commit messages should have "part {{index}}" for {file}', + ) + return rv |