Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 198 insertions, 0 deletions

diff --git a/python/l10n/test_fluent_migrations/__init__.py b/python/l10n/test_fluent_migrations/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/l10n/test_fluent_migrations/__init__.py
diff --git a/python/l10n/test_fluent_migrations/fmt.py b/python/l10n/test_fluent_migrations/fmt.py
new file mode 100644
index 0000000000..198158fa5a
--- /dev/null
+++ b/python/l10n/test_fluent_migrations/fmt.py
@@ -0,0 +1,198 @@
+import codecs
+import logging
+import os
+import re
+import shutil
+import sys
+from datetime import datetime, timedelta
+from difflib import unified_diff
+from typing import Iterable
+
+import hglib
+from compare_locales.merge import merge_channels
+from compare_locales.paths.configparser import TOMLParser
+from compare_locales.paths.files import ProjectFiles
+from fluent.migrate.repo_client import RepoClient, git
+from fluent.migrate.validator import Validator
+from fluent.syntax import FluentParser, FluentSerializer
+from mach.util import get_state_dir
+from mozpack.path import join, normpath
+from mozversioncontrol.repoupdate import update_git_repo, update_mercurial_repo
+
+L10N_SOURCE_NAME = "l10n-source"
+L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
+
+STRINGS_NAME = "gecko-strings"
+STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
+
+PULL_AFTER = timedelta(days=2)
+
+
+def inspect_migration(path):
+    """Validate recipe and extract some metadata."""
+    return Validator.validate(path)
+
+
+def prepare_directories(cmd, use_git=False):
+    """
+    Ensure object dir exists,
+    and that repo dir has a relatively up-to-date clone of l10n-source or gecko-strings.
+
+    We run this once per mach invocation, for all tested migrations.
+    """
+    obj_dir = join(cmd.topobjdir, "python", "l10n")
+    if not os.path.exists(obj_dir):
+        os.makedirs(obj_dir)
+
+    if use_git:
+        repo_dir = join(get_state_dir(), L10N_SOURCE_NAME)
+        marker = join(repo_dir, ".git", "l10n_pull_marker")
+    else:
+        repo_dir = join(get_state_dir(), STRINGS_NAME)
+        marker = join(repo_dir, ".hg", "l10n_pull_marker")
+
+    try:
+        last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+        skip_clone = datetime.now() < last_pull + PULL_AFTER
+    except OSError:
+        skip_clone = False
+    if not skip_clone:
+        if use_git:
+            update_git_repo(L10N_SOURCE_REPO, repo_dir)
+        else:
+            update_mercurial_repo(STRINGS_REPO, repo_dir)
+        with open(marker, "w") as fh:
+            fh.flush()
+
+    return obj_dir, repo_dir
+
+
+def diff_resources(left_path, right_path):
+    parser = FluentParser(with_spans=False)
+    serializer = FluentSerializer(with_junk=True)
+    lines = []
+    for p in (left_path, right_path):
+        with codecs.open(p, encoding="utf-8") as fh:
+            res = parser.parse(fh.read())
+            lines.append(serializer.serialize(res).splitlines(True))
+    sys.stdout.writelines(
+        chunk for chunk in unified_diff(lines[0], lines[1], left_path, right_path)
+    )
+
+
+def test_migration(
+    cmd,
+    obj_dir: str,
+    repo_dir: str,
+    use_git: bool,
+    to_test: list[str],
+    references: Iterable[str],
+):
+    """Test the given recipe.
+
+    This creates a workdir by l10n-merging gecko-strings and the m-c source,
+    to mimic gecko-strings after the patch to test landed.
+    It then runs the recipe with a gecko-strings clone as localization, both
+    dry and wet.
+    It inspects the generated commits, and shows a diff between the merged
+    reference and the generated content.
+    The diff is intended to be visually inspected. Some changes might be
+    expected, in particular when formatting of the en-US strings is different.
+    """
+    rv = 0
+    migration_name = os.path.splitext(os.path.split(to_test)[1])[0]
+    work_dir = join(obj_dir, migration_name)
+
+    paths = os.path.normpath(to_test).split(os.sep)
+    # Migration modules should be in a sub-folder of l10n.
+    migration_module = (
+        ".".join(paths[paths.index("l10n") + 1 : -1]) + "." + migration_name
+    )
+
+    if os.path.exists(work_dir):
+        shutil.rmtree(work_dir)
+    os.makedirs(join(work_dir, "reference"))
+    l10n_toml = join(cmd.topsrcdir, cmd.substs["MOZ_BUILD_APP"], "locales", "l10n.toml")
+    pc = TOMLParser().parse(l10n_toml, env={"l10n_base": work_dir})
+    pc.set_locales(["reference"])
+    files = ProjectFiles("reference", [pc])
+    ref_root = join(work_dir, "reference")
+    for ref in references:
+        if ref != normpath(ref):
+            cmd.log(
+                logging.ERROR,
+                "fluent-migration-test",
+                {"file": to_test, "ref": ref},
+                'Reference path "{ref}" needs to be normalized for {file}',
+            )
+            rv = 1
+            continue
+        full_ref = join(ref_root, ref)
+        m = files.match(full_ref)
+        if m is None:
+            raise ValueError("Bad reference path: " + ref)
+        m_c_path = m[1]
+        g_s_path = join(work_dir, L10N_SOURCE_NAME if use_git else STRINGS_NAME, ref)
+        resources = [
+            b"" if not os.path.exists(f) else open(f, "rb").read()
+            for f in (g_s_path, m_c_path)
+        ]
+        ref_dir = os.path.dirname(full_ref)
+        if not os.path.exists(ref_dir):
+            os.makedirs(ref_dir)
+        open(full_ref, "wb").write(merge_channels(ref, resources))
+    l10n_root = join(work_dir, "en-US")
+    if use_git:
+        git(work_dir, "clone", repo_dir, l10n_root)
+    else:
+        hglib.clone(source=repo_dir, dest=l10n_root)
+    client = RepoClient(l10n_root)
+    old_tip = client.head()
+    run_migration = [
+        cmd._virtualenv_manager.python_path,
+        "-m",
+        "fluent.migrate.tool",
+        "--lang",
+        "en-US",
+        "--reference-dir",
+        ref_root,
+        "--localization-dir",
+        l10n_root,
+        "--dry-run",
+        migration_module,
+    ]
+    cmd.run_process(run_migration, cwd=work_dir, line_handler=print)
+    # drop --dry-run
+    run_migration.pop(-2)
+    cmd.run_process(run_migration, cwd=work_dir, line_handler=print)
+    tip = client.head()
+    if old_tip == tip:
+        cmd.log(
+            logging.WARN,
+            "fluent-migration-test",
+            {"file": to_test},
+            "No migration applied for {file}",
+        )
+        return rv
+    for ref in references:
+        diff_resources(join(ref_root, ref), join(l10n_root, ref))
+    messages = client.log(old_tip, tip)
+    bug = re.search("[0-9]{5,}", migration_name)
+    # Just check first message for bug number, they're all following the same pattern
+    if bug is None or bug.group() not in messages[0]:
+        rv = 1
+        cmd.log(
+            logging.ERROR,
+            "fluent-migration-test",
+            {"file": to_test},
+            "Missing or wrong bug number for {file}",
+        )
+    if any("part {}".format(n + 1) not in msg for n, msg in enumerate(messages)):
+        rv = 1
+        cmd.log(
+            logging.ERROR,
+            "fluent-migration-test",
+            {"file": to_test},
+            'Commit messages should have "part {{index}}" for {file}',
+        )
+    return rv