Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 315 insertions, 0 deletions

diff --git a/python/l10n/mozxchannel/__init__.py b/python/l10n/mozxchannel/__init__.py
new file mode 100644
index 0000000000..66ee3966ca
--- /dev/null
+++ b/python/l10n/mozxchannel/__init__.py
@@ -0,0 +1,150 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import hglib
+from compare_locales import merge
+from mozpack import path as mozpath
+
+from . import projectconfig, source
+
+
+def get_default_config(topsrcdir, strings_path):
+    assert isinstance(topsrcdir, Path)
+    assert isinstance(strings_path, Path)
+    return {
+        "strings": {
+            "path": strings_path,
+            "url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/",
+            "heads": {"default": "default"},
+            "update_on_pull": True,
+            "push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/",
+        },
+        "source": {
+            "mozilla-unified": {
+                "path": topsrcdir,
+                "url": "https://hg.mozilla.org/mozilla-unified/",
+                "heads": {
+                    # This list of repositories is ordered, starting with the
+                    # one with the most recent content (central) to the oldest
+                    # (ESR). In case two ESR versions are supported, the oldest
+                    # ESR goes last (e.g. esr78 goes after esr91).
+                    "central": "mozilla-central",
+                    "beta": "releases/mozilla-beta",
+                    "release": "releases/mozilla-release",
+                    "esr102": "releases/mozilla-esr102",
+                },
+                "config_files": [
+                    "browser/locales/l10n.toml",
+                    "mobile/android/locales/l10n.toml",
+                ],
+            },
+        },
+    }
+
+
+@dataclass
+class TargetRevs:
+    target: bytes = None
+    revs: list = field(default_factory=list)
+
+
+@dataclass
+class CommitRev:
+    repo: str
+    rev: bytes
+
+    @property
+    def message(self):
+        return (
+            f"X-Channel-Repo: {self.repo}\n"
+            f'X-Channel-Revision: {self.rev.decode("ascii")}'
+        )
+
+
+class CrossChannelCreator:
+    def __init__(self, config):
+        self.config = config
+        self.strings_path = config["strings"]["path"]
+        self.message = (
+            f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}"
+        )
+
+    def create_content(self):
+        self.prune_target()
+        revs = []
+        for repo_name, repo_config in self.config["source"].items():
+            with hglib.open(repo_config["path"]) as repo:
+                revs.extend(self.create_for_repo(repo, repo_name, repo_config))
+        self.commit(revs)
+        return 0
+
+    def prune_target(self):
+        for leaf in self.config["strings"]["path"].iterdir():
+            if leaf.name == ".hg":
+                continue
+            shutil.rmtree(leaf)
+
+    def create_for_repo(self, repo, repo_name, repo_config):
+        print(f"Processing {repo_name} in {repo_config['path']}")
+        source_target_revs = defaultdict(TargetRevs)
+        revs_for_commit = []
+        parse_kwargs = {
+            "env": {"l10n_base": str(self.strings_path.parent)},
+            "ignore_missing_includes": True,
+        }
+        for head, head_name in repo_config["heads"].items():
+            print(f"Gathering files for {head}")
+            rev = repo.log(revrange=head)[0].node
+            revs_for_commit.append(CommitRev(head_name, rev))
+            p = source.HgTOMLParser(repo, rev)
+            project_configs = []
+            for config_file in repo_config["config_files"]:
+                project_configs.append(p.parse(config_file, **parse_kwargs))
+                project_configs[-1].set_locales(["en-US"], deep=True)
+            hgfiles = source.HGFiles(repo, rev, project_configs)
+            for targetpath, refpath, _, _ in hgfiles:
+                source_target_revs[refpath].revs.append(rev)
+                source_target_revs[refpath].target = targetpath
+        root = repo.root()
+        print(f"Writing {repo_name} content to target")
+        for refpath, targetrevs in source_target_revs.items():
+            local_ref = mozpath.relpath(refpath, root)
+            content = self.get_content(local_ref, repo, targetrevs.revs)
+            target_dir = mozpath.dirname(targetrevs.target)
+            if not os.path.isdir(target_dir):
+                os.makedirs(target_dir)
+            with open(targetrevs.target, "wb") as fh:
+                fh.write(content)
+        return revs_for_commit
+
+    def commit(self, revs):
+        message = self.message + "\n\n"
+        if "TASK_ID" in os.environ:
+            message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n"
+        message += "\n".join(rev.message for rev in revs)
+        with hglib.open(self.strings_path) as repo:
+            repo.commit(message=message, addremove=True)
+
+    def get_content(self, local_ref, repo, revs):
+        if local_ref.endswith(b".toml"):
+            return self.get_config_content(local_ref, repo, revs)
+        if len(revs) < 2:
+            return repo.cat([b"path:" + local_ref], rev=revs[0])
+        contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs]
+        try:
+            return merge.merge_channels(local_ref.decode("utf-8"), contents)
+        except merge.MergeNotSupportedError:
+            return contents[0]
+
+    def get_config_content(self, local_ref, repo, revs):
+        # We don't support merging toml files
+        content = repo.cat([b"path:" + local_ref], rev=revs[0])
+        return projectconfig.process_config(content)
diff --git a/python/l10n/mozxchannel/projectconfig.py b/python/l10n/mozxchannel/projectconfig.py
new file mode 100644
index 0000000000..23d8120a3c
--- /dev/null
+++ b/python/l10n/mozxchannel/projectconfig.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+from compare_locales import mozpath
+
+# The local path where we write the config files to
+TARGET_PATH = b"_configs"
+
+
+def process_config(toml_content):
+    """Process TOML configuration content to match the l10n setup for
+    the reference localization, return target_path and content.
+
+    The code adjusts basepath, [[paths]], and [[includes]]
+    """
+    # adjust basepath in content. '.' works in practice, also in theory?
+    new_base = mozpath.relpath(b".", TARGET_PATH)
+    if not new_base:
+        new_base = b"."  # relpath to '.' is '', sadly
+    base_line = b'\nbasepath = "%s"' % new_base
+    content1 = re.sub(br"^\s*basepath\s*=\s*.+", base_line, toml_content, flags=re.M)
+
+    # process [[paths]]
+    start = 0
+    content2 = b""
+    for m in re.finditer(
+        br"\[\[\s*paths\s*\]\].+?(?=\[|\Z)", content1, re.M | re.DOTALL
+    ):
+        content2 += content1[start : m.start()]
+        path_content = m.group()
+        l10n_line = re.search(br"^\s*l10n\s*=.*$", path_content, flags=re.M).group()
+        # remove variable expansions
+        new_reference = re.sub(br"{\s*\S+\s*}", b"", l10n_line)
+        # make the l10n a reference line
+        new_reference = re.sub(br"^(\s*)l10n(\s*=)", br"\1reference\2", new_reference)
+        content2 += re.sub(
+            br"^\s*reference\s*=.*$", new_reference, path_content, flags=re.M
+        )
+        start = m.end()
+    content2 += content1[start:]
+
+    start = 0
+    content3 = b""
+    for m in re.finditer(
+        br"\[\[\s*includes\s*\]\].+?(?=\[|\Z)", content2, re.M | re.DOTALL
+    ):
+        content3 += content2[start : m.start()]
+        include_content = m.group()
+        m_ = re.search(br'^\s*path = "(.+?)"', include_content, flags=re.M)
+        content3 += (
+            include_content[: m_.start(1)]
+            + generate_filename(m_.group(1))
+            + include_content[m_.end(1) :]
+        )
+        start = m.end()
+    content3 += content2[start:]
+
+    return content3
+
+
+def generate_filename(path):
+    segs = path.split(b"/")
+    # strip /locales/ from filename
+    segs = [seg for seg in segs if seg != b"locales"]
+    # strip variables from filename
+    segs = [seg for seg in segs if not seg.startswith(b"{") and not seg.endswith(b"}")]
+    if segs[-1] == b"l10n.toml":
+        segs.pop()
+        segs[-1] += b".toml"
+    outpath = b"-".join(segs)
+    if TARGET_PATH != b".":
+        # prepend the target path, if it's not '.'
+        outpath = mozpath.join(TARGET_PATH, outpath)
+    return outpath
diff --git a/python/l10n/mozxchannel/source.py b/python/l10n/mozxchannel/source.py
new file mode 100644
index 0000000000..b9d2067980
--- /dev/null
+++ b/python/l10n/mozxchannel/source.py
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import toml
+from compare_locales import mozpath, paths
+from compare_locales.paths.matcher import expand
+
+from .projectconfig import generate_filename
+
+
+class HGFiles(paths.ProjectFiles):
+    def __init__(self, repo, rev, projects):
+        self.repo = repo
+        self.ctx = repo[rev]
+        self.root = repo.root()
+        self.manifest = None
+        self.configs_map = {}
+        # get paths for our TOML files
+        for p in projects:
+            all_configpaths = {
+                mozpath.abspath(c.path).encode("utf-8") for c in p.configs
+            }
+            for refpath in all_configpaths:
+                local_path = mozpath.relpath(refpath, self.root)
+                if local_path not in self.ctx:
+                    print("ignoring", refpath)
+                    continue
+                targetpath = b"/".join(
+                    (
+                        expand(None, "{l10n_base}", p.environ).encode("utf-8"),
+                        b"en-US",
+                        generate_filename(local_path),
+                    )
+                )
+                self.configs_map[refpath] = targetpath
+        super(HGFiles, self).__init__("en-US", projects)
+        for m in self.matchers:
+            m["l10n"].encoding = "utf-8"
+            if "reference" in m:
+                m["reference"].encoding = "utf-8"
+        if self.exclude:
+            for m in self.exclude.matchers:
+                m["l10n"].encoding = "utf-8"
+                if "reference" in m:
+                    m["reference"].encoding = "utf-8"
+
+    def _files(self, matcher):
+        for f in self.ctx.manifest():
+            f = mozpath.join(self.root, f)
+            if matcher.match(f):
+                yield f
+
+    def __iter__(self):
+        for t in super(HGFiles, self).__iter__():
+            yield t
+        for refpath, targetpath in self.configs_map.items():
+            yield targetpath, refpath, None, set()
+
+    def match(self, path):
+        m = super(HGFiles, self).match(path)
+        if m:
+            return m
+        for refpath, targetpath in self.configs_map.items():
+            if path in [refpath, targetpath]:
+                return targetpath, refpath, None, set()
+
+
+class HgTOMLParser(paths.TOMLParser):
+    "subclass to load from our hg context"
+
+    def __init__(self, repo, rev):
+        self.repo = repo
+        self.rev = rev
+        self.root = repo.root().decode("utf-8")
+
+    def load(self, parse_ctx):
+        try:
+            path = parse_ctx.path
+            local_path = "path:" + mozpath.relpath(path, self.root)
+            data = self.repo.cat(files=[local_path.encode("utf-8")], rev=self.rev)
+        except Exception:
+            raise paths.ConfigNotFound(parse_ctx.path)
+
+        try:
+            parse_ctx.data = toml.loads(data.decode())
+        except toml.TomlDecodeError as e:
+            raise RuntimeError(f"In file '{parse_ctx.path}':\n  {e!s}") from e