diff options
Diffstat (limited to 'python/l10n/mozxchannel')
| -rw-r--r-- | python/l10n/mozxchannel/__init__.py | 150 | ||||
| -rw-r--r-- | python/l10n/mozxchannel/projectconfig.py | 77 | ||||
| -rw-r--r-- | python/l10n/mozxchannel/source.py | 84 |
3 files changed, 311 insertions, 0 deletions
diff --git a/python/l10n/mozxchannel/__init__.py b/python/l10n/mozxchannel/__init__.py new file mode 100644 index 0000000000..66ee3966ca --- /dev/null +++ b/python/l10n/mozxchannel/__init__.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +import hglib +from compare_locales import merge +from mozpack import path as mozpath + +from . import projectconfig, source + + +def get_default_config(topsrcdir, strings_path): + assert isinstance(topsrcdir, Path) + assert isinstance(strings_path, Path) + return { + "strings": { + "path": strings_path, + "url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/", + "heads": {"default": "default"}, + "update_on_pull": True, + "push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/", + }, + "source": { + "mozilla-unified": { + "path": topsrcdir, + "url": "https://hg.mozilla.org/mozilla-unified/", + "heads": { + # This list of repositories is ordered, starting with the + # one with the most recent content (central) to the oldest + # (ESR). In case two ESR versions are supported, the oldest + # ESR goes last (e.g. esr78 goes after esr91). + "central": "mozilla-central", + "beta": "releases/mozilla-beta", + "release": "releases/mozilla-release", + "esr102": "releases/mozilla-esr102", + }, + "config_files": [ + "browser/locales/l10n.toml", + "mobile/android/locales/l10n.toml", + ], + }, + }, + } + + +@dataclass +class TargetRevs: + target: bytes = None + revs: list = field(default_factory=list) + + +@dataclass +class CommitRev: + repo: str + rev: bytes + + @property + def message(self): + return ( + f"X-Channel-Repo: {self.repo}\n" + f'X-Channel-Revision: {self.rev.decode("ascii")}' + ) + + +class CrossChannelCreator: + def __init__(self, config): + self.config = config + self.strings_path = config["strings"]["path"] + self.message = ( + f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}" + ) + + def create_content(self): + self.prune_target() + revs = [] + for repo_name, repo_config in self.config["source"].items(): + with hglib.open(repo_config["path"]) as repo: + revs.extend(self.create_for_repo(repo, repo_name, repo_config)) + self.commit(revs) + return 0 + + def prune_target(self): + for leaf in self.config["strings"]["path"].iterdir(): + if leaf.name == ".hg": + continue + shutil.rmtree(leaf) + + def create_for_repo(self, repo, repo_name, repo_config): + print(f"Processing {repo_name} in {repo_config['path']}") + source_target_revs = defaultdict(TargetRevs) + revs_for_commit = [] + parse_kwargs = { + "env": {"l10n_base": str(self.strings_path.parent)}, + "ignore_missing_includes": True, + } + for head, head_name in repo_config["heads"].items(): + print(f"Gathering files for {head}") + rev = repo.log(revrange=head)[0].node + revs_for_commit.append(CommitRev(head_name, rev)) + p = source.HgTOMLParser(repo, rev) + project_configs = [] + for config_file in repo_config["config_files"]: + project_configs.append(p.parse(config_file, **parse_kwargs)) + project_configs[-1].set_locales(["en-US"], deep=True) + hgfiles = source.HGFiles(repo, rev, project_configs) + for targetpath, refpath, _, _ in hgfiles: + source_target_revs[refpath].revs.append(rev) + source_target_revs[refpath].target = targetpath + root = repo.root() + print(f"Writing {repo_name} content to target") + for refpath, targetrevs in source_target_revs.items(): + local_ref = mozpath.relpath(refpath, root) + content = self.get_content(local_ref, repo, targetrevs.revs) + target_dir = mozpath.dirname(targetrevs.target) + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + with open(targetrevs.target, "wb") as fh: + fh.write(content) + return revs_for_commit + + def commit(self, revs): + message = self.message + "\n\n" + if "TASK_ID" in os.environ: + message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n" + message += "\n".join(rev.message for rev in revs) + with hglib.open(self.strings_path) as repo: + repo.commit(message=message, addremove=True) + + def get_content(self, local_ref, repo, revs): + if local_ref.endswith(b".toml"): + return self.get_config_content(local_ref, repo, revs) + if len(revs) < 2: + return repo.cat([b"path:" + local_ref], rev=revs[0]) + contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs] + try: + return merge.merge_channels(local_ref.decode("utf-8"), contents) + except merge.MergeNotSupportedError: + return contents[0] + + def get_config_content(self, local_ref, repo, revs): + # We don't support merging toml files + content = repo.cat([b"path:" + local_ref], rev=revs[0]) + return projectconfig.process_config(content) diff --git a/python/l10n/mozxchannel/projectconfig.py b/python/l10n/mozxchannel/projectconfig.py new file mode 100644 index 0000000000..23d8120a3c --- /dev/null +++ b/python/l10n/mozxchannel/projectconfig.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + +from compare_locales import mozpath + +# The local path where we write the config files to +TARGET_PATH = b"_configs" + + +def process_config(toml_content): + """Process TOML configuration content to match the l10n setup for + the reference localization, return target_path and content. + + The code adjusts basepath, [[paths]], and [[includes]] + """ + # adjust basepath in content. '.' works in practice, also in theory? + new_base = mozpath.relpath(b".", TARGET_PATH) + if not new_base: + new_base = b"." # relpath to '.' is '', sadly + base_line = b'\nbasepath = "%s"' % new_base + content1 = re.sub(br"^\s*basepath\s*=\s*.+", base_line, toml_content, flags=re.M) + + # process [[paths]] + start = 0 + content2 = b"" + for m in re.finditer( + br"\[\[\s*paths\s*\]\].+?(?=\[|\Z)", content1, re.M | re.DOTALL + ): + content2 += content1[start : m.start()] + path_content = m.group() + l10n_line = re.search(br"^\s*l10n\s*=.*$", path_content, flags=re.M).group() + # remove variable expansions + new_reference = re.sub(br"{\s*\S+\s*}", b"", l10n_line) + # make the l10n a reference line + new_reference = re.sub(br"^(\s*)l10n(\s*=)", br"\1reference\2", new_reference) + content2 += re.sub( + br"^\s*reference\s*=.*$", new_reference, path_content, flags=re.M + ) + start = m.end() + content2 += content1[start:] + + start = 0 + content3 = b"" + for m in re.finditer( + br"\[\[\s*includes\s*\]\].+?(?=\[|\Z)", content2, re.M | re.DOTALL + ): + content3 += content2[start : m.start()] + include_content = m.group() + m_ = re.search(br'^\s*path = "(.+?)"', include_content, flags=re.M) + content3 += ( + include_content[: m_.start(1)] + + generate_filename(m_.group(1)) + + include_content[m_.end(1) :] + ) + start = m.end() + content3 += content2[start:] + + return content3 + + +def generate_filename(path): + segs = path.split(b"/") + # strip /locales/ from filename + segs = [seg for seg in segs if seg != b"locales"] + # strip variables from filename + segs = [seg for seg in segs if not seg.startswith(b"{") and not seg.endswith(b"}")] + if segs[-1] == b"l10n.toml": + segs.pop() + segs[-1] += b".toml" + outpath = b"-".join(segs) + if TARGET_PATH != b".": + # prepend the target path, if it's not '.' + outpath = mozpath.join(TARGET_PATH, outpath) + return outpath diff --git a/python/l10n/mozxchannel/source.py b/python/l10n/mozxchannel/source.py new file mode 100644 index 0000000000..683a7b0640 --- /dev/null +++ b/python/l10n/mozxchannel/source.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytoml as toml +from compare_locales import mozpath, paths +from compare_locales.paths.matcher import expand + +from .projectconfig import generate_filename + + +class HGFiles(paths.ProjectFiles): + def __init__(self, repo, rev, projects): + self.repo = repo + self.ctx = repo[rev] + self.root = repo.root() + self.manifest = None + self.configs_map = {} + # get paths for our TOML files + for p in projects: + all_configpaths = { + mozpath.abspath(c.path).encode("utf-8") for c in p.configs + } + for refpath in all_configpaths: + local_path = mozpath.relpath(refpath, self.root) + if local_path not in self.ctx: + print("ignoring", refpath) + continue + targetpath = b"/".join( + ( + expand(None, "{l10n_base}", p.environ).encode("utf-8"), + b"en-US", + generate_filename(local_path), + ) + ) + self.configs_map[refpath] = targetpath + super(HGFiles, self).__init__("en-US", projects) + for m in self.matchers: + m["l10n"].encoding = "utf-8" + if "reference" in m: + m["reference"].encoding = "utf-8" + if self.exclude: + for m in self.exclude.matchers: + m["l10n"].encoding = "utf-8" + if "reference" in m: + m["reference"].encoding = "utf-8" + + def _files(self, matcher): + for f in self.ctx.manifest(): + f = mozpath.join(self.root, f) + if matcher.match(f): + yield f + + def __iter__(self): + for t in super(HGFiles, self).__iter__(): + yield t + for refpath, targetpath in self.configs_map.items(): + yield targetpath, refpath, None, set() + + def match(self, path): + m = super(HGFiles, self).match(path) + if m: + return m + for refpath, targetpath in self.configs_map.items(): + if path in [refpath, targetpath]: + return targetpath, refpath, None, set() + + +class HgTOMLParser(paths.TOMLParser): + "subclass to load from our hg context" + + def __init__(self, repo, rev): + self.repo = repo + self.rev = rev + self.root = repo.root().decode("utf-8") + + def load(self, parse_ctx): + try: + path = parse_ctx.path + local_path = "path:" + mozpath.relpath(path, self.root) + data = self.repo.cat(files=[local_path.encode("utf-8")], rev=self.rev) + except Exception: + raise paths.ConfigNotFound(parse_ctx.path) + parse_ctx.data = toml.loads(data, filename=parse_ctx.path) |