summaryrefslogtreecommitdiffstats
path: root/python/l10n/mozxchannel/__init__.py
blob: a531cdcaab64b4e44667885050e23de5509f425a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this,
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import shutil
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path

import hglib
from compare_locales import merge
from mozpack import path as mozpath

from . import projectconfig, source


def get_default_config(topsrcdir, strings_path):
    assert isinstance(topsrcdir, Path)
    assert isinstance(strings_path, Path)
    return {
        "strings": {
            "path": strings_path,
            "url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/",
            "heads": {"default": "default"},
            "update_on_pull": True,
            "push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/",
        },
        "source": {
            "mozilla-unified": {
                "path": topsrcdir,
                "url": "https://hg.mozilla.org/mozilla-unified/",
                "heads": {
                    # This list of repositories is ordered, starting with the
                    # one with the most recent content (central) to the oldest
                    # (ESR). In case two ESR versions are supported, the oldest
                    # ESR goes last (e.g. esr78 goes after esr91).
                    "central": "mozilla-central",
                    "beta": "releases/mozilla-beta",
                    "release": "releases/mozilla-release",
                    "esr115": "releases/mozilla-esr115",
                },
                "config_files": [
                    "browser/locales/l10n.toml",
                    "mobile/android/locales/l10n.toml",
                ],
            },
        },
    }


@dataclass
class TargetRevs:
    target: bytes = None
    revs: list = field(default_factory=list)


@dataclass
class CommitRev:
    repo: str
    rev: bytes

    @property
    def message(self):
        return (
            f"X-Channel-Repo: {self.repo}\n"
            f'X-Channel-Revision: {self.rev.decode("ascii")}'
        )


class CrossChannelCreator:
    def __init__(self, config):
        self.config = config
        self.strings_path = config["strings"]["path"]
        self.message = (
            f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}"
        )

    def create_content(self):
        self.prune_target()
        revs = []
        for repo_name, repo_config in self.config["source"].items():
            with hglib.open(repo_config["path"]) as repo:
                revs.extend(self.create_for_repo(repo, repo_name, repo_config))
        self.commit(revs)
        return 0

    def prune_target(self):
        for leaf in self.config["strings"]["path"].iterdir():
            if leaf.name == ".hg":
                continue
            shutil.rmtree(leaf)

    def create_for_repo(self, repo, repo_name, repo_config):
        print(f"Processing {repo_name} in {repo_config['path']}")
        source_target_revs = defaultdict(TargetRevs)
        revs_for_commit = []
        parse_kwargs = {
            "env": {"l10n_base": str(self.strings_path.parent)},
            "ignore_missing_includes": True,
        }
        for head, head_name in repo_config["heads"].items():
            print(f"Gathering files for {head}")
            rev = repo.log(revrange=head)[0].node
            revs_for_commit.append(CommitRev(head_name, rev))
            p = source.HgTOMLParser(repo, rev)
            project_configs = []
            for config_file in repo_config["config_files"]:
                project_configs.append(p.parse(config_file, **parse_kwargs))
                project_configs[-1].set_locales(["en-US"], deep=True)
            hgfiles = source.HGFiles(repo, rev, project_configs)
            for targetpath, refpath, _, _ in hgfiles:
                source_target_revs[refpath].revs.append(rev)
                source_target_revs[refpath].target = targetpath
        root = repo.root()
        print(f"Writing {repo_name} content to target")
        for refpath, targetrevs in source_target_revs.items():
            local_ref = mozpath.relpath(refpath, root)
            content = self.get_content(local_ref, repo, targetrevs.revs)
            target_dir = mozpath.dirname(targetrevs.target)
            if not os.path.isdir(target_dir):
                os.makedirs(target_dir)
            with open(targetrevs.target, "wb") as fh:
                fh.write(content)
        return revs_for_commit

    def commit(self, revs):
        message = self.message + "\n\n"
        if "TASK_ID" in os.environ:
            message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n"
        message += "\n".join(rev.message for rev in revs)
        with hglib.open(self.strings_path) as repo:
            repo.commit(message=message, addremove=True)

    def get_content(self, local_ref, repo, revs):
        if local_ref.endswith(b".toml"):
            return self.get_config_content(local_ref, repo, revs)
        if len(revs) < 2:
            return repo.cat([b"path:" + local_ref], rev=revs[0])
        contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs]
        try:
            return merge.merge_channels(local_ref.decode("utf-8"), contents)
        except merge.MergeNotSupportedError:
            return contents[0]

    def get_config_content(self, local_ref, repo, revs):
        # We don't support merging toml files
        content = repo.cat([b"path:" + local_ref], rev=revs[0])
        return projectconfig.process_config(content)