1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this,
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import shutil
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
import hglib
from compare_locales import merge
from mozpack import path as mozpath
from . import projectconfig, source
def get_default_config(topsrcdir, strings_path):
assert isinstance(topsrcdir, Path)
assert isinstance(strings_path, Path)
return {
"strings": {
"path": strings_path,
"url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/",
"heads": {"default": "default"},
"update_on_pull": True,
"push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/",
},
"source": {
"mozilla-unified": {
"path": topsrcdir,
"url": "https://hg.mozilla.org/mozilla-unified/",
"heads": {
# This list of repositories is ordered, starting with the
# one with the most recent content (central) to the oldest
# (ESR). In case two ESR versions are supported, the oldest
# ESR goes last (e.g. esr78 goes after esr91).
"central": "mozilla-central",
"beta": "releases/mozilla-beta",
"release": "releases/mozilla-release",
"esr102": "releases/mozilla-esr102",
},
"config_files": [
"browser/locales/l10n.toml",
"mobile/android/locales/l10n.toml",
],
},
},
}
@dataclass
class TargetRevs:
target: bytes = None
revs: list = field(default_factory=list)
@dataclass
class CommitRev:
repo: str
rev: bytes
@property
def message(self):
return (
f"X-Channel-Repo: {self.repo}\n"
f'X-Channel-Revision: {self.rev.decode("ascii")}'
)
class CrossChannelCreator:
def __init__(self, config):
self.config = config
self.strings_path = config["strings"]["path"]
self.message = (
f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}"
)
def create_content(self):
self.prune_target()
revs = []
for repo_name, repo_config in self.config["source"].items():
with hglib.open(repo_config["path"]) as repo:
revs.extend(self.create_for_repo(repo, repo_name, repo_config))
self.commit(revs)
return 0
def prune_target(self):
for leaf in self.config["strings"]["path"].iterdir():
if leaf.name == ".hg":
continue
shutil.rmtree(leaf)
def create_for_repo(self, repo, repo_name, repo_config):
print(f"Processing {repo_name} in {repo_config['path']}")
source_target_revs = defaultdict(TargetRevs)
revs_for_commit = []
parse_kwargs = {
"env": {"l10n_base": str(self.strings_path.parent)},
"ignore_missing_includes": True,
}
for head, head_name in repo_config["heads"].items():
print(f"Gathering files for {head}")
rev = repo.log(revrange=head)[0].node
revs_for_commit.append(CommitRev(head_name, rev))
p = source.HgTOMLParser(repo, rev)
project_configs = []
for config_file in repo_config["config_files"]:
project_configs.append(p.parse(config_file, **parse_kwargs))
project_configs[-1].set_locales(["en-US"], deep=True)
hgfiles = source.HGFiles(repo, rev, project_configs)
for targetpath, refpath, _, _ in hgfiles:
source_target_revs[refpath].revs.append(rev)
source_target_revs[refpath].target = targetpath
root = repo.root()
print(f"Writing {repo_name} content to target")
for refpath, targetrevs in source_target_revs.items():
local_ref = mozpath.relpath(refpath, root)
content = self.get_content(local_ref, repo, targetrevs.revs)
target_dir = mozpath.dirname(targetrevs.target)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
with open(targetrevs.target, "wb") as fh:
fh.write(content)
return revs_for_commit
def commit(self, revs):
message = self.message + "\n\n"
if "TASK_ID" in os.environ:
message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n"
message += "\n".join(rev.message for rev in revs)
with hglib.open(self.strings_path) as repo:
repo.commit(message=message, addremove=True)
def get_content(self, local_ref, repo, revs):
if local_ref.endswith(b".toml"):
return self.get_config_content(local_ref, repo, revs)
if len(revs) < 2:
return repo.cat([b"path:" + local_ref], rev=revs[0])
contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs]
try:
return merge.merge_channels(local_ref.decode("utf-8"), contents)
except merge.MergeNotSupportedError:
return contents[0]
def get_config_content(self, local_ref, repo, revs):
# We don't support merging toml files
content = repo.cat([b"path:" + local_ref], rev=revs[0])
return projectconfig.process_config(content)
|
