1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import sys
GECKO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
MOZHARNESS = os.path.join(GECKO_PATH, "testing/mozharness")
COMM_PYTHON_L10N = os.path.join(GECKO_PATH, "comm/python/l10n")
sys.path.insert(1, MOZHARNESS)
sys.path.insert(1, COMM_PYTHON_L10N)
from zstandard import ZstdCompressor
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.automation import AutomationMixin
from mozharness.mozilla.l10n.locales import LocalesMixin
from mozpack.archive import create_tar_from_files
from mozpack.copier import FileRegistry
from mozpack.files import FileFinder
from tbxchannel.l10n_merge import COMM_STRINGS_PATTERNS, GECKO_STRINGS_PATTERNS
class CommMultiLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
config_options = [
[
[
"--locale-list",
],
{
"action": "store",
"dest": "locale_list",
"type": "string",
"help": "File with locales to include. Either all-locales or shipped-locales",
},
],
[
[
"--comm-locales-file",
],
{
"action": "store",
"dest": "hg_comm_locales_file",
"type": "string",
"help": "File with HG revision of comm-l10n monorepo to use",
},
],
[
[
"--browser-locales-file",
],
{
"action": "store",
"dest": "locales_file",
"type": "string",
"help": "File with HG revisions of l10n-central repositories",
},
],
]
def __init__(self, require_config_file=False):
buildscript_kwargs = {
"all_actions": [
"clone-gecko-locales",
"clone-monorepo",
"merge-repos",
"pack-merged",
"gen-changesets",
],
"config": {
"ignore_locales": ["en-US"],
"log_name": "multi_locale",
"hg_merged_dir": "l10n_merged",
"objdir": "obj-build",
"upload_file": "strings_all.tar.zst",
"changesets_file": "l10n-changesets.json",
},
}
LocalesMixin.__init__(self)
BaseScript.__init__(
self,
config_options=self.config_options,
require_config_file=require_config_file,
**buildscript_kwargs,
)
self.upload_env = None
self.file_registry = None
self.comm_l10n_revision = None
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(CommMultiLocale, self).query_abs_dirs()
c = self.config
dirs = {}
dirs["abs_checkout_dir"] = os.path.abspath(os.path.join(abs_dirs["abs_src_dir"], ".."))
dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"])
# Needs to match abs_dirs["abs_l10n_dir"] set in mozharness.mozilla.l10n.locales
dirs["abs_l10n_central_dir"] = os.path.abspath(
os.path.join(dirs["abs_checkout_dir"], "l10n-central")
)
dirs["abs_comm_l10n_dir"] = os.path.abspath(
os.path.join(dirs["abs_checkout_dir"], "comm-l10n")
)
dirs["abs_merged_dir"] = os.path.abspath(
os.path.join(dirs["abs_work_dir"], "l10n-central")
)
for key in dirs.keys():
if key not in abs_dirs:
abs_dirs[key] = dirs[key]
self.abs_dirs = abs_dirs
return self.abs_dirs
def _query_upload_env(self):
"""returns the environment used for the upload step"""
if self.upload_env:
return self.upload_env
config = self.config
upload_env = self.query_env(partial_env=config.get("upload_env"))
self.upload_env = upload_env
return self.upload_env
def _ensure_upload_path(self):
env = self._query_upload_env()
if "UPLOAD_PATH" in env and not os.path.exists(env["UPLOAD_PATH"]):
self.mkdir_p(env["UPLOAD_PATH"])
def get_gecko_l10n_revisions(self):
# Populate self.locales with Thunderbird's locales, and revisions
# from browser/locales/l10n-changesets.json
c = self.config
ignore_locales = c.get("ignore_locales", [])
dirs = self.query_abs_dirs()
locale_list = os.path.join(dirs["abs_src_dir"], c["locale_list"])
locales = self.parse_locales_file(locale_list)
locale_changesets_file = os.path.join(dirs["abs_src_dir"], c["locales_file"])
# parse_locales_file fills in self.l10n_revisions with changesets
self.parse_locales_file(locale_changesets_file)
for locale in ignore_locales:
if locale in locales:
self.debug("Ignoring locale %s." % locale)
locales.remove(locale)
self.locales = locales
# Actions {{{2
def clone_gecko_locales(self):
self.get_gecko_l10n_revisions()
self.pull_locale_source()
def clone_monorepo(self):
c = self.config
dirs = self.query_abs_dirs()
locales_file = os.path.join(dirs["abs_src_dir"], c["hg_comm_locales_file"])
locales_data = {}
if locales_file.endswith(".json"):
with open(locales_file) as fh:
locales_data = json.load(fh)
# would use en-US, but it's not in this file!
self.comm_l10n_revision = locales_data.get("en-GB", {}).get("revision")
if self.comm_l10n_revision:
self.mkdir_p(dirs["abs_checkout_dir"])
self.vcs_checkout(
dest=dirs["abs_comm_l10n_dir"],
repo=c["hg_comm_l10n_repo"],
branch=self.comm_l10n_revision,
vcs="hg",
)
else:
raise Exception(
f"Unable to find revision from comm-l10n repo using "
f"{c['hg_comm_locales_file']}."
)
def merge_repos(self):
dirs = self.query_abs_dirs()
if os.path.exists(dirs["abs_merged_dir"]):
self.rmtree(dirs["abs_merged_dir"])
file_registry = FileRegistry()
def add_to_registry(base_path, patterns):
finder = FileFinder(base_path)
for pattern in patterns:
for _lang in self.locales:
for _filepath, _fileobj in finder.find(pattern.format(lang=_lang)):
_filepath = os.path.join("l10n-central", _filepath)
file_registry.add(_filepath, _fileobj)
add_to_registry(dirs["abs_l10n_central_dir"], GECKO_STRINGS_PATTERNS)
add_to_registry(dirs["abs_comm_l10n_dir"], COMM_STRINGS_PATTERNS)
self.file_registry = file_registry
def pack_merged(self):
self._ensure_upload_path()
upload_path = self.config["upload_env"]["UPLOAD_PATH"]
archive_path = os.path.join(upload_path, self.config["upload_file"])
with open(archive_path, "wb") as f:
with ZstdCompressor().stream_writer(f) as z:
create_tar_from_files(z, dict(self.file_registry))
def gen_changesets(self):
changeset_data = {
"gecko_strings": self.gecko_locale_revisions,
"comm_strings": {
"repo": self.config["hg_comm_l10n_repo"],
"revision": self.comm_l10n_revision,
},
}
upload_path = self.config["upload_env"]["UPLOAD_PATH"]
changesets_file = os.path.join(upload_path, self.config["changesets_file"])
with open(changesets_file, "w") as f:
json.dump(changeset_data, f, sort_keys=True, indent=2)
if __name__ == "__main__":
single_locale = CommMultiLocale()
single_locale.run_and_exit()
|
