1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from datetime import datetime, timedelta
from compare_locales import parser
from compare_locales.lint.linter import L10nLinter
from compare_locales.lint.util import l10n_base_reference_and_tests
from compare_locales.paths import ProjectFiles, TOMLParser
from mach import util as mach_util
from mozlint import pathutils, result
from mozpack import path as mozpath
from mozversioncontrol import MissingVCSTool
from mozversioncontrol.repoupdate import update_git_repo, update_mercurial_repo
L10N_SOURCE_NAME = "l10n-source"
L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
STRINGS_NAME = "gecko-strings"
STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
PULL_AFTER = timedelta(days=2)
# Wrapper to call lint_strings with mozilla-central configuration
# comm-central defines its own wrapper since comm-central strings are
# in separate repositories
def lint(paths, lintconfig, **lintargs):
extra_args = lintargs.get("extra_args") or []
name = L10N_SOURCE_NAME if "--l10n-git" in extra_args else STRINGS_NAME
return lint_strings(name, paths, lintconfig, **lintargs)
def lint_strings(name, paths, lintconfig, **lintargs):
l10n_base = mach_util.get_state_dir()
root = lintargs["root"]
exclude = lintconfig.get("exclude")
extensions = lintconfig.get("extensions")
# Load l10n.toml configs
l10nconfigs = load_configs(lintconfig["l10n_configs"], root, l10n_base, name)
# If l10n.yml is included in the provided paths, validate it against the
# TOML files, then remove it to avoid parsing it as a localizable resource.
if lintconfig["path"] in paths:
results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
paths.remove(lintconfig["path"])
lintconfig["include"].remove(mozpath.relpath(lintconfig["path"], root))
else:
results = []
all_files = []
for p in paths:
fp = pathutils.FilterPath(p)
if fp.isdir:
for _, fileobj in fp.finder:
all_files.append(fileobj.path)
if fp.isfile:
all_files.append(p)
# Filter out files explicitly excluded in the l10n.yml configuration.
# `browser/locales/en-US/firefox-l10n.js` is a good example.
all_files, _ = pathutils.filterpaths(
lintargs["root"],
all_files,
lintconfig["include"],
exclude=exclude,
extensions=extensions,
)
# Filter again, our directories might have picked up files that should be
# excluded in l10n.yml
skips = {p for p in all_files if not parser.hasParser(p)}
results.extend(
result.from_config(
lintconfig,
level="warning",
path=path,
message="file format not supported in compare-locales",
)
for path in skips
)
all_files = [p for p in all_files if p not in skips]
files = ProjectFiles(name, l10nconfigs)
get_reference_and_tests = l10n_base_reference_and_tests(files)
linter = MozL10nLinter(lintconfig)
results += linter.lint(all_files, get_reference_and_tests)
return results
# Similar to the lint/lint_strings wrapper setup, for comm-central support.
def gecko_strings_setup(**lint_args):
extra_args = lint_args.get("extra_args") or []
if "--l10n-git" in extra_args:
return source_repo_setup(L10N_SOURCE_REPO, L10N_SOURCE_NAME)
else:
return strings_repo_setup(STRINGS_REPO, STRINGS_NAME)
def source_repo_setup(repo: str, name: str):
gs = mozpath.join(mach_util.get_state_dir(), name)
marker = mozpath.join(gs, ".git", "l10n_pull_marker")
try:
last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
skip_clone = datetime.now() < last_pull + PULL_AFTER
except OSError:
skip_clone = False
if skip_clone:
return
try:
update_git_repo(repo, gs)
except MissingVCSTool:
if os.environ.get("MOZ_AUTOMATION"):
raise
print("warning: l10n linter requires Git but was unable to find 'git'")
return 1
with open(marker, "w") as fh:
fh.flush()
def strings_repo_setup(repo: str, name: str):
gs = mozpath.join(mach_util.get_state_dir(), name)
marker = mozpath.join(gs, ".hg", "l10n_pull_marker")
try:
last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
skip_clone = datetime.now() < last_pull + PULL_AFTER
except OSError:
skip_clone = False
if skip_clone:
return
try:
update_mercurial_repo(repo, gs)
except MissingVCSTool:
if os.environ.get("MOZ_AUTOMATION"):
raise
print("warning: l10n linter requires Mercurial but was unable to find 'hg'")
return 1
with open(marker, "w") as fh:
fh.flush()
def load_configs(l10n_configs, root, l10n_base, locale):
"""Load l10n configuration files specified in the linter configuration."""
configs = []
env = {"l10n_base": l10n_base}
for toml in l10n_configs:
cfg = TOMLParser().parse(
mozpath.join(root, toml), env=env, ignore_missing_includes=True
)
cfg.set_locales([locale], deep=True)
configs.append(cfg)
return configs
def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
"""Check l10n.yml config against l10n.toml configs."""
reference_paths = set(
mozpath.relpath(p["reference"].prefix, lintargs["root"])
for project in l10nconfigs
for config in project.configs
for p in config.paths
)
# Just check for directories
reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
missing_in_yml = [
refd for refd in reference_dirs if refd not in lintconfig["include"]
]
# These might be subdirectories in the config, though
missing_in_yml = [
d
for d in missing_in_yml
if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
]
if missing_in_yml:
dirs = ", ".join(missing_in_yml)
return [
result.from_config(
lintconfig,
path=lintconfig["path"],
message="l10n.yml out of sync with l10n.toml, add: " + dirs,
)
]
return []
class MozL10nLinter(L10nLinter):
"""Subclass linter to generate the right result type."""
def __init__(self, lintconfig):
super(MozL10nLinter, self).__init__()
self.lintconfig = lintconfig
def lint(self, files, get_reference_and_tests):
return [
result.from_config(self.lintconfig, **result_data)
for result_data in super(MozL10nLinter, self).lint(
files, get_reference_and_tests
)
]
|
