summaryrefslogtreecommitdiffstats
path: root/tools/lint/python
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:44:51 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:44:51 +0000
commit9e3c08db40b8916968b9f30096c7be3f00ce9647 (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /tools/lint/python
parentInitial commit. (diff)
downloadthunderbird-9e3c08db40b8916968b9f30096c7be3f00ce9647.tar.xz
thunderbird-9e3c08db40b8916968b9f30096c7be3f00ce9647.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tools/lint/python')
-rw-r--r--tools/lint/python/__init__.py3
-rw-r--r--tools/lint/python/black.py179
-rw-r--r--tools/lint/python/black_requirements.in4
-rw-r--r--tools/lint/python/black_requirements.txt117
-rw-r--r--tools/lint/python/l10n_lint.py171
-rw-r--r--tools/lint/python/ruff.py187
-rw-r--r--tools/lint/python/ruff_requirements.in1
-rw-r--r--tools/lint/python/ruff_requirements.txt25
8 files changed, 687 insertions, 0 deletions
diff --git a/tools/lint/python/__init__.py b/tools/lint/python/__init__.py
new file mode 100644
index 0000000000..c580d191c1
--- /dev/null
+++ b/tools/lint/python/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/tools/lint/python/black.py b/tools/lint/python/black.py
new file mode 100644
index 0000000000..ac975e62a2
--- /dev/null
+++ b/tools/lint/python/black.py
@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+
+import mozpack.path as mozpath
+from mozfile import which
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozprocess import ProcessHandler
+
+here = os.path.abspath(os.path.dirname(__file__))
+BLACK_REQUIREMENTS_PATH = os.path.join(here, "black_requirements.txt")
+
+BLACK_INSTALL_ERROR = """
+Unable to install correct version of black
+Try to install it manually with:
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ BLACK_REQUIREMENTS_PATH
+)
+
+
+def default_bindir():
+ # We use sys.prefix to find executables as that gets modified with
+ # virtualenv's activate_this.py, whereas sys.executable doesn't.
+ if platform.system() == "Windows":
+ return os.path.join(sys.prefix, "Scripts")
+ else:
+ return os.path.join(sys.prefix, "bin")
+
+
+def get_black_version(binary):
+ """
+ Returns found binary's version
+ """
+ try:
+ output = subprocess.check_output(
+ [binary, "--version"],
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+ except subprocess.CalledProcessError as e:
+ output = e.output
+ try:
+ # Accept `black.EXE, version ...` on Windows.
+ # for old version of black, the output is
+ # black, version 21.4b2
+ # From black 21.11b1, the output is like
+ # black, 21.11b1 (compiled: no)
+ return re.match(r"black.*,( version)? (\S+)", output)[2]
+ except TypeError as e:
+ print("Could not parse the version '{}'".format(output))
+ print("Error: {}".format(e))
+
+
+def parse_issues(config, output, paths, *, log):
+ would_reformat = re.compile("^would reformat (.*)$", re.I)
+ reformatted = re.compile("^reformatted (.*)$", re.I)
+ cannot_reformat = re.compile("^error: cannot format (.*?): (.*)$", re.I)
+ results = []
+ for line in output:
+ line = line.decode("utf-8")
+ if line.startswith("All done!") or line.startswith("Oh no!"):
+ break
+
+ match = would_reformat.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "error"}
+ results.append(result.from_config(config, **res))
+ continue
+
+ match = reformatted.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "warning", "message": "reformatted"}
+ results.append(result.from_config(config, **res))
+ continue
+
+ match = cannot_reformat.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "error", "message": match.group(2)}
+ results.append(result.from_config(config, **res))
+ continue
+
+ log.debug(f"Unhandled line: {line}")
+ return results
+
+
+class BlackProcess(ProcessHandler):
+ def __init__(self, config, *args, **kwargs):
+ self.config = config
+ kwargs["stream"] = False
+ ProcessHandler.__init__(self, *args, **kwargs)
+
+ def run(self, *args, **kwargs):
+ orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+ ProcessHandler.run(self, *args, **kwargs)
+ signal.signal(signal.SIGINT, orig)
+
+
+def run_process(config, cmd):
+ proc = BlackProcess(config, cmd)
+ proc.run()
+ try:
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+
+ return proc.output
+
+
+def setup(root, **lintargs):
+ log = lintargs["log"]
+ virtualenv_bin_path = lintargs.get("virtualenv_bin_path")
+ # Using `which` searches multiple directories and handles `.exe` on Windows.
+ binary = which("black", path=(virtualenv_bin_path, default_bindir()))
+
+ if binary and os.path.exists(binary):
+ binary = mozpath.normsep(binary)
+ log.debug("Looking for black at {}".format(binary))
+ version = get_black_version(binary)
+ versions = [
+ line.split()[0].strip()
+ for line in open(BLACK_REQUIREMENTS_PATH).readlines()
+ if line.startswith("black==")
+ ]
+ if ["black=={}".format(version)] == versions:
+ log.debug("Black is present with expected version {}".format(version))
+ return 0
+ else:
+ log.debug("Black is present but unexpected version {}".format(version))
+
+ log.debug("Black needs to be installed or updated")
+ virtualenv_manager = lintargs["virtualenv_manager"]
+ try:
+ virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH, quiet=True)
+ except subprocess.CalledProcessError:
+ print(BLACK_INSTALL_ERROR)
+ return 1
+
+
+def run_black(config, paths, fix=None, *, log, virtualenv_bin_path):
+ fixed = 0
+ binary = os.path.join(virtualenv_bin_path or default_bindir(), "black")
+
+ log.debug("Black version {}".format(get_black_version(binary)))
+
+ cmd_args = [binary]
+ if not fix:
+ cmd_args.append("--check")
+
+ base_command = cmd_args + paths
+ log.debug("Command: {}".format(" ".join(base_command)))
+ output = parse_issues(config, run_process(config, base_command), paths, log=log)
+
+ # black returns an issue for fixed files as well
+ for eachIssue in output:
+ if eachIssue.message == "reformatted":
+ fixed += 1
+
+ return {"results": output, "fixed": fixed}
+
+
+def lint(paths, config, fix=None, **lintargs):
+ files = list(expand_exclusions(paths, config, lintargs["root"]))
+
+ return run_black(
+ config,
+ files,
+ fix=fix,
+ log=lintargs["log"],
+ virtualenv_bin_path=lintargs.get("virtualenv_bin_path"),
+ )
diff --git a/tools/lint/python/black_requirements.in b/tools/lint/python/black_requirements.in
new file mode 100644
index 0000000000..e5efa47492
--- /dev/null
+++ b/tools/lint/python/black_requirements.in
@@ -0,0 +1,4 @@
+black==21.11b1
+typing-extensions==3.10.0.2
+dataclasses==0.6
+
diff --git a/tools/lint/python/black_requirements.txt b/tools/lint/python/black_requirements.txt
new file mode 100644
index 0000000000..ffa9ef3564
--- /dev/null
+++ b/tools/lint/python/black_requirements.txt
@@ -0,0 +1,117 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+# pip-compile --generate-hashes --output-file=tools/lint/python/black_requirements.txt tools/lint/python/black_requirements.in
+#
+black==21.11b1 \
+ --hash=sha256:802c6c30b637b28645b7fde282ed2569c0cd777dbe493a41b6a03c1d903f99ac \
+ --hash=sha256:a042adbb18b3262faad5aff4e834ff186bb893f95ba3a8013f09de1e5569def2
+ # via -r tools/lint/python/black_requirements.in
+click==8.0.3 \
+ --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \
+ --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b
+ # via black
+dataclasses==0.6 \
+ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \
+ --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84
+ # via -r tools/lint/python/black_requirements.in
+mypy-extensions==0.4.3 \
+ --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
+ --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8
+ # via black
+pathspec==0.9.0 \
+ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \
+ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1
+ # via black
+platformdirs==2.4.0 \
+ --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \
+ --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d
+ # via black
+regex==2021.11.10 \
+ --hash=sha256:05b7d6d7e64efe309972adab77fc2af8907bb93217ec60aa9fe12a0dad35874f \
+ --hash=sha256:0617383e2fe465732af4509e61648b77cbe3aee68b6ac8c0b6fe934db90be5cc \
+ --hash=sha256:07856afef5ffcc052e7eccf3213317fbb94e4a5cd8177a2caa69c980657b3cb4 \
+ --hash=sha256:162abfd74e88001d20cb73ceaffbfe601469923e875caf9118333b1a4aaafdc4 \
+ --hash=sha256:2207ae4f64ad3af399e2d30dde66f0b36ae5c3129b52885f1bffc2f05ec505c8 \
+ --hash=sha256:30ab804ea73972049b7a2a5c62d97687d69b5a60a67adca07eb73a0ddbc9e29f \
+ --hash=sha256:3b5df18db1fccd66de15aa59c41e4f853b5df7550723d26aa6cb7f40e5d9da5a \
+ --hash=sha256:3c5fb32cc6077abad3bbf0323067636d93307c9fa93e072771cf9a64d1c0f3ef \
+ --hash=sha256:416c5f1a188c91e3eb41e9c8787288e707f7d2ebe66e0a6563af280d9b68478f \
+ --hash=sha256:432bd15d40ed835a51617521d60d0125867f7b88acf653e4ed994a1f8e4995dc \
+ --hash=sha256:4aaa4e0705ef2b73dd8e36eeb4c868f80f8393f5f4d855e94025ce7ad8525f50 \
+ --hash=sha256:537ca6a3586931b16a85ac38c08cc48f10fc870a5b25e51794c74df843e9966d \
+ --hash=sha256:53db2c6be8a2710b359bfd3d3aa17ba38f8aa72a82309a12ae99d3c0c3dcd74d \
+ --hash=sha256:5537f71b6d646f7f5f340562ec4c77b6e1c915f8baae822ea0b7e46c1f09b733 \
+ --hash=sha256:6650f16365f1924d6014d2ea770bde8555b4a39dc9576abb95e3cd1ff0263b36 \
+ --hash=sha256:666abff54e474d28ff42756d94544cdfd42e2ee97065857413b72e8a2d6a6345 \
+ --hash=sha256:68a067c11463de2a37157930d8b153005085e42bcb7ad9ca562d77ba7d1404e0 \
+ --hash=sha256:780b48456a0f0ba4d390e8b5f7c661fdd218934388cde1a974010a965e200e12 \
+ --hash=sha256:788aef3549f1924d5c38263104dae7395bf020a42776d5ec5ea2b0d3d85d6646 \
+ --hash=sha256:7ee1227cf08b6716c85504aebc49ac827eb88fcc6e51564f010f11a406c0a667 \
+ --hash=sha256:7f301b11b9d214f83ddaf689181051e7f48905568b0c7017c04c06dfd065e244 \
+ --hash=sha256:83ee89483672b11f8952b158640d0c0ff02dc43d9cb1b70c1564b49abe92ce29 \
+ --hash=sha256:85bfa6a5413be0ee6c5c4a663668a2cad2cbecdee367630d097d7823041bdeec \
+ --hash=sha256:9345b6f7ee578bad8e475129ed40123d265464c4cfead6c261fd60fc9de00bcf \
+ --hash=sha256:93a5051fcf5fad72de73b96f07d30bc29665697fb8ecdfbc474f3452c78adcf4 \
+ --hash=sha256:962b9a917dd7ceacbe5cd424556914cb0d636001e393b43dc886ba31d2a1e449 \
+ --hash=sha256:98ba568e8ae26beb726aeea2273053c717641933836568c2a0278a84987b2a1a \
+ --hash=sha256:a3feefd5e95871872673b08636f96b61ebef62971eab044f5124fb4dea39919d \
+ --hash=sha256:b43c2b8a330a490daaef5a47ab114935002b13b3f9dc5da56d5322ff218eeadb \
+ --hash=sha256:b483c9d00a565633c87abd0aaf27eb5016de23fed952e054ecc19ce32f6a9e7e \
+ --hash=sha256:ba05430e819e58544e840a68b03b28b6d328aff2e41579037e8bab7653b37d83 \
+ --hash=sha256:ca5f18a75e1256ce07494e245cdb146f5a9267d3c702ebf9b65c7f8bd843431e \
+ --hash=sha256:d5ca078bb666c4a9d1287a379fe617a6dccd18c3e8a7e6c7e1eb8974330c626a \
+ --hash=sha256:da1a90c1ddb7531b1d5ff1e171b4ee61f6345119be7351104b67ff413843fe94 \
+ --hash=sha256:dba70f30fd81f8ce6d32ddeef37d91c8948e5d5a4c63242d16a2b2df8143aafc \
+ --hash=sha256:dd33eb9bdcfbabab3459c9ee651d94c842bc8a05fabc95edf4ee0c15a072495e \
+ --hash=sha256:e0538c43565ee6e703d3a7c3bdfe4037a5209250e8502c98f20fea6f5fdf2965 \
+ --hash=sha256:e1f54b9b4b6c53369f40028d2dd07a8c374583417ee6ec0ea304e710a20f80a0 \
+ --hash=sha256:e32d2a2b02ccbef10145df9135751abea1f9f076e67a4e261b05f24b94219e36 \
+ --hash=sha256:e71255ba42567d34a13c03968736c5d39bb4a97ce98188fafb27ce981115beec \
+ --hash=sha256:ed2e07c6a26ed4bea91b897ee2b0835c21716d9a469a96c3e878dc5f8c55bb23 \
+ --hash=sha256:eef2afb0fd1747f33f1ee3e209bce1ed582d1896b240ccc5e2697e3275f037c7 \
+ --hash=sha256:f23222527b307970e383433daec128d769ff778d9b29343fb3496472dc20dabe \
+ --hash=sha256:f341ee2df0999bfdf7a95e448075effe0db212a59387de1a70690e4acb03d4c6 \
+ --hash=sha256:f7f325be2804246a75a4f45c72d4ce80d2443ab815063cdf70ee8fb2ca59ee1b \
+ --hash=sha256:f8af619e3be812a2059b212064ea7a640aff0568d972cd1b9e920837469eb3cb \
+ --hash=sha256:fa8c626d6441e2d04b6ee703ef2d1e17608ad44c7cb75258c09dd42bacdfc64b \
+ --hash=sha256:fbb9dc00e39f3e6c0ef48edee202f9520dafb233e8b51b06b8428cfcb92abd30 \
+ --hash=sha256:fff55f3ce50a3ff63ec8e2a8d3dd924f1941b250b0aac3d3d42b687eeff07a8e
+ # via black
+tomli==1.2.2 \
+ --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \
+ --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade
+ # via black
+typed-ast==1.5.4 \
+ --hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \
+ --hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \
+ --hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \
+ --hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \
+ --hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \
+ --hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \
+ --hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \
+ --hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \
+ --hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \
+ --hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \
+ --hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \
+ --hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \
+ --hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \
+ --hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \
+ --hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \
+ --hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \
+ --hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \
+ --hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \
+ --hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \
+ --hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \
+ --hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \
+ --hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \
+ --hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \
+ --hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66
+typing-extensions==3.10.0.2 \
+ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
+ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \
+ --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34
+ # via
+ # -r tools/lint/python/black_requirements.in
+ # black
diff --git a/tools/lint/python/l10n_lint.py b/tools/lint/python/l10n_lint.py
new file mode 100644
index 0000000000..ef3269ef2a
--- /dev/null
+++ b/tools/lint/python/l10n_lint.py
@@ -0,0 +1,171 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from datetime import datetime, timedelta
+
+import mozversioncontrol.repoupdate
+from compare_locales import parser
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import l10n_base_reference_and_tests
+from compare_locales.paths import ProjectFiles, TOMLParser
+from mach import util as mach_util
+from mozlint import pathutils, result
+from mozpack import path as mozpath
+
+LOCALE = "gecko-strings"
+STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
+
+PULL_AFTER = timedelta(days=2)
+
+# Wrapper to call lint_strings with mozilla-central configuration
+# comm-central defines its own wrapper since comm-central strings are
+# in separate repositories
+def lint(paths, lintconfig, **lintargs):
+ return lint_strings(LOCALE, paths, lintconfig, **lintargs)
+
+
+def lint_strings(locale, paths, lintconfig, **lintargs):
+ l10n_base = mach_util.get_state_dir()
+ root = lintargs["root"]
+ exclude = lintconfig.get("exclude")
+ extensions = lintconfig.get("extensions")
+
+ # Load l10n.toml configs
+ l10nconfigs = load_configs(lintconfig, root, l10n_base, locale)
+
+ # Check include paths in l10n.yml if it's in our given paths
+ # Only the l10n.yml will show up here, but if the l10n.toml files
+ # change, we also get the l10n.yml as the toml files are listed as
+ # support files.
+ if lintconfig["path"] in paths:
+ results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
+ paths.remove(lintconfig["path"])
+ else:
+ results = []
+
+ all_files = []
+ for p in paths:
+ fp = pathutils.FilterPath(p)
+ if fp.isdir:
+ for _, fileobj in fp.finder:
+ all_files.append(fileobj.path)
+ if fp.isfile:
+ all_files.append(p)
+ # Filter again, our directories might have picked up files the
+ # explicitly excluded in the l10n.yml configuration.
+ # `browser/locales/en-US/firefox-l10n.js` is a good example.
+ all_files, _ = pathutils.filterpaths(
+ lintargs["root"],
+ all_files,
+ lintconfig["include"],
+ exclude=exclude,
+ extensions=extensions,
+ )
+ # These should be excluded in l10n.yml
+ skips = {p for p in all_files if not parser.hasParser(p)}
+ results.extend(
+ result.from_config(
+ lintconfig,
+ level="warning",
+ path=path,
+ message="file format not supported in compare-locales",
+ )
+ for path in skips
+ )
+ all_files = [p for p in all_files if p not in skips]
+ files = ProjectFiles(locale, l10nconfigs)
+
+ get_reference_and_tests = l10n_base_reference_and_tests(files)
+
+ linter = MozL10nLinter(lintconfig)
+ results += linter.lint(all_files, get_reference_and_tests)
+ return results
+
+
+# Similar to the lint/lint_strings wrapper setup, for comm-central support.
+def gecko_strings_setup(**lint_args):
+ return strings_repo_setup(STRINGS_REPO, LOCALE)
+
+
+def strings_repo_setup(repo, locale):
+ gs = mozpath.join(mach_util.get_state_dir(), locale)
+ marker = mozpath.join(gs, ".hg", "l10n_pull_marker")
+ try:
+ last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+ skip_clone = datetime.now() < last_pull + PULL_AFTER
+ except OSError:
+ skip_clone = False
+ if skip_clone:
+ return
+ try:
+ hg = mozversioncontrol.get_tool_path("hg")
+ except mozversioncontrol.MissingVCSTool:
+ if os.environ.get("MOZ_AUTOMATION"):
+ raise
+ print("warning: l10n linter requires Mercurial but was unable to find 'hg'")
+ return 1
+ mozversioncontrol.repoupdate.update_mercurial_repo(hg, repo, gs)
+ with open(marker, "w") as fh:
+ fh.flush()
+
+
+def load_configs(lintconfig, root, l10n_base, locale):
+ """Load l10n configuration files specified in the linter configuration."""
+ configs = []
+ env = {"l10n_base": l10n_base}
+ for toml in lintconfig["l10n_configs"]:
+ cfg = TOMLParser().parse(
+ mozpath.join(root, toml), env=env, ignore_missing_includes=True
+ )
+ cfg.set_locales([locale], deep=True)
+ configs.append(cfg)
+ return configs
+
+
+def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
+ """Check l10n.yml config against l10n.toml configs."""
+ reference_paths = set(
+ mozpath.relpath(p["reference"].prefix, lintargs["root"])
+ for project in l10nconfigs
+ for config in project.configs
+ for p in config.paths
+ )
+ # Just check for directories
+ reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
+ missing_in_yml = [
+ refd for refd in reference_dirs if refd not in lintconfig["include"]
+ ]
+ # These might be subdirectories in the config, though
+ missing_in_yml = [
+ d
+ for d in missing_in_yml
+ if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
+ ]
+ if missing_in_yml:
+ dirs = ", ".join(missing_in_yml)
+ return [
+ result.from_config(
+ lintconfig,
+ path=lintconfig["path"],
+ message="l10n.yml out of sync with l10n.toml, add: " + dirs,
+ )
+ ]
+ return []
+
+
+class MozL10nLinter(L10nLinter):
+ """Subclass linter to generate the right result type."""
+
+ def __init__(self, lintconfig):
+ super(MozL10nLinter, self).__init__()
+ self.lintconfig = lintconfig
+
+ def lint(self, files, get_reference_and_tests):
+ return [
+ result.from_config(self.lintconfig, **result_data)
+ for result_data in super(MozL10nLinter, self).lint(
+ files, get_reference_and_tests
+ )
+ ]
diff --git a/tools/lint/python/ruff.py b/tools/lint/python/ruff.py
new file mode 100644
index 0000000000..d8096199f3
--- /dev/null
+++ b/tools/lint/python/ruff.py
@@ -0,0 +1,187 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+import mozfile
+from mozlint import result
+from mozprocess.processhandler import ProcessHandler
+
+here = os.path.abspath(os.path.dirname(__file__))
+RUFF_REQUIREMENTS_PATH = os.path.join(here, "ruff_requirements.txt")
+
+RUFF_NOT_FOUND = """
+Could not find ruff! Install ruff and try again.
+
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ RUFF_REQUIREMENTS_PATH
+)
+
+
+RUFF_INSTALL_ERROR = """
+Unable to install correct version of ruff!
+Try to install it manually with:
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ RUFF_REQUIREMENTS_PATH
+)
+
+
+def default_bindir():
+ # We use sys.prefix to find executables as that gets modified with
+ # virtualenv's activate_this.py, whereas sys.executable doesn't.
+ if platform.system() == "Windows":
+ return os.path.join(sys.prefix, "Scripts")
+ else:
+ return os.path.join(sys.prefix, "bin")
+
+
+def get_ruff_version(binary):
+ """
+ Returns found binary's version
+ """
+ try:
+ output = subprocess.check_output(
+ [binary, "--version"],
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ except subprocess.CalledProcessError as e:
+ output = e.output
+
+ matches = re.match(r"ruff ([0-9\.]+)", output)
+ if matches:
+ return matches[1]
+ print("Error: Could not parse the version '{}'".format(output))
+
+
+def setup(root, log, **lintargs):
+ virtualenv_bin_path = lintargs.get("virtualenv_bin_path")
+ binary = mozfile.which("ruff", path=(virtualenv_bin_path, default_bindir()))
+
+ if binary and os.path.isfile(binary):
+ log.debug(f"Looking for ruff at {binary}")
+ version = get_ruff_version(binary)
+ versions = [
+ line.split()[0].strip()
+ for line in open(RUFF_REQUIREMENTS_PATH).readlines()
+ if line.startswith("ruff==")
+ ]
+ if [f"ruff=={version}"] == versions:
+ log.debug("ruff is present with expected version {}".format(version))
+ return 0
+ else:
+ log.debug("ruff is present but unexpected version {}".format(version))
+
+ virtualenv_manager = lintargs["virtualenv_manager"]
+ try:
+ virtualenv_manager.install_pip_requirements(RUFF_REQUIREMENTS_PATH, quiet=True)
+ except subprocess.CalledProcessError:
+ print(RUFF_INSTALL_ERROR)
+ return 1
+
+
+class RuffProcess(ProcessHandler):
+ def __init__(self, config, *args, **kwargs):
+ self.config = config
+ self.stderr = []
+ kwargs["stream"] = False
+ kwargs["universal_newlines"] = True
+ ProcessHandler.__init__(self, *args, **kwargs)
+
+ def run(self, *args, **kwargs):
+ orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+ ProcessHandler.run(self, *args, **kwargs)
+ signal.signal(signal.SIGINT, orig)
+
+
+def run_process(config, cmd, **kwargs):
+ proc = RuffProcess(config, cmd, **kwargs)
+ proc.run()
+ try:
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+
+ return "\n".join(proc.output)
+
+
+def lint(paths, config, log, **lintargs):
+ fixed = 0
+ results = []
+
+ if not paths:
+ return {"results": results, "fixed": fixed}
+
+ # Currently ruff only lints non `.py` files if they are explicitly passed
+ # in. So we need to find any non-py files manually. This can be removed
+ # after https://github.com/charliermarsh/ruff/issues/3410 is fixed.
+ exts = [e for e in config["extensions"] if e != "py"]
+ non_py_files = []
+ for path in paths:
+ p = Path(path)
+ if not p.is_dir():
+ continue
+ for ext in exts:
+ non_py_files.extend([str(f) for f in p.glob(f"**/*.{ext}")])
+
+ args = ["ruff", "check", "--force-exclude"] + paths + non_py_files
+
+ if config["exclude"]:
+ args.append(f"--extend-exclude={','.join(config['exclude'])}")
+
+ process_kwargs = {"processStderrLine": lambda line: log.debug(line)}
+
+ warning_rules = set(config.get("warning-rules", []))
+ if lintargs.get("fix"):
+ # Do a first pass with --fix-only as the json format doesn't return the
+ # number of fixed issues.
+ fix_args = args + ["--fix-only"]
+
+ # Don't fix warnings to limit unrelated changes sneaking into patches.
+ fix_args.append(f"--extend-ignore={','.join(warning_rules)}")
+ output = run_process(config, fix_args, **process_kwargs)
+ matches = re.match(r"Fixed (\d+) errors?.", output)
+ if matches:
+ fixed = int(matches[1])
+
+ log.debug(f"Running with args: {args}")
+
+ output = run_process(config, args + ["--format=json"], **process_kwargs)
+ if not output:
+ return []
+
+ try:
+ issues = json.loads(output)
+ except json.JSONDecodeError:
+ log.error(f"could not parse output: {output}")
+ return []
+
+ for issue in issues:
+ res = {
+ "path": issue["filename"],
+ "lineno": issue["location"]["row"],
+ "column": issue["location"]["column"],
+ "lineoffset": issue["end_location"]["row"] - issue["location"]["row"],
+ "message": issue["message"],
+ "rule": issue["code"],
+ "level": "error",
+ }
+ if any(issue["code"].startswith(w) for w in warning_rules):
+ res["level"] = "warning"
+
+ if issue["fix"]:
+ res["hint"] = issue["fix"]["message"]
+
+ results.append(result.from_config(config, **res))
+
+ return {"results": results, "fixed": fixed}
diff --git a/tools/lint/python/ruff_requirements.in b/tools/lint/python/ruff_requirements.in
new file mode 100644
index 0000000000..af3ee57638
--- /dev/null
+++ b/tools/lint/python/ruff_requirements.in
@@ -0,0 +1 @@
+ruff
diff --git a/tools/lint/python/ruff_requirements.txt b/tools/lint/python/ruff_requirements.txt
new file mode 100644
index 0000000000..1c8943c26b
--- /dev/null
+++ b/tools/lint/python/ruff_requirements.txt
@@ -0,0 +1,25 @@
+#
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
+#
+# pip-compile --generate-hashes ruff_requirements.in
+#
+ruff==0.0.254 \
+ --hash=sha256:059a380c08e849b6f312479b18cc63bba2808cff749ad71555f61dd930e3c9a2 \
+ --hash=sha256:09c764bc2bd80c974f7ce1f73a46092c286085355a5711126af351b9ae4bea0c \
+ --hash=sha256:0deb1d7226ea9da9b18881736d2d96accfa7f328c67b7410478cc064ad1fa6aa \
+ --hash=sha256:0eb66c9520151d3bd950ea43b3a088618a8e4e10a5014a72687881e6f3606312 \
+ --hash=sha256:27d39d697fdd7df1f2a32c1063756ee269ad8d5345c471ee3ca450636d56e8c6 \
+ --hash=sha256:2fc21d060a3197ac463596a97d9b5db2d429395938b270ded61dd60f0e57eb21 \
+ --hash=sha256:688379050ae05394a6f9f9c8471587fd5dcf22149bd4304a4ede233cc4ef89a1 \
+ --hash=sha256:8deba44fd563361c488dedec90dc330763ee0c01ba54e17df54ef5820079e7e0 \
+ --hash=sha256:ac1429be6d8bd3db0bf5becac3a38bd56f8421447790c50599cd90fd53417ec4 \
+ --hash=sha256:b3f15d5d033fd3dcb85d982d6828ddab94134686fac2c02c13a8822aa03e1321 \
+ --hash=sha256:b435afc4d65591399eaf4b2af86e441a71563a2091c386cadf33eaa11064dc09 \
+ --hash=sha256:c38291bda4c7b40b659e8952167f386e86ec29053ad2f733968ff1d78b4c7e15 \
+ --hash=sha256:d4385cdd30153b7aa1d8f75dfd1ae30d49c918ead7de07e69b7eadf0d5538a1f \
+ --hash=sha256:dd58c500d039fb381af8d861ef456c3e94fd6855c3d267d6c6718c9a9fe07be0 \
+ --hash=sha256:e15742df0f9a3615fbdc1ee9a243467e97e75bf88f86d363eee1ed42cedab1ec \
+ --hash=sha256:ef20bf798ffe634090ad3dc2e8aa6a055f08c448810a2f800ab716cc18b80107 \
+ --hash=sha256:f70dc93bc9db15cccf2ed2a831938919e3e630993eeea6aba5c84bc274237885
+ # via -r ruff_requirements.in