summaryrefslogtreecommitdiffstats
path: root/tools/lint/python
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /tools/lint/python
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tools/lint/python')
-rw-r--r--tools/lint/python/__init__.py3
-rw-r--r--tools/lint/python/black.py168
-rw-r--r--tools/lint/python/black_requirements.in5
-rw-r--r--tools/lint/python/black_requirements.txt115
-rw-r--r--tools/lint/python/l10n_lint.py202
-rw-r--r--tools/lint/python/ruff.py176
-rw-r--r--tools/lint/python/ruff_requirements.in2
-rw-r--r--tools/lint/python/ruff_requirements.txt29
8 files changed, 700 insertions, 0 deletions
diff --git a/tools/lint/python/__init__.py b/tools/lint/python/__init__.py
new file mode 100644
index 0000000000..c580d191c1
--- /dev/null
+++ b/tools/lint/python/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/tools/lint/python/black.py b/tools/lint/python/black.py
new file mode 100644
index 0000000000..1e38dd669c
--- /dev/null
+++ b/tools/lint/python/black.py
@@ -0,0 +1,168 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+
+import mozpack.path as mozpath
+from mozfile import which
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+
+here = os.path.abspath(os.path.dirname(__file__))
+BLACK_REQUIREMENTS_PATH = os.path.join(here, "black_requirements.txt")
+
+BLACK_INSTALL_ERROR = """
+Unable to install correct version of black
+Try to install it manually with:
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ BLACK_REQUIREMENTS_PATH
+)
+
+
+def default_bindir():
+ # We use sys.prefix to find executables as that gets modified with
+ # virtualenv's activate_this.py, whereas sys.executable doesn't.
+ if platform.system() == "Windows":
+ return os.path.join(sys.prefix, "Scripts")
+ else:
+ return os.path.join(sys.prefix, "bin")
+
+
+def get_black_version(binary):
+ """
+ Returns found binary's version
+ """
+ try:
+ output = subprocess.check_output(
+ [binary, "--version"],
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+ except subprocess.CalledProcessError as e:
+ output = e.output
+ try:
+ # Accept `black.EXE, version ...` on Windows.
+ # for old version of black, the output is
+ # black, version 21.4b2
+ # From black 21.11b1, the output is like
+ # black, 21.11b1 (compiled: no)
+ return re.match(r"black.*,( version)? (\S+)", output)[2]
+ except TypeError as e:
+ print("Could not parse the version '{}'".format(output))
+ print("Error: {}".format(e))
+
+
+def parse_issues(config, output, paths, *, log):
+ would_reformat = re.compile("^would reformat (.*)$", re.I)
+ reformatted = re.compile("^reformatted (.*)$", re.I)
+ cannot_reformat = re.compile("^error: cannot format (.*?): (.*)$", re.I)
+ results = []
+ for l in output.split(b"\n"):
+ line = l.decode("utf-8").rstrip("\r\n")
+ if line.startswith("All done!") or line.startswith("Oh no!"):
+ break
+
+ match = would_reformat.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "error"}
+ results.append(result.from_config(config, **res))
+ continue
+
+ match = reformatted.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "warning", "message": "reformatted"}
+ results.append(result.from_config(config, **res))
+ continue
+
+ match = cannot_reformat.match(line)
+ if match:
+ res = {"path": match.group(1), "level": "error", "message": match.group(2)}
+ results.append(result.from_config(config, **res))
+ continue
+
+ log.debug(f"Unhandled line: {line}")
+ return results
+
+
+def run_process(config, cmd):
+ orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ signal.signal(signal.SIGINT, orig)
+ try:
+ output, _ = proc.communicate()
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+
+ return output
+
+
+def setup(root, **lintargs):
+ log = lintargs["log"]
+ virtualenv_bin_path = lintargs.get("virtualenv_bin_path")
+ # Using `which` searches multiple directories and handles `.exe` on Windows.
+ binary = which("black", path=(virtualenv_bin_path, default_bindir()))
+
+ if binary and os.path.exists(binary):
+ binary = mozpath.normsep(binary)
+ log.debug("Looking for black at {}".format(binary))
+ version = get_black_version(binary)
+ versions = [
+ line.split()[0].strip()
+ for line in open(BLACK_REQUIREMENTS_PATH).readlines()
+ if line.startswith("black==")
+ ]
+ if ["black=={}".format(version)] == versions:
+ log.debug("Black is present with expected version {}".format(version))
+ return 0
+ else:
+ log.debug("Black is present but unexpected version {}".format(version))
+
+ log.debug("Black needs to be installed or updated")
+ virtualenv_manager = lintargs["virtualenv_manager"]
+ try:
+ virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH, quiet=True)
+ except subprocess.CalledProcessError:
+ print(BLACK_INSTALL_ERROR)
+ return 1
+
+
+def run_black(config, paths, fix=None, *, log, virtualenv_bin_path):
+ fixed = 0
+ binary = os.path.join(virtualenv_bin_path or default_bindir(), "black")
+
+ log.debug("Black version {}".format(get_black_version(binary)))
+
+ cmd_args = [binary]
+ if not fix:
+ cmd_args.append("--check")
+
+ base_command = cmd_args + paths
+ log.debug("Command: {}".format(" ".join(base_command)))
+ output = parse_issues(config, run_process(config, base_command), paths, log=log)
+
+ # black returns an issue for fixed files as well
+ for eachIssue in output:
+ if eachIssue.message == "reformatted":
+ fixed += 1
+
+ return {"results": output, "fixed": fixed}
+
+
+def lint(paths, config, fix=None, **lintargs):
+ files = list(expand_exclusions(paths, config, lintargs["root"]))
+
+ return run_black(
+ config,
+ files,
+ fix=fix,
+ log=lintargs["log"],
+ virtualenv_bin_path=lintargs.get("virtualenv_bin_path"),
+ )
diff --git a/tools/lint/python/black_requirements.in b/tools/lint/python/black_requirements.in
new file mode 100644
index 0000000000..dfe5a54c7b
--- /dev/null
+++ b/tools/lint/python/black_requirements.in
@@ -0,0 +1,5 @@
+black==23.3.0
+typing-extensions==3.10.0.2
+dataclasses==0.6
+typed-ast==1.4.2; python_version < '3.8'
+pkgutil-resolve-name==1.3.10 ; python_version < '3.9'
diff --git a/tools/lint/python/black_requirements.txt b/tools/lint/python/black_requirements.txt
new file mode 100644
index 0000000000..9e89927775
--- /dev/null
+++ b/tools/lint/python/black_requirements.txt
@@ -0,0 +1,115 @@
+#
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
+#
+# pip-compile --config=pyproject.toml --generate-hashes --output-file=tools/lint/python/black_requirements.txt ./tools/lint/python/black_requirements.in
+#
+black==23.3.0 \
+ --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \
+ --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \
+ --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \
+ --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \
+ --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \
+ --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \
+ --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \
+ --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \
+ --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \
+ --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \
+ --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \
+ --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \
+ --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \
+ --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \
+ --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \
+ --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \
+ --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \
+ --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \
+ --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \
+ --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \
+ --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \
+ --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \
+ --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \
+ --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \
+ --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3
+ # via -r ./tools/lint/python/black_requirements.in
+click==8.0.3 \
+ --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \
+ --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b
+ # via black
+dataclasses==0.6 \
+ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \
+ --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84
+ # via -r ./tools/lint/python/black_requirements.in
+importlib-metadata==6.7.0 \
+ --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
+ --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
+ # via click
+mypy-extensions==0.4.3 \
+ --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
+ --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8
+ # via black
+packaging==23.1 \
+ --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+ --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+ # via black
+pathspec==0.9.0 \
+ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \
+ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1
+ # via black
+pkgutil-resolve-name==1.3.10 ; python_version < "3.9" \
+ --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \
+ --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e
+ # via -r ./tools/lint/python/black_requirements.in
+platformdirs==2.4.0 \
+ --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \
+ --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d
+ # via black
+tomli==1.2.2 \
+ --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \
+ --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade
+ # via black
+typed-ast==1.4.2 ; python_version < "3.8" \
+ --hash=sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1 \
+ --hash=sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d \
+ --hash=sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6 \
+ --hash=sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd \
+ --hash=sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37 \
+ --hash=sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151 \
+ --hash=sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07 \
+ --hash=sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440 \
+ --hash=sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70 \
+ --hash=sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496 \
+ --hash=sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea \
+ --hash=sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400 \
+ --hash=sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc \
+ --hash=sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606 \
+ --hash=sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc \
+ --hash=sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581 \
+ --hash=sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412 \
+ --hash=sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a \
+ --hash=sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2 \
+ --hash=sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787 \
+ --hash=sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f \
+ --hash=sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937 \
+ --hash=sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64 \
+ --hash=sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487 \
+ --hash=sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b \
+ --hash=sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41 \
+ --hash=sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a \
+ --hash=sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3 \
+ --hash=sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166 \
+ --hash=sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10
+ # via
+ # -r ./tools/lint/python/black_requirements.in
+ # black
+typing-extensions==3.10.0.2 \
+ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
+ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \
+ --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34
+ # via
+ # -r ./tools/lint/python/black_requirements.in
+ # black
+ # importlib-metadata
+zipp==3.15.0 \
+ --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
+ --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
+ # via importlib-metadata
diff --git a/tools/lint/python/l10n_lint.py b/tools/lint/python/l10n_lint.py
new file mode 100644
index 0000000000..158cb5f7e6
--- /dev/null
+++ b/tools/lint/python/l10n_lint.py
@@ -0,0 +1,202 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from datetime import datetime, timedelta
+
+from compare_locales import parser
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import l10n_base_reference_and_tests
+from compare_locales.paths import ProjectFiles, TOMLParser
+from mach import util as mach_util
+from mozlint import pathutils, result
+from mozpack import path as mozpath
+from mozversioncontrol import MissingVCSTool
+from mozversioncontrol.repoupdate import update_git_repo, update_mercurial_repo
+
+L10N_SOURCE_NAME = "l10n-source"
+L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
+
+STRINGS_NAME = "gecko-strings"
+STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
+
+PULL_AFTER = timedelta(days=2)
+
+
+# Wrapper to call lint_strings with mozilla-central configuration
+# comm-central defines its own wrapper since comm-central strings are
+# in separate repositories
+def lint(paths, lintconfig, **lintargs):
+ extra_args = lintargs.get("extra_args") or []
+ name = L10N_SOURCE_NAME if "--l10n-git" in extra_args else STRINGS_NAME
+ return lint_strings(name, paths, lintconfig, **lintargs)
+
+
+def lint_strings(name, paths, lintconfig, **lintargs):
+ l10n_base = mach_util.get_state_dir()
+ root = lintargs["root"]
+ exclude = lintconfig.get("exclude")
+ extensions = lintconfig.get("extensions")
+
+ # Load l10n.toml configs
+ l10nconfigs = load_configs(lintconfig, root, l10n_base, name)
+
+ # Check include paths in l10n.yml if it's in our given paths
+ # Only the l10n.yml will show up here, but if the l10n.toml files
+ # change, we also get the l10n.yml as the toml files are listed as
+ # support files.
+ if lintconfig["path"] in paths:
+ results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
+ paths.remove(lintconfig["path"])
+ else:
+ results = []
+
+ all_files = []
+ for p in paths:
+ fp = pathutils.FilterPath(p)
+ if fp.isdir:
+ for _, fileobj in fp.finder:
+ all_files.append(fileobj.path)
+ if fp.isfile:
+ all_files.append(p)
+ # Filter again, our directories might have picked up files the
+ # explicitly excluded in the l10n.yml configuration.
+ # `browser/locales/en-US/firefox-l10n.js` is a good example.
+ all_files, _ = pathutils.filterpaths(
+ lintargs["root"],
+ all_files,
+ lintconfig["include"],
+ exclude=exclude,
+ extensions=extensions,
+ )
+ # These should be excluded in l10n.yml
+ skips = {p for p in all_files if not parser.hasParser(p)}
+ results.extend(
+ result.from_config(
+ lintconfig,
+ level="warning",
+ path=path,
+ message="file format not supported in compare-locales",
+ )
+ for path in skips
+ )
+ all_files = [p for p in all_files if p not in skips]
+ files = ProjectFiles(name, l10nconfigs)
+
+ get_reference_and_tests = l10n_base_reference_and_tests(files)
+
+ linter = MozL10nLinter(lintconfig)
+ results += linter.lint(all_files, get_reference_and_tests)
+ return results
+
+
+# Similar to the lint/lint_strings wrapper setup, for comm-central support.
+def gecko_strings_setup(**lint_args):
+ extra_args = lint_args.get("extra_args") or []
+ if "--l10n-git" in extra_args:
+ return source_repo_setup(L10N_SOURCE_REPO, L10N_SOURCE_NAME)
+ else:
+ return strings_repo_setup(STRINGS_REPO, STRINGS_NAME)
+
+
+def source_repo_setup(repo: str, name: str):
+ gs = mozpath.join(mach_util.get_state_dir(), name)
+ marker = mozpath.join(gs, ".git", "l10n_pull_marker")
+ try:
+ last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+ skip_clone = datetime.now() < last_pull + PULL_AFTER
+ except OSError:
+ skip_clone = False
+ if skip_clone:
+ return
+ try:
+ update_git_repo(repo, gs)
+ except MissingVCSTool:
+ if os.environ.get("MOZ_AUTOMATION"):
+ raise
+ print("warning: l10n linter requires Git but was unable to find 'git'")
+ return 1
+ with open(marker, "w") as fh:
+ fh.flush()
+
+
+def strings_repo_setup(repo: str, name: str):
+ gs = mozpath.join(mach_util.get_state_dir(), name)
+ marker = mozpath.join(gs, ".hg", "l10n_pull_marker")
+ try:
+ last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+ skip_clone = datetime.now() < last_pull + PULL_AFTER
+ except OSError:
+ skip_clone = False
+ if skip_clone:
+ return
+ try:
+ update_mercurial_repo(repo, gs)
+ except MissingVCSTool:
+ if os.environ.get("MOZ_AUTOMATION"):
+ raise
+ print("warning: l10n linter requires Mercurial but was unable to find 'hg'")
+ return 1
+ with open(marker, "w") as fh:
+ fh.flush()
+
+
+def load_configs(lintconfig, root, l10n_base, locale):
+ """Load l10n configuration files specified in the linter configuration."""
+ configs = []
+ env = {"l10n_base": l10n_base}
+ for toml in lintconfig["l10n_configs"]:
+ cfg = TOMLParser().parse(
+ mozpath.join(root, toml), env=env, ignore_missing_includes=True
+ )
+ cfg.set_locales([locale], deep=True)
+ configs.append(cfg)
+ return configs
+
+
+def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
+ """Check l10n.yml config against l10n.toml configs."""
+ reference_paths = set(
+ mozpath.relpath(p["reference"].prefix, lintargs["root"])
+ for project in l10nconfigs
+ for config in project.configs
+ for p in config.paths
+ )
+ # Just check for directories
+ reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
+ missing_in_yml = [
+ refd for refd in reference_dirs if refd not in lintconfig["include"]
+ ]
+ # These might be subdirectories in the config, though
+ missing_in_yml = [
+ d
+ for d in missing_in_yml
+ if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
+ ]
+ if missing_in_yml:
+ dirs = ", ".join(missing_in_yml)
+ return [
+ result.from_config(
+ lintconfig,
+ path=lintconfig["path"],
+ message="l10n.yml out of sync with l10n.toml, add: " + dirs,
+ )
+ ]
+ return []
+
+
+class MozL10nLinter(L10nLinter):
+ """Subclass linter to generate the right result type."""
+
+ def __init__(self, lintconfig):
+ super(MozL10nLinter, self).__init__()
+ self.lintconfig = lintconfig
+
+ def lint(self, files, get_reference_and_tests):
+ return [
+ result.from_config(self.lintconfig, **result_data)
+ for result_data in super(MozL10nLinter, self).lint(
+ files, get_reference_and_tests
+ )
+ ]
diff --git a/tools/lint/python/ruff.py b/tools/lint/python/ruff.py
new file mode 100644
index 0000000000..349320e153
--- /dev/null
+++ b/tools/lint/python/ruff.py
@@ -0,0 +1,176 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+import mozfile
+from mozlint import result
+
+here = os.path.abspath(os.path.dirname(__file__))
+RUFF_REQUIREMENTS_PATH = os.path.join(here, "ruff_requirements.txt")
+
+RUFF_NOT_FOUND = """
+Could not find ruff! Install ruff and try again.
+
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ RUFF_REQUIREMENTS_PATH
+)
+
+
+RUFF_INSTALL_ERROR = """
+Unable to install correct version of ruff!
+Try to install it manually with:
+ $ pip install -U --require-hashes -r {}
+""".strip().format(
+ RUFF_REQUIREMENTS_PATH
+)
+
+
+def default_bindir():
+ # We use sys.prefix to find executables as that gets modified with
+ # virtualenv's activate_this.py, whereas sys.executable doesn't.
+ if platform.system() == "Windows":
+ return os.path.join(sys.prefix, "Scripts")
+ else:
+ return os.path.join(sys.prefix, "bin")
+
+
+def get_ruff_version(binary):
+ """
+ Returns found binary's version
+ """
+ try:
+ output = subprocess.check_output(
+ [binary, "--version"],
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ except subprocess.CalledProcessError as e:
+ output = e.output
+
+ matches = re.match(r"ruff ([0-9\.]+)", output)
+ if matches:
+ return matches[1]
+ print("Error: Could not parse the version '{}'".format(output))
+
+
+def setup(root, log, **lintargs):
+ virtualenv_bin_path = lintargs.get("virtualenv_bin_path")
+ binary = mozfile.which("ruff", path=(virtualenv_bin_path, default_bindir()))
+
+ if binary and os.path.isfile(binary):
+ log.debug(f"Looking for ruff at {binary}")
+ version = get_ruff_version(binary)
+ versions = [
+ line.split()[0].strip()
+ for line in open(RUFF_REQUIREMENTS_PATH).readlines()
+ if line.startswith("ruff==")
+ ]
+ if [f"ruff=={version}"] == versions:
+ log.debug("ruff is present with expected version {}".format(version))
+ return 0
+ else:
+ log.debug("ruff is present but unexpected version {}".format(version))
+
+ virtualenv_manager = lintargs["virtualenv_manager"]
+ try:
+ virtualenv_manager.install_pip_requirements(RUFF_REQUIREMENTS_PATH, quiet=True)
+ except subprocess.CalledProcessError:
+ print(RUFF_INSTALL_ERROR)
+ return 1
+
+
+def run_process(config, cmd, **kwargs):
+ orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+ proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
+ )
+ signal.signal(signal.SIGINT, orig)
+ try:
+ output, _ = proc.communicate()
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+
+ return output
+
+
+def lint(paths, config, log, **lintargs):
+ fixed = 0
+ results = []
+
+ if not paths:
+ return {"results": results, "fixed": fixed}
+
+ # Currently ruff only lints non `.py` files if they are explicitly passed
+ # in. So we need to find any non-py files manually. This can be removed
+ # after https://github.com/charliermarsh/ruff/issues/3410 is fixed.
+ exts = [e for e in config["extensions"] if e != "py"]
+ non_py_files = []
+ for path in paths:
+ p = Path(path)
+ if not p.is_dir():
+ continue
+ for ext in exts:
+ non_py_files.extend([str(f) for f in p.glob(f"**/*.{ext}")])
+
+ args = ["ruff", "check", "--force-exclude"] + paths + non_py_files
+
+ if config["exclude"]:
+ args.append(f"--extend-exclude={','.join(config['exclude'])}")
+
+ process_kwargs = {"processStderrLine": lambda line: log.debug(line)}
+
+ warning_rules = set(config.get("warning-rules", []))
+ if lintargs.get("fix"):
+ # Do a first pass with --fix-only as the json format doesn't return the
+ # number of fixed issues.
+ fix_args = args + ["--fix-only"]
+
+ # Don't fix warnings to limit unrelated changes sneaking into patches.
+ fix_args.append(f"--extend-ignore={','.join(warning_rules)}")
+ output = run_process(config, fix_args, **process_kwargs)
+ matches = re.match(r"Fixed (\d+) errors?.", output)
+ if matches:
+ fixed = int(matches[1])
+
+ log.debug(f"Running with args: {args}")
+
+ output = run_process(config, args + ["--format=json"], **process_kwargs)
+ if not output:
+ return []
+
+ try:
+ issues = json.loads(output)
+ except json.JSONDecodeError:
+ log.error(f"could not parse output: {output}")
+ return []
+
+ for issue in issues:
+ res = {
+ "path": issue["filename"],
+ "lineno": issue["location"]["row"],
+ "column": issue["location"]["column"],
+ "lineoffset": issue["end_location"]["row"] - issue["location"]["row"],
+ "message": issue["message"],
+ "rule": issue["code"],
+ "level": "error",
+ }
+ if any(issue["code"].startswith(w) for w in warning_rules):
+ res["level"] = "warning"
+
+ if issue["fix"]:
+ res["hint"] = issue["fix"]["message"]
+
+ results.append(result.from_config(config, **res))
+
+ return {"results": results, "fixed": fixed}
diff --git a/tools/lint/python/ruff_requirements.in b/tools/lint/python/ruff_requirements.in
new file mode 100644
index 0000000000..84b2a3cfd0
--- /dev/null
+++ b/tools/lint/python/ruff_requirements.in
@@ -0,0 +1,2 @@
+ruff
+pkgutil-resolve-name==1.3.10 ; python_version < '3.9'
diff --git a/tools/lint/python/ruff_requirements.txt b/tools/lint/python/ruff_requirements.txt
new file mode 100644
index 0000000000..a9db7c1097
--- /dev/null
+++ b/tools/lint/python/ruff_requirements.txt
@@ -0,0 +1,29 @@
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+# pip-compile --generate-hashes --output-file=tools/lint/python/ruff_requirements.txt tools/lint/python/ruff_requirements.in
+#
+pkgutil-resolve-name==1.3.10 ; python_version < "3.9" \
+ --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \
+ --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e
+ # via -r tools/lint/python/ruff_requirements.in
+ruff==0.0.254 \
+ --hash=sha256:059a380c08e849b6f312479b18cc63bba2808cff749ad71555f61dd930e3c9a2 \
+ --hash=sha256:09c764bc2bd80c974f7ce1f73a46092c286085355a5711126af351b9ae4bea0c \
+ --hash=sha256:0deb1d7226ea9da9b18881736d2d96accfa7f328c67b7410478cc064ad1fa6aa \
+ --hash=sha256:0eb66c9520151d3bd950ea43b3a088618a8e4e10a5014a72687881e6f3606312 \
+ --hash=sha256:27d39d697fdd7df1f2a32c1063756ee269ad8d5345c471ee3ca450636d56e8c6 \
+ --hash=sha256:2fc21d060a3197ac463596a97d9b5db2d429395938b270ded61dd60f0e57eb21 \
+ --hash=sha256:688379050ae05394a6f9f9c8471587fd5dcf22149bd4304a4ede233cc4ef89a1 \
+ --hash=sha256:8deba44fd563361c488dedec90dc330763ee0c01ba54e17df54ef5820079e7e0 \
+ --hash=sha256:ac1429be6d8bd3db0bf5becac3a38bd56f8421447790c50599cd90fd53417ec4 \
+ --hash=sha256:b3f15d5d033fd3dcb85d982d6828ddab94134686fac2c02c13a8822aa03e1321 \
+ --hash=sha256:b435afc4d65591399eaf4b2af86e441a71563a2091c386cadf33eaa11064dc09 \
+ --hash=sha256:c38291bda4c7b40b659e8952167f386e86ec29053ad2f733968ff1d78b4c7e15 \
+ --hash=sha256:d4385cdd30153b7aa1d8f75dfd1ae30d49c918ead7de07e69b7eadf0d5538a1f \
+ --hash=sha256:dd58c500d039fb381af8d861ef456c3e94fd6855c3d267d6c6718c9a9fe07be0 \
+ --hash=sha256:e15742df0f9a3615fbdc1ee9a243467e97e75bf88f86d363eee1ed42cedab1ec \
+ --hash=sha256:ef20bf798ffe634090ad3dc2e8aa6a055f08c448810a2f800ab716cc18b80107 \
+ --hash=sha256:f70dc93bc9db15cccf2ed2a831938919e3e630993eeea6aba5c84bc274237885
+ # via -r tools/lint/python/ruff_requirements.in