diff options
Diffstat (limited to 'tools/lint/python')
| -rw-r--r-- | tools/lint/python/__init__.py | 3 | ||||
| -rw-r--r-- | tools/lint/python/black.py | 179 | ||||
| -rw-r--r-- | tools/lint/python/black_requirements.in | 4 | ||||
| -rw-r--r-- | tools/lint/python/black_requirements.txt | 117 | ||||
| -rw-r--r-- | tools/lint/python/l10n_lint.py | 171 | ||||
| -rw-r--r-- | tools/lint/python/ruff.py | 187 | ||||
| -rw-r--r-- | tools/lint/python/ruff_requirements.in | 1 | ||||
| -rw-r--r-- | tools/lint/python/ruff_requirements.txt | 25 |
8 files changed, 687 insertions, 0 deletions
diff --git a/tools/lint/python/__init__.py b/tools/lint/python/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/lint/python/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/lint/python/black.py b/tools/lint/python/black.py new file mode 100644 index 0000000000..ac975e62a2 --- /dev/null +++ b/tools/lint/python/black.py @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import platform +import re +import signal +import subprocess +import sys + +import mozpack.path as mozpath +from mozfile import which +from mozlint import result +from mozlint.pathutils import expand_exclusions +from mozprocess import ProcessHandler + +here = os.path.abspath(os.path.dirname(__file__)) +BLACK_REQUIREMENTS_PATH = os.path.join(here, "black_requirements.txt") + +BLACK_INSTALL_ERROR = """ +Unable to install correct version of black +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + BLACK_REQUIREMENTS_PATH +) + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +def get_black_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + try: + # Accept `black.EXE, version ...` on Windows. + # for old version of black, the output is + # black, version 21.4b2 + # From black 21.11b1, the output is like + # black, 21.11b1 (compiled: no) + return re.match(r"black.*,( version)? (\S+)", output)[2] + except TypeError as e: + print("Could not parse the version '{}'".format(output)) + print("Error: {}".format(e)) + + +def parse_issues(config, output, paths, *, log): + would_reformat = re.compile("^would reformat (.*)$", re.I) + reformatted = re.compile("^reformatted (.*)$", re.I) + cannot_reformat = re.compile("^error: cannot format (.*?): (.*)$", re.I) + results = [] + for line in output: + line = line.decode("utf-8") + if line.startswith("All done!") or line.startswith("Oh no!"): + break + + match = would_reformat.match(line) + if match: + res = {"path": match.group(1), "level": "error"} + results.append(result.from_config(config, **res)) + continue + + match = reformatted.match(line) + if match: + res = {"path": match.group(1), "level": "warning", "message": "reformatted"} + results.append(result.from_config(config, **res)) + continue + + match = cannot_reformat.match(line) + if match: + res = {"path": match.group(1), "level": "error", "message": match.group(2)} + results.append(result.from_config(config, **res)) + continue + + log.debug(f"Unhandled line: {line}") + return results + + +class BlackProcess(ProcessHandler): + def __init__(self, config, *args, **kwargs): + self.config = config + kwargs["stream"] = False + ProcessHandler.__init__(self, *args, **kwargs) + + def run(self, *args, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + ProcessHandler.run(self, *args, **kwargs) + signal.signal(signal.SIGINT, orig) + + +def run_process(config, cmd): + proc = BlackProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return proc.output + + +def setup(root, **lintargs): + log = lintargs["log"] + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + # Using `which` searches multiple directories and handles `.exe` on Windows. + binary = which("black", path=(virtualenv_bin_path, default_bindir())) + + if binary and os.path.exists(binary): + binary = mozpath.normsep(binary) + log.debug("Looking for black at {}".format(binary)) + version = get_black_version(binary) + versions = [ + line.split()[0].strip() + for line in open(BLACK_REQUIREMENTS_PATH).readlines() + if line.startswith("black==") + ] + if ["black=={}".format(version)] == versions: + log.debug("Black is present with expected version {}".format(version)) + return 0 + else: + log.debug("Black is present but unexpected version {}".format(version)) + + log.debug("Black needs to be installed or updated") + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH, quiet=True) + except subprocess.CalledProcessError: + print(BLACK_INSTALL_ERROR) + return 1 + + +def run_black(config, paths, fix=None, *, log, virtualenv_bin_path): + fixed = 0 + binary = os.path.join(virtualenv_bin_path or default_bindir(), "black") + + log.debug("Black version {}".format(get_black_version(binary))) + + cmd_args = [binary] + if not fix: + cmd_args.append("--check") + + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(base_command))) + output = parse_issues(config, run_process(config, base_command), paths, log=log) + + # black returns an issue for fixed files as well + for eachIssue in output: + if eachIssue.message == "reformatted": + fixed += 1 + + return {"results": output, "fixed": fixed} + + +def lint(paths, config, fix=None, **lintargs): + files = list(expand_exclusions(paths, config, lintargs["root"])) + + return run_black( + config, + files, + fix=fix, + log=lintargs["log"], + virtualenv_bin_path=lintargs.get("virtualenv_bin_path"), + ) diff --git a/tools/lint/python/black_requirements.in b/tools/lint/python/black_requirements.in new file mode 100644 index 0000000000..e5efa47492 --- /dev/null +++ b/tools/lint/python/black_requirements.in @@ -0,0 +1,4 @@ +black==21.11b1 +typing-extensions==3.10.0.2 +dataclasses==0.6 + diff --git a/tools/lint/python/black_requirements.txt b/tools/lint/python/black_requirements.txt new file mode 100644 index 0000000000..ffa9ef3564 --- /dev/null +++ b/tools/lint/python/black_requirements.txt @@ -0,0 +1,117 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file=tools/lint/python/black_requirements.txt tools/lint/python/black_requirements.in +# +black==21.11b1 \ + --hash=sha256:802c6c30b637b28645b7fde282ed2569c0cd777dbe493a41b6a03c1d903f99ac \ + --hash=sha256:a042adbb18b3262faad5aff4e834ff186bb893f95ba3a8013f09de1e5569def2 + # via -r tools/lint/python/black_requirements.in +click==8.0.3 \ + --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ + --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b + # via black +dataclasses==0.6 \ + --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ + --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 + # via -r tools/lint/python/black_requirements.in +mypy-extensions==0.4.3 \ + --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ + --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 + # via black +pathspec==0.9.0 \ + --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ + --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 + # via black +platformdirs==2.4.0 \ + --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ + --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d + # via black +regex==2021.11.10 \ + --hash=sha256:05b7d6d7e64efe309972adab77fc2af8907bb93217ec60aa9fe12a0dad35874f \ + --hash=sha256:0617383e2fe465732af4509e61648b77cbe3aee68b6ac8c0b6fe934db90be5cc \ + --hash=sha256:07856afef5ffcc052e7eccf3213317fbb94e4a5cd8177a2caa69c980657b3cb4 \ + --hash=sha256:162abfd74e88001d20cb73ceaffbfe601469923e875caf9118333b1a4aaafdc4 \ + --hash=sha256:2207ae4f64ad3af399e2d30dde66f0b36ae5c3129b52885f1bffc2f05ec505c8 \ + --hash=sha256:30ab804ea73972049b7a2a5c62d97687d69b5a60a67adca07eb73a0ddbc9e29f \ + --hash=sha256:3b5df18db1fccd66de15aa59c41e4f853b5df7550723d26aa6cb7f40e5d9da5a \ + --hash=sha256:3c5fb32cc6077abad3bbf0323067636d93307c9fa93e072771cf9a64d1c0f3ef \ + --hash=sha256:416c5f1a188c91e3eb41e9c8787288e707f7d2ebe66e0a6563af280d9b68478f \ + --hash=sha256:432bd15d40ed835a51617521d60d0125867f7b88acf653e4ed994a1f8e4995dc \ + --hash=sha256:4aaa4e0705ef2b73dd8e36eeb4c868f80f8393f5f4d855e94025ce7ad8525f50 \ + --hash=sha256:537ca6a3586931b16a85ac38c08cc48f10fc870a5b25e51794c74df843e9966d \ + --hash=sha256:53db2c6be8a2710b359bfd3d3aa17ba38f8aa72a82309a12ae99d3c0c3dcd74d \ + --hash=sha256:5537f71b6d646f7f5f340562ec4c77b6e1c915f8baae822ea0b7e46c1f09b733 \ + --hash=sha256:6650f16365f1924d6014d2ea770bde8555b4a39dc9576abb95e3cd1ff0263b36 \ + --hash=sha256:666abff54e474d28ff42756d94544cdfd42e2ee97065857413b72e8a2d6a6345 \ + --hash=sha256:68a067c11463de2a37157930d8b153005085e42bcb7ad9ca562d77ba7d1404e0 \ + --hash=sha256:780b48456a0f0ba4d390e8b5f7c661fdd218934388cde1a974010a965e200e12 \ + --hash=sha256:788aef3549f1924d5c38263104dae7395bf020a42776d5ec5ea2b0d3d85d6646 \ + --hash=sha256:7ee1227cf08b6716c85504aebc49ac827eb88fcc6e51564f010f11a406c0a667 \ + --hash=sha256:7f301b11b9d214f83ddaf689181051e7f48905568b0c7017c04c06dfd065e244 \ + --hash=sha256:83ee89483672b11f8952b158640d0c0ff02dc43d9cb1b70c1564b49abe92ce29 \ + --hash=sha256:85bfa6a5413be0ee6c5c4a663668a2cad2cbecdee367630d097d7823041bdeec \ + --hash=sha256:9345b6f7ee578bad8e475129ed40123d265464c4cfead6c261fd60fc9de00bcf \ + --hash=sha256:93a5051fcf5fad72de73b96f07d30bc29665697fb8ecdfbc474f3452c78adcf4 \ + --hash=sha256:962b9a917dd7ceacbe5cd424556914cb0d636001e393b43dc886ba31d2a1e449 \ + --hash=sha256:98ba568e8ae26beb726aeea2273053c717641933836568c2a0278a84987b2a1a \ + --hash=sha256:a3feefd5e95871872673b08636f96b61ebef62971eab044f5124fb4dea39919d \ + --hash=sha256:b43c2b8a330a490daaef5a47ab114935002b13b3f9dc5da56d5322ff218eeadb \ + --hash=sha256:b483c9d00a565633c87abd0aaf27eb5016de23fed952e054ecc19ce32f6a9e7e \ + --hash=sha256:ba05430e819e58544e840a68b03b28b6d328aff2e41579037e8bab7653b37d83 \ + --hash=sha256:ca5f18a75e1256ce07494e245cdb146f5a9267d3c702ebf9b65c7f8bd843431e \ + --hash=sha256:d5ca078bb666c4a9d1287a379fe617a6dccd18c3e8a7e6c7e1eb8974330c626a \ + --hash=sha256:da1a90c1ddb7531b1d5ff1e171b4ee61f6345119be7351104b67ff413843fe94 \ + --hash=sha256:dba70f30fd81f8ce6d32ddeef37d91c8948e5d5a4c63242d16a2b2df8143aafc \ + --hash=sha256:dd33eb9bdcfbabab3459c9ee651d94c842bc8a05fabc95edf4ee0c15a072495e \ + --hash=sha256:e0538c43565ee6e703d3a7c3bdfe4037a5209250e8502c98f20fea6f5fdf2965 \ + --hash=sha256:e1f54b9b4b6c53369f40028d2dd07a8c374583417ee6ec0ea304e710a20f80a0 \ + --hash=sha256:e32d2a2b02ccbef10145df9135751abea1f9f076e67a4e261b05f24b94219e36 \ + --hash=sha256:e71255ba42567d34a13c03968736c5d39bb4a97ce98188fafb27ce981115beec \ + --hash=sha256:ed2e07c6a26ed4bea91b897ee2b0835c21716d9a469a96c3e878dc5f8c55bb23 \ + --hash=sha256:eef2afb0fd1747f33f1ee3e209bce1ed582d1896b240ccc5e2697e3275f037c7 \ + --hash=sha256:f23222527b307970e383433daec128d769ff778d9b29343fb3496472dc20dabe \ + --hash=sha256:f341ee2df0999bfdf7a95e448075effe0db212a59387de1a70690e4acb03d4c6 \ + --hash=sha256:f7f325be2804246a75a4f45c72d4ce80d2443ab815063cdf70ee8fb2ca59ee1b \ + --hash=sha256:f8af619e3be812a2059b212064ea7a640aff0568d972cd1b9e920837469eb3cb \ + --hash=sha256:fa8c626d6441e2d04b6ee703ef2d1e17608ad44c7cb75258c09dd42bacdfc64b \ + --hash=sha256:fbb9dc00e39f3e6c0ef48edee202f9520dafb233e8b51b06b8428cfcb92abd30 \ + --hash=sha256:fff55f3ce50a3ff63ec8e2a8d3dd924f1941b250b0aac3d3d42b687eeff07a8e + # via black +tomli==1.2.2 \ + --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ + --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade + # via black +typed-ast==1.5.4 \ + --hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \ + --hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \ + --hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \ + --hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \ + --hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \ + --hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \ + --hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \ + --hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \ + --hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \ + --hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \ + --hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \ + --hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \ + --hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \ + --hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \ + --hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \ + --hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \ + --hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \ + --hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \ + --hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \ + --hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \ + --hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \ + --hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \ + --hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \ + --hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66 +typing-extensions==3.10.0.2 \ + --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ + --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ + --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 + # via + # -r tools/lint/python/black_requirements.in + # black diff --git a/tools/lint/python/l10n_lint.py b/tools/lint/python/l10n_lint.py new file mode 100644 index 0000000000..ef3269ef2a --- /dev/null +++ b/tools/lint/python/l10n_lint.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from datetime import datetime, timedelta + +import mozversioncontrol.repoupdate +from compare_locales import parser +from compare_locales.lint.linter import L10nLinter +from compare_locales.lint.util import l10n_base_reference_and_tests +from compare_locales.paths import ProjectFiles, TOMLParser +from mach import util as mach_util +from mozlint import pathutils, result +from mozpack import path as mozpath + +LOCALE = "gecko-strings" +STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings" + +PULL_AFTER = timedelta(days=2) + +# Wrapper to call lint_strings with mozilla-central configuration +# comm-central defines its own wrapper since comm-central strings are +# in separate repositories +def lint(paths, lintconfig, **lintargs): + return lint_strings(LOCALE, paths, lintconfig, **lintargs) + + +def lint_strings(locale, paths, lintconfig, **lintargs): + l10n_base = mach_util.get_state_dir() + root = lintargs["root"] + exclude = lintconfig.get("exclude") + extensions = lintconfig.get("extensions") + + # Load l10n.toml configs + l10nconfigs = load_configs(lintconfig, root, l10n_base, locale) + + # Check include paths in l10n.yml if it's in our given paths + # Only the l10n.yml will show up here, but if the l10n.toml files + # change, we also get the l10n.yml as the toml files are listed as + # support files. + if lintconfig["path"] in paths: + results = validate_linter_includes(lintconfig, l10nconfigs, lintargs) + paths.remove(lintconfig["path"]) + else: + results = [] + + all_files = [] + for p in paths: + fp = pathutils.FilterPath(p) + if fp.isdir: + for _, fileobj in fp.finder: + all_files.append(fileobj.path) + if fp.isfile: + all_files.append(p) + # Filter again, our directories might have picked up files the + # explicitly excluded in the l10n.yml configuration. + # `browser/locales/en-US/firefox-l10n.js` is a good example. + all_files, _ = pathutils.filterpaths( + lintargs["root"], + all_files, + lintconfig["include"], + exclude=exclude, + extensions=extensions, + ) + # These should be excluded in l10n.yml + skips = {p for p in all_files if not parser.hasParser(p)} + results.extend( + result.from_config( + lintconfig, + level="warning", + path=path, + message="file format not supported in compare-locales", + ) + for path in skips + ) + all_files = [p for p in all_files if p not in skips] + files = ProjectFiles(locale, l10nconfigs) + + get_reference_and_tests = l10n_base_reference_and_tests(files) + + linter = MozL10nLinter(lintconfig) + results += linter.lint(all_files, get_reference_and_tests) + return results + + +# Similar to the lint/lint_strings wrapper setup, for comm-central support. +def gecko_strings_setup(**lint_args): + return strings_repo_setup(STRINGS_REPO, LOCALE) + + +def strings_repo_setup(repo, locale): + gs = mozpath.join(mach_util.get_state_dir(), locale) + marker = mozpath.join(gs, ".hg", "l10n_pull_marker") + try: + last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime) + skip_clone = datetime.now() < last_pull + PULL_AFTER + except OSError: + skip_clone = False + if skip_clone: + return + try: + hg = mozversioncontrol.get_tool_path("hg") + except mozversioncontrol.MissingVCSTool: + if os.environ.get("MOZ_AUTOMATION"): + raise + print("warning: l10n linter requires Mercurial but was unable to find 'hg'") + return 1 + mozversioncontrol.repoupdate.update_mercurial_repo(hg, repo, gs) + with open(marker, "w") as fh: + fh.flush() + + +def load_configs(lintconfig, root, l10n_base, locale): + """Load l10n configuration files specified in the linter configuration.""" + configs = [] + env = {"l10n_base": l10n_base} + for toml in lintconfig["l10n_configs"]: + cfg = TOMLParser().parse( + mozpath.join(root, toml), env=env, ignore_missing_includes=True + ) + cfg.set_locales([locale], deep=True) + configs.append(cfg) + return configs + + +def validate_linter_includes(lintconfig, l10nconfigs, lintargs): + """Check l10n.yml config against l10n.toml configs.""" + reference_paths = set( + mozpath.relpath(p["reference"].prefix, lintargs["root"]) + for project in l10nconfigs + for config in project.configs + for p in config.paths + ) + # Just check for directories + reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p)) + missing_in_yml = [ + refd for refd in reference_dirs if refd not in lintconfig["include"] + ] + # These might be subdirectories in the config, though + missing_in_yml = [ + d + for d in missing_in_yml + if not any(d.startswith(parent + "/") for parent in lintconfig["include"]) + ] + if missing_in_yml: + dirs = ", ".join(missing_in_yml) + return [ + result.from_config( + lintconfig, + path=lintconfig["path"], + message="l10n.yml out of sync with l10n.toml, add: " + dirs, + ) + ] + return [] + + +class MozL10nLinter(L10nLinter): + """Subclass linter to generate the right result type.""" + + def __init__(self, lintconfig): + super(MozL10nLinter, self).__init__() + self.lintconfig = lintconfig + + def lint(self, files, get_reference_and_tests): + return [ + result.from_config(self.lintconfig, **result_data) + for result_data in super(MozL10nLinter, self).lint( + files, get_reference_and_tests + ) + ] diff --git a/tools/lint/python/ruff.py b/tools/lint/python/ruff.py new file mode 100644 index 0000000000..d8096199f3 --- /dev/null +++ b/tools/lint/python/ruff.py @@ -0,0 +1,187 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import re +import signal +import subprocess +import sys +from pathlib import Path + +import mozfile +from mozlint import result +from mozprocess.processhandler import ProcessHandler + +here = os.path.abspath(os.path.dirname(__file__)) +RUFF_REQUIREMENTS_PATH = os.path.join(here, "ruff_requirements.txt") + +RUFF_NOT_FOUND = """ +Could not find ruff! Install ruff and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +RUFF_INSTALL_ERROR = """ +Unable to install correct version of ruff! +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +def get_ruff_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + + matches = re.match(r"ruff ([0-9\.]+)", output) + if matches: + return matches[1] + print("Error: Could not parse the version '{}'".format(output)) + + +def setup(root, log, **lintargs): + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + binary = mozfile.which("ruff", path=(virtualenv_bin_path, default_bindir())) + + if binary and os.path.isfile(binary): + log.debug(f"Looking for ruff at {binary}") + version = get_ruff_version(binary) + versions = [ + line.split()[0].strip() + for line in open(RUFF_REQUIREMENTS_PATH).readlines() + if line.startswith("ruff==") + ] + if [f"ruff=={version}"] == versions: + log.debug("ruff is present with expected version {}".format(version)) + return 0 + else: + log.debug("ruff is present but unexpected version {}".format(version)) + + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements(RUFF_REQUIREMENTS_PATH, quiet=True) + except subprocess.CalledProcessError: + print(RUFF_INSTALL_ERROR) + return 1 + + +class RuffProcess(ProcessHandler): + def __init__(self, config, *args, **kwargs): + self.config = config + self.stderr = [] + kwargs["stream"] = False + kwargs["universal_newlines"] = True + ProcessHandler.__init__(self, *args, **kwargs) + + def run(self, *args, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + ProcessHandler.run(self, *args, **kwargs) + signal.signal(signal.SIGINT, orig) + + +def run_process(config, cmd, **kwargs): + proc = RuffProcess(config, cmd, **kwargs) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return "\n".join(proc.output) + + +def lint(paths, config, log, **lintargs): + fixed = 0 + results = [] + + if not paths: + return {"results": results, "fixed": fixed} + + # Currently ruff only lints non `.py` files if they are explicitly passed + # in. So we need to find any non-py files manually. This can be removed + # after https://github.com/charliermarsh/ruff/issues/3410 is fixed. + exts = [e for e in config["extensions"] if e != "py"] + non_py_files = [] + for path in paths: + p = Path(path) + if not p.is_dir(): + continue + for ext in exts: + non_py_files.extend([str(f) for f in p.glob(f"**/*.{ext}")]) + + args = ["ruff", "check", "--force-exclude"] + paths + non_py_files + + if config["exclude"]: + args.append(f"--extend-exclude={','.join(config['exclude'])}") + + process_kwargs = {"processStderrLine": lambda line: log.debug(line)} + + warning_rules = set(config.get("warning-rules", [])) + if lintargs.get("fix"): + # Do a first pass with --fix-only as the json format doesn't return the + # number of fixed issues. + fix_args = args + ["--fix-only"] + + # Don't fix warnings to limit unrelated changes sneaking into patches. + fix_args.append(f"--extend-ignore={','.join(warning_rules)}") + output = run_process(config, fix_args, **process_kwargs) + matches = re.match(r"Fixed (\d+) errors?.", output) + if matches: + fixed = int(matches[1]) + + log.debug(f"Running with args: {args}") + + output = run_process(config, args + ["--format=json"], **process_kwargs) + if not output: + return [] + + try: + issues = json.loads(output) + except json.JSONDecodeError: + log.error(f"could not parse output: {output}") + return [] + + for issue in issues: + res = { + "path": issue["filename"], + "lineno": issue["location"]["row"], + "column": issue["location"]["column"], + "lineoffset": issue["end_location"]["row"] - issue["location"]["row"], + "message": issue["message"], + "rule": issue["code"], + "level": "error", + } + if any(issue["code"].startswith(w) for w in warning_rules): + res["level"] = "warning" + + if issue["fix"]: + res["hint"] = issue["fix"]["message"] + + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/python/ruff_requirements.in b/tools/lint/python/ruff_requirements.in new file mode 100644 index 0000000000..af3ee57638 --- /dev/null +++ b/tools/lint/python/ruff_requirements.in @@ -0,0 +1 @@ +ruff diff --git a/tools/lint/python/ruff_requirements.txt b/tools/lint/python/ruff_requirements.txt new file mode 100644 index 0000000000..1c8943c26b --- /dev/null +++ b/tools/lint/python/ruff_requirements.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --generate-hashes ruff_requirements.in +# +ruff==0.0.254 \ + --hash=sha256:059a380c08e849b6f312479b18cc63bba2808cff749ad71555f61dd930e3c9a2 \ + --hash=sha256:09c764bc2bd80c974f7ce1f73a46092c286085355a5711126af351b9ae4bea0c \ + --hash=sha256:0deb1d7226ea9da9b18881736d2d96accfa7f328c67b7410478cc064ad1fa6aa \ + --hash=sha256:0eb66c9520151d3bd950ea43b3a088618a8e4e10a5014a72687881e6f3606312 \ + --hash=sha256:27d39d697fdd7df1f2a32c1063756ee269ad8d5345c471ee3ca450636d56e8c6 \ + --hash=sha256:2fc21d060a3197ac463596a97d9b5db2d429395938b270ded61dd60f0e57eb21 \ + --hash=sha256:688379050ae05394a6f9f9c8471587fd5dcf22149bd4304a4ede233cc4ef89a1 \ + --hash=sha256:8deba44fd563361c488dedec90dc330763ee0c01ba54e17df54ef5820079e7e0 \ + --hash=sha256:ac1429be6d8bd3db0bf5becac3a38bd56f8421447790c50599cd90fd53417ec4 \ + --hash=sha256:b3f15d5d033fd3dcb85d982d6828ddab94134686fac2c02c13a8822aa03e1321 \ + --hash=sha256:b435afc4d65591399eaf4b2af86e441a71563a2091c386cadf33eaa11064dc09 \ + --hash=sha256:c38291bda4c7b40b659e8952167f386e86ec29053ad2f733968ff1d78b4c7e15 \ + --hash=sha256:d4385cdd30153b7aa1d8f75dfd1ae30d49c918ead7de07e69b7eadf0d5538a1f \ + --hash=sha256:dd58c500d039fb381af8d861ef456c3e94fd6855c3d267d6c6718c9a9fe07be0 \ + --hash=sha256:e15742df0f9a3615fbdc1ee9a243467e97e75bf88f86d363eee1ed42cedab1ec \ + --hash=sha256:ef20bf798ffe634090ad3dc2e8aa6a055f08c448810a2f800ab716cc18b80107 \ + --hash=sha256:f70dc93bc9db15cccf2ed2a831938919e3e630993eeea6aba5c84bc274237885 + # via -r ruff_requirements.in |