diff options
Diffstat (limited to '')
| -rw-r--r-- | tools/lint/python/flake8.py | 215 |
1 files changed, 215 insertions, 0 deletions
diff --git a/tools/lint/python/flake8.py b/tools/lint/python/flake8.py new file mode 100644 index 0000000000..88fec87822 --- /dev/null +++ b/tools/lint/python/flake8.py @@ -0,0 +1,215 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import subprocess +import sys + +import mozfile +import mozpack.path as mozpath +from mozlint import result +from mozlint.pathutils import expand_exclusions + +here = os.path.abspath(os.path.dirname(__file__)) +FLAKE8_REQUIREMENTS_PATH = os.path.join(here, "flake8_requirements.txt") + +FLAKE8_NOT_FOUND = """ +Could not find flake8! Install flake8 and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + FLAKE8_REQUIREMENTS_PATH +) + + +FLAKE8_INSTALL_ERROR = """ +Unable to install correct version of flake8 +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + FLAKE8_REQUIREMENTS_PATH +) + +LINE_OFFSETS = { + # continuation line under-indented for hanging indent + "E121": (-1, 2), + # continuation line missing indentation or outdented + "E122": (-1, 2), + # continuation line over-indented for hanging indent + "E126": (-1, 2), + # continuation line over-indented for visual indent + "E127": (-1, 2), + # continuation line under-indented for visual indent + "E128": (-1, 2), + # continuation line unaligned for hanging indend + "E131": (-1, 2), + # expected 1 blank line, found 0 + "E301": (-1, 2), + # expected 2 blank lines, found 1 + "E302": (-2, 3), +} +"""Maps a flake8 error to a lineoffset tuple. + +The offset is of the form (lineno_offset, num_lines) and is passed +to the lineoffset property of an `Issue`. +""" + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +class NothingToLint(Exception): + """Exception used to bail out of flake8's internals if all the specified + files were excluded. + """ + + +def setup(root, **lintargs): + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements( + FLAKE8_REQUIREMENTS_PATH, quiet=True + ) + except subprocess.CalledProcessError: + print(FLAKE8_INSTALL_ERROR) + return 1 + + +def lint(paths, config, **lintargs): + + root = lintargs["root"] + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + config_path = os.path.join(root, ".flake8") + + results = run(paths, config, **lintargs) + fixed = 0 + + if lintargs.get("fix"): + # fix and run again to count remaining issues + fixed = len(results) + fix_cmd = [ + os.path.join(virtualenv_bin_path or default_bindir(), "autopep8"), + "--global-config", + config_path, + "--in-place", + "--recursive", + ] + + if config.get("exclude"): + fix_cmd.extend(["--exclude", ",".join(config["exclude"])]) + + subprocess.call(fix_cmd + paths) + + results = run(paths, config, **lintargs) + + fixed = fixed - len(results) + + return {"results": results, "fixed": fixed} + + +def run(paths, config, **lintargs): + from flake8 import __version__ as flake8_version + from flake8.main.application import Application + + log = lintargs["log"] + root = lintargs["root"] + config_path = os.path.join(root, ".flake8") + + # Run flake8. + app = Application() + log.debug("flake8 version={}".format(flake8_version)) + + output_file = mozfile.NamedTemporaryFile(mode="r") + flake8_cmd = [ + "--config", + config_path, + "--output-file", + output_file.name, + "--format", + '{"path":"%(path)s","lineno":%(row)s,' + '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', + "--filename", + ",".join(["*.{}".format(e) for e in config["extensions"]]), + ] + log.debug("Command: {}".format(" ".join(flake8_cmd))) + + orig_make_file_checker_manager = app.make_file_checker_manager + + def wrap_make_file_checker_manager(self): + """Flake8 is very inefficient when it comes to applying exclusion + rules, using `expand_exclusions` to turn directories into a list of + relevant python files is an order of magnitude faster. + + Hooking into flake8 here also gives us a convenient place to merge the + `exclude` rules specified in the root .flake8 with the ones added by + tools/lint/mach_commands.py. + """ + # Ignore exclude rules if `--no-filter` was passed in. + config.setdefault("exclude", []) + if lintargs.get("use_filters", True): + config["exclude"].extend(map(mozpath.normpath, self.options.exclude)) + + # Since we use the root .flake8 file to store exclusions, we haven't + # properly filtered the paths through mozlint's `filterpaths` function + # yet. This mimics that though there could be other edge cases that are + # different. Maybe we should call `filterpaths` directly, though for + # now that doesn't appear to be necessary. + filtered = [ + p for p in paths if not any(p.startswith(e) for e in config["exclude"]) + ] + + self.options.filenames = self.options.filenames + list( + expand_exclusions(filtered, config, root) + ) + + if not self.options.filenames: + raise NothingToLint + return orig_make_file_checker_manager() + + app.make_file_checker_manager = wrap_make_file_checker_manager.__get__( + app, Application + ) + + # Make sure to run from repository root so exclusions are joined to the + # repository root and not the current working directory. + oldcwd = os.getcwd() + os.chdir(root) + try: + app.run(flake8_cmd) + except NothingToLint: + pass + finally: + os.chdir(oldcwd) + + results = [] + + WARNING_RULES = set(config.get("warning-rules", [])) + + def process_line(line): + # Escape slashes otherwise JSON conversion will not work + line = line.replace("\\", "\\\\") + try: + res = json.loads(line) + except ValueError: + print("Non JSON output from linter, will not be processed: {}".format(line)) + return + + if res.get("code") in LINE_OFFSETS: + res["lineoffset"] = LINE_OFFSETS[res["code"]] + + if res["rule"] in WARNING_RULES: + res["level"] = "warning" + + results.append(result.from_config(config, **res)) + + list(map(process_line, output_file.readlines())) + return results |