diff options
Diffstat (limited to 'python/mozlint/mozlint')
-rw-r--r-- | python/mozlint/mozlint/__init__.py | 7 | ||||
-rw-r--r-- | python/mozlint/mozlint/cli.py | 445 | ||||
-rw-r--r-- | python/mozlint/mozlint/editor.py | 57 | ||||
-rw-r--r-- | python/mozlint/mozlint/errors.py | 33 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/__init__.py | 31 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/compact.py | 41 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/stylish.py | 156 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/summary.py | 50 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/treeherder.py | 34 | ||||
-rw-r--r-- | python/mozlint/mozlint/formatters/unix.py | 33 | ||||
-rw-r--r-- | python/mozlint/mozlint/parser.py | 130 | ||||
-rw-r--r-- | python/mozlint/mozlint/pathutils.py | 313 | ||||
-rw-r--r-- | python/mozlint/mozlint/result.py | 163 | ||||
-rw-r--r-- | python/mozlint/mozlint/roller.py | 421 | ||||
-rw-r--r-- | python/mozlint/mozlint/types.py | 214 | ||||
-rw-r--r-- | python/mozlint/mozlint/util/__init__.py | 0 | ||||
-rw-r--r-- | python/mozlint/mozlint/util/implementation.py | 35 | ||||
-rw-r--r-- | python/mozlint/mozlint/util/string.py | 9 |
18 files changed, 2172 insertions, 0 deletions
diff --git a/python/mozlint/mozlint/__init__.py b/python/mozlint/mozlint/__init__.py new file mode 100644 index 0000000000..bcab4a48b1 --- /dev/null +++ b/python/mozlint/mozlint/__init__.py @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# flake8: noqa + +from .result import Issue +from .roller import LintRoller diff --git a/python/mozlint/mozlint/cli.py b/python/mozlint/mozlint/cli.py new file mode 100644 index 0000000000..0262173367 --- /dev/null +++ b/python/mozlint/mozlint/cli.py @@ -0,0 +1,445 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +from argparse import REMAINDER, SUPPRESS, ArgumentParser +from pathlib import Path + +from mozlint.errors import NoValidLinter +from mozlint.formatters import all_formatters + + +class MozlintParser(ArgumentParser): + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": None, + "help": "Paths to file or directories to lint, like " + "'browser/components/loop' or 'mobile/android'. " + "If not provided, defaults to the files changed according " + "to --outgoing and --workdir.", + }, + ], + [ + ["-l", "--linter"], + { + "dest": "linters", + "default": [], + "action": "append", + "help": "Linters to run, e.g 'eslint'. By default all linters " + "are run for all the appropriate files.", + }, + ], + [ + ["--list"], + { + "dest": "list_linters", + "default": False, + "action": "store_true", + "help": "List all available linters and exit.", + }, + ], + [ + ["-W", "--warnings"], + { + "const": True, + "nargs": "?", + "choices": ["soft"], + "dest": "show_warnings", + "help": "Display and fail on warnings in addition to errors. " + "--warnings=soft can be used to report warnings but only fail " + "on errors.", + }, + ], + [ + ["-v", "--verbose"], + { + "dest": "show_verbose", + "default": False, + "action": "store_true", + "help": "Enable verbose logging.", + }, + ], + [ + ["-f", "--format"], + { + "dest": "formats", + "action": "append", + "help": "Formatter to use. Defaults to 'stylish' on stdout. " + "You can specify an optional path as --format formatter:path " + "that will be used instead of stdout. " + "You can also use multiple formatters at the same time. " + "Formatters available: {}.".format(", ".join(all_formatters.keys())), + }, + ], + [ + ["-n", "--no-filter"], + { + "dest": "use_filters", + "default": True, + "action": "store_false", + "help": "Ignore all filtering. This is useful for quickly " + "testing a directory that otherwise wouldn't be run, " + "without needing to modify the config file.", + }, + ], + [ + ["--include-third-party"], + { + "dest": "include_third-party", + "default": False, + "action": "store_true", + "help": "Also run the linter(s) on third-party code", + }, + ], + [ + ["-o", "--outgoing"], + { + "const": True, + "nargs": "?", + "help": "Lint files touched by commits that are not on the remote repository. " + "Without arguments, finds the default remote that would be pushed to. " + "The remote branch can also be specified manually. Works with " + "mercurial or git.", + }, + ], + [ + ["-w", "--workdir"], + { + "const": "all", + "nargs": "?", + "choices": ["staged", "all"], + "help": "Lint files touched by changes in the working directory " + "(i.e haven't been committed yet). On git, --workdir=staged " + "can be used to only consider staged files. Works with " + "mercurial or git.", + }, + ], + [ + ["-r", "--rev"], + { + "default": None, + "type": str, + "help": "Lint files touched by changes in revisions described by REV. " + "For mercurial, it may be any revset. For git, it is a single tree-ish.", + }, + ], + [ + ["--fix"], + { + "action": "store_true", + "default": False, + "help": "Fix lint errors if possible. Any errors that could not be fixed " + "will be printed as normal.", + }, + ], + [ + ["--edit"], + { + "action": "store_true", + "default": False, + "help": "Each file containing lint errors will be opened in $EDITOR one after " + "the other.", + }, + ], + [ + ["--setup"], + { + "action": "store_true", + "default": False, + "help": "Bootstrap linter dependencies without running any of the linters.", + }, + ], + [ + ["-j", "--jobs"], + { + "default": None, + "dest": "num_procs", + "type": int, + "help": "Number of worker processes to spawn when running linters. " + "Defaults to the number of cores in your CPU.", + }, + ], + # Paths to check for linter configurations. + # Default: tools/lint set in tools/lint/mach_commands.py + [ + ["--config-path"], + { + "action": "append", + "default": [], + "dest": "config_paths", + "help": SUPPRESS, + }, + ], + [ + ["--check-exclude-list"], + { + "dest": "check_exclude_list", + "default": False, + "action": "store_true", + "help": "Run linters for all the paths in the exclude list.", + }, + ], + [ + ["extra_args"], + { + "nargs": REMAINDER, + "help": "Extra arguments that will be forwarded to the underlying linter.", + }, + ], + ] + + def __init__(self, **kwargs): + ArgumentParser.__init__(self, usage=self.__doc__, **kwargs) + + for cli, args in self.arguments: + self.add_argument(*cli, **args) + + def parse_known_args(self, *args, **kwargs): + # Allow '-wo' or '-ow' as shorthand for both --workdir and --outgoing. + for token in ("-wo", "-ow"): + if token in args[0]: + i = args[0].index(token) + args[0].pop(i) + args[0][i:i] = [token[:2], "-" + token[2]] + + # This is here so the eslint mach command doesn't lose 'extra_args' + # when using mach's dispatch functionality. + args, extra = ArgumentParser.parse_known_args(self, *args, **kwargs) + args.extra_args = extra + + self.validate(args) + return args, extra + + def validate(self, args): + if args.edit and not os.environ.get("EDITOR"): + self.error("must set the $EDITOR environment variable to use --edit") + + if args.paths: + invalid = [p for p in args.paths if not os.path.exists(p)] + if invalid: + self.error( + "the following paths do not exist:\n{}".format("\n".join(invalid)) + ) + + if args.formats: + formats = [] + for fmt in args.formats: + if isinstance(fmt, tuple): # format is already processed + formats.append(fmt) + continue + + path = None + if ":" in fmt: + # Detect optional formatter path + pos = fmt.index(":") + fmt, path = fmt[:pos], os.path.realpath(fmt[pos + 1 :]) + + # Check path is writable + fmt_dir = os.path.dirname(path) + if not os.access(fmt_dir, os.W_OK | os.X_OK): + self.error( + "the following directory is not writable: {}".format( + fmt_dir + ) + ) + + if fmt not in all_formatters.keys(): + self.error( + "the following formatter is not available: {}".format(fmt) + ) + + formats.append((fmt, path)) + args.formats = formats + else: + # Can't use argparse default or this choice will be always present + args.formats = [("stylish", None)] + + +def find_linters(config_paths, linters=None): + lints = {} + for search_path in config_paths: + if not os.path.isdir(search_path): + continue + + sys.path.insert(0, search_path) + files = os.listdir(search_path) + for f in files: + name = os.path.basename(f) + + if not name.endswith(".yml"): + continue + + name = name.rsplit(".", 1)[0] + + if linters and name not in linters: + continue + + lints[name] = os.path.join(search_path, f) + + linters_not_found = list(set(linters).difference(set(lints.keys()))) + return {"lint_paths": lints.values(), "linters_not_found": linters_not_found} + + +def get_exclude_list_output(result, paths): + # Store the paths of all the subdirectories leading to the error files + error_file_paths = set() + for issues in result.issues.values(): + error_file = issues[0].relpath + error_file_paths.add(error_file) + parent_dir = os.path.dirname(error_file) + while parent_dir: + error_file_paths.add(parent_dir) + parent_dir = os.path.dirname(parent_dir) + + paths = [os.path.dirname(path) if path[-1] == "/" else path for path in paths] + # Remove all the error paths to get the list of green paths + green_paths = sorted(set(paths).difference(error_file_paths)) + + if green_paths: + out = ( + "The following list of paths are now green " + "and can be removed from the exclude list:\n\n" + ) + out += "\n".join(green_paths) + + else: + out = "No path in the exclude list is green." + + return out + + +def run( + paths, + linters, + formats, + outgoing, + workdir, + rev, + edit, + check_exclude_list, + setup=False, + list_linters=False, + num_procs=None, + virtualenv_manager=None, + setupargs=None, + **lintargs +): + from mozlint import LintRoller, formatters + from mozlint.editor import edit_issues + + lintargs["config_paths"] = [ + os.path.join(lintargs["root"], p) for p in lintargs["config_paths"] + ] + + # Always perform exhaustive linting for exclude list paths + lintargs["use_filters"] = lintargs["use_filters"] and not check_exclude_list + + if list_linters: + lint_paths = find_linters(lintargs["config_paths"], linters) + linters = [ + os.path.splitext(os.path.basename(l))[0] for l in lint_paths["lint_paths"] + ] + print("\n".join(sorted(linters))) + print( + "\nNote that clang-tidy checks are not run as part of this " + "command, but using the static-analysis command." + ) + return 0 + + lint = LintRoller(setupargs=setupargs or {}, **lintargs) + linters_info = find_linters(lintargs["config_paths"], linters) + + result = None + + try: + + lint.read(linters_info["lint_paths"]) + + if check_exclude_list: + if len(lint.linters) > 1: + print("error: specify a single linter to check with `-l/--linter`") + return 1 + paths = lint.linters[0]["local_exclude"] + + if ( + not paths + and Path.cwd() == Path(lint.root) + and not (outgoing or workdir or rev) + ): + print( + "warning: linting the entire repo takes a long time, using --outgoing and " + "--workdir instead. If you want to lint the entire repo, run `./mach lint .`" + ) + # Setting the default values + outgoing = True + workdir = "all" + + # Always run bootstrapping, but return early if --setup was passed in. + ret = lint.setup(virtualenv_manager=virtualenv_manager) + if setup: + return ret + + if linters_info["linters_not_found"] != []: + raise NoValidLinter + + # run all linters + result = lint.roll( + paths, outgoing=outgoing, workdir=workdir, rev=rev, num_procs=num_procs + ) + except NoValidLinter as e: + result = lint.result + print(str(e)) + + if edit and result.issues: + edit_issues(result) + result = lint.roll(result.issues.keys(), num_procs=num_procs) + + for every in linters_info["linters_not_found"]: + result.failed_setup.add(every) + + if check_exclude_list: + # Get and display all those paths in the exclude list which are + # now green and can be safely removed from the list + out = get_exclude_list_output(result, paths) + print(out, file=sys.stdout) + return result.returncode + + for formatter_name, path in formats: + formatter = formatters.get(formatter_name) + + out = formatter(result) + # We do this only for `json` that is mostly used in automation + if not out and formatter_name == "json": + out = "{}" + + if out: + fh = open(path, "w") if path else sys.stdout + + if not path and fh.encoding == "ascii": + # If sys.stdout.encoding is ascii, printing output will fail + # due to the stylish formatter's use of unicode characters. + # Ideally the user should fix their environment by setting + # `LC_ALL=C.UTF-8` or similar. But this is a common enough + # problem that we help them out a little here by manually + # encoding and writing to the stdout buffer directly. + out += "\n" + fh.buffer.write(out.encode("utf-8", errors="replace")) + fh.buffer.flush() + else: + print(out, file=fh) + + if path: + fh.close() + + return result.returncode + + +if __name__ == "__main__": + parser = MozlintParser() + args = vars(parser.parse_args()) + sys.exit(run(**args)) diff --git a/python/mozlint/mozlint/editor.py b/python/mozlint/mozlint/editor.py new file mode 100644 index 0000000000..1738892f93 --- /dev/null +++ b/python/mozlint/mozlint/editor.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import tempfile + +from mozlint import formatters + + +def get_editor(): + return os.environ.get("EDITOR") + + +def edit_issues(result): + if not result.issues: + return + + editor = get_editor() + if not editor: + print("warning: could not find a default editor") + return + + name = os.path.basename(editor) + if name in ("vim", "nvim", "gvim"): + cmd = [ + editor, + # need errorformat to match both Error and Warning, with or without a column + "--cmd", + "set errorformat+=%f:\\ line\\ %l\\\\,\\ col\\ %c\\\\,\\ %trror\\ -\\ %m", + "--cmd", + "set errorformat+=%f:\\ line\\ %l\\\\,\\ col\\ %c\\\\,\\ %tarning\\ -\\ %m", + "--cmd", + "set errorformat+=%f:\\ line\\ %l\\\\,\\ %trror\\ -\\ %m", + "--cmd", + "set errorformat+=%f:\\ line\\ %l\\\\,\\ %tarning\\ -\\ %m", + # start with quickfix window opened + "-c", + "copen", + # running with -q seems to open an empty buffer in addition to the + # first file, this removes that empty buffer + "-c", + "1bd", + ] + + with tempfile.NamedTemporaryFile(mode="w") as fh: + s = formatters.get("compact", summary=False)(result) + fh.write(s) + fh.flush() + + cmd.extend(["-q", fh.name]) + subprocess.call(cmd) + + else: + for path, errors in result.issues.items(): + subprocess.call([editor, path]) diff --git a/python/mozlint/mozlint/errors.py b/python/mozlint/mozlint/errors.py new file mode 100644 index 0000000000..4b36f00f69 --- /dev/null +++ b/python/mozlint/mozlint/errors.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class LintException(Exception): + pass + + +class LinterNotFound(LintException): + def __init__(self, path): + LintException.__init__(self, "Could not find lint file '{}'".format(path)) + + +class NoValidLinter(LintException): + def __init__(self): + LintException.__init__( + self, + "Invalid linters given, run again using valid linters or no linters", + ) + + +class LinterParseError(LintException): + def __init__(self, path, message): + LintException.__init__(self, "{}: {}".format(path, message)) + + +class LintersNotConfigured(LintException): + def __init__(self): + LintException.__init__( + self, + "No linters registered! Use `LintRoller.read` " "to register a linter.", + ) diff --git a/python/mozlint/mozlint/formatters/__init__.py b/python/mozlint/mozlint/formatters/__init__.py new file mode 100644 index 0000000000..e50616216f --- /dev/null +++ b/python/mozlint/mozlint/formatters/__init__.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json + +from ..result import IssueEncoder +from .compact import CompactFormatter +from .stylish import StylishFormatter +from .summary import SummaryFormatter +from .treeherder import TreeherderFormatter +from .unix import UnixFormatter + + +class JSONFormatter(object): + def __call__(self, result): + return json.dumps(result.issues, cls=IssueEncoder) + + +all_formatters = { + "compact": CompactFormatter, + "json": JSONFormatter, + "stylish": StylishFormatter, + "summary": SummaryFormatter, + "treeherder": TreeherderFormatter, + "unix": UnixFormatter, +} + + +def get(name, **fmtargs): + return all_formatters[name](**fmtargs) diff --git a/python/mozlint/mozlint/formatters/compact.py b/python/mozlint/mozlint/formatters/compact.py new file mode 100644 index 0000000000..54ee194215 --- /dev/null +++ b/python/mozlint/mozlint/formatters/compact.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import attr + +from ..result import Issue + + +class CompactFormatter(object): + """Formatter for compact output. + + This formatter prints one error per line, mimicking the + eslint 'compact' formatter. + """ + + # If modifying this format, please also update the vim errorformats in editor.py + fmt = "{path}: line {lineno}{column}, {level} - {message} ({rule})" + + def __init__(self, summary=True): + self.summary = summary + + def __call__(self, result): + message = [] + num_problems = 0 + for path, errors in sorted(result.issues.items()): + num_problems += len(errors) + for err in errors: + assert isinstance(err, Issue) + + d = attr.asdict(err) + d["column"] = ", col %s" % d["column"] if d["column"] else "" + d["level"] = d["level"].capitalize() + d["rule"] = d["rule"] or d["linter"] + message.append(self.fmt.format(**d)) + + if self.summary and num_problems: + message.append( + "\n{} problem{}".format(num_problems, "" if num_problems == 1 else "s") + ) + return "\n".join(message) diff --git a/python/mozlint/mozlint/formatters/stylish.py b/python/mozlint/mozlint/formatters/stylish.py new file mode 100644 index 0000000000..3f80bc7ad2 --- /dev/null +++ b/python/mozlint/mozlint/formatters/stylish.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozterm import Terminal + +from ..result import Issue +from ..util.string import pluralize + + +class StylishFormatter(object): + """Formatter based on the eslint default.""" + + _indent_ = " " + + # Colors later on in the list are fallbacks in case the terminal + # doesn't support colors earlier in the list. + # See http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html + _colors = { + "grey": [247, 8, 7], + "red": [1], + "green": [2], + "yellow": [3], + "brightred": [9, 1], + "brightyellow": [11, 3], + } + + fmt = """ + {c1}{lineno}{column} {c2}{level}{normal} {message} {c1}{rule}({linter}){normal} +{diff}""".lstrip( + "\n" + ) + fmt_summary = ( + "{t.bold}{c}\u2716 {problem} ({error}, {warning}{failure}, {fixed}){t.normal}" + ) + + def __init__(self, disable_colors=False): + self.term = Terminal(disable_styling=disable_colors) + self.num_colors = self.term.number_of_colors + + def color(self, color): + for num in self._colors[color]: + if num < self.num_colors: + return self.term.color(num) + return "" + + def _reset_max(self): + self.max_lineno = 0 + self.max_column = 0 + self.max_level = 0 + self.max_message = 0 + + def _update_max(self, err): + """Calculates the longest length of each token for spacing.""" + self.max_lineno = max(self.max_lineno, len(str(err.lineno))) + if err.column: + self.max_column = max(self.max_column, len(str(err.column))) + self.max_level = max(self.max_level, len(str(err.level))) + self.max_message = max(self.max_message, len(err.message)) + + def _get_colored_diff(self, diff): + if not diff: + return "" + + new_diff = "" + for line in diff.split("\n"): + if line.startswith("+"): + new_diff += self.color("green") + elif line.startswith("-"): + new_diff += self.color("red") + else: + new_diff += self.term.normal + new_diff += self._indent_ + line + "\n" + return new_diff + + def __call__(self, result): + message = [] + failed = result.failed + + num_errors = 0 + num_warnings = 0 + num_fixed = result.fixed + for path, errors in sorted(result.issues.items()): + self._reset_max() + + message.append(self.term.underline(path)) + # Do a first pass to calculate required padding + for err in errors: + assert isinstance(err, Issue) + self._update_max(err) + if err.level == "error": + num_errors += 1 + else: + num_warnings += 1 + + for err in sorted( + errors, key=lambda e: (int(e.lineno), int(e.column or 0)) + ): + if err.column: + col = ":" + str(err.column).ljust(self.max_column) + else: + col = "".ljust(self.max_column + 1) + + args = { + "normal": self.term.normal, + "c1": self.color("grey"), + "c2": self.color("red") + if err.level == "error" + else self.color("yellow"), + "lineno": str(err.lineno).rjust(self.max_lineno), + "column": col, + "level": err.level.ljust(self.max_level), + "rule": "{} ".format(err.rule) if err.rule else "", + "linter": err.linter.lower(), + "message": err.message.ljust(self.max_message), + "diff": self._get_colored_diff(err.diff).ljust(self.max_message), + } + message.append(self.fmt.format(**args).rstrip().rstrip("\n")) + + message.append("") # newline + + # If there were failures, make it clear which linters failed + for fail in failed: + message.append( + "{c}A failure occurred in the {name} linter.".format( + c=self.color("brightred"), name=fail + ) + ) + + # Print a summary + message.append( + self.fmt_summary.format( + t=self.term, + c=self.color("brightred") + if num_errors or failed + else self.color("brightyellow"), + problem=pluralize("problem", num_errors + num_warnings + len(failed)), + error=pluralize("error", num_errors), + warning=pluralize( + "warning", num_warnings or result.total_suppressed_warnings + ), + failure=", {}".format(pluralize("failure", len(failed))) + if failed + else "", + fixed="{} fixed".format(num_fixed), + ) + ) + + if result.total_suppressed_warnings > 0 and num_errors == 0: + message.append( + "(pass {c1}-W/--warnings{c2} to see warnings.)".format( + c1=self.color("grey"), c2=self.term.normal + ) + ) + + return "\n".join(message) diff --git a/python/mozlint/mozlint/formatters/summary.py b/python/mozlint/mozlint/formatters/summary.py new file mode 100644 index 0000000000..e6ecf37508 --- /dev/null +++ b/python/mozlint/mozlint/formatters/summary.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from collections import defaultdict + +import mozpack.path as mozpath + +from ..util.string import pluralize + + +class SummaryFormatter(object): + def __init__(self, depth=None): + self.depth = depth or int(os.environ.get("MOZLINT_SUMMARY_DEPTH", 1)) + + def __call__(self, result): + paths = set( + list(result.issues.keys()) + list(result.suppressed_warnings.keys()) + ) + + commonprefix = mozpath.commonprefix([mozpath.abspath(p) for p in paths]) + commonprefix = commonprefix.rsplit("/", 1)[0] + "/" + + summary = defaultdict(lambda: [0, 0]) + for path in paths: + abspath = mozpath.abspath(path) + assert abspath.startswith(commonprefix) + + if abspath != commonprefix: + parts = mozpath.split(mozpath.relpath(abspath, commonprefix))[ + : self.depth + ] + abspath = mozpath.join(commonprefix, *parts) + + summary[abspath][0] += len( + [r for r in result.issues[path] if r.level == "error"] + ) + summary[abspath][1] += len( + [r for r in result.issues[path] if r.level == "warning"] + ) + summary[abspath][1] += result.suppressed_warnings[path] + + msg = [] + for path, (errors, warnings) in sorted(summary.items()): + warning_str = ( + ", {}".format(pluralize("warning", warnings)) if warnings else "" + ) + msg.append("{}: {}{}".format(path, pluralize("error", errors), warning_str)) + return "\n".join(msg) diff --git a/python/mozlint/mozlint/formatters/treeherder.py b/python/mozlint/mozlint/formatters/treeherder.py new file mode 100644 index 0000000000..66c7c59eee --- /dev/null +++ b/python/mozlint/mozlint/formatters/treeherder.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import attr + +from ..result import Issue + + +class TreeherderFormatter(object): + """Formatter for treeherder friendly output. + + This formatter looks ugly, but prints output such that + treeherder is able to highlight the errors and warnings. + This is a stop-gap until bug 1276486 is fixed. + """ + + fmt = "TEST-UNEXPECTED-{level} | {path}:{lineno}{column} | {message} ({rule})" + + def __call__(self, result): + message = [] + for path, errors in sorted(result.issues.items()): + for err in errors: + assert isinstance(err, Issue) + + d = attr.asdict(err) + d["column"] = ":%s" % d["column"] if d["column"] else "" + d["level"] = d["level"].upper() + d["rule"] = d["rule"] or d["linter"] + message.append(self.fmt.format(**d)) + + if not message: + message.append("No lint issues found.") + return "\n".join(message) diff --git a/python/mozlint/mozlint/formatters/unix.py b/python/mozlint/mozlint/formatters/unix.py new file mode 100644 index 0000000000..ae096f3e2e --- /dev/null +++ b/python/mozlint/mozlint/formatters/unix.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import attr + +from ..result import Issue + + +class UnixFormatter(object): + """Formatter that respects Unix output conventions frequently + employed by preprocessors and compilers. The format is + `<FILENAME>:<LINE>[:<COL>]: <RULE> <LEVEL>: <MESSAGE>`. + + """ + + fmt = "{path}:{lineno}:{column} {rule} {level}: {message}" + + def __call__(self, result): + msg = [] + + for path, errors in sorted(result.issues.items()): + for err in errors: + assert isinstance(err, Issue) + + slots = attr.asdict(err) + slots["path"] = slots["relpath"] + slots["column"] = "%d:" % slots["column"] if slots["column"] else "" + slots["rule"] = slots["rule"] or slots["linter"] + + msg.append(self.fmt.format(**slots)) + + return "\n".join(msg) diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py new file mode 100644 index 0000000000..eac502495b --- /dev/null +++ b/python/mozlint/mozlint/parser.py @@ -0,0 +1,130 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import yaml + +from .errors import LinterNotFound, LinterParseError +from .types import supported_types + +GLOBAL_SUPPORT_FILES = [] + + +class Parser(object): + """Reads and validates lint configuration files.""" + + required_attributes = ( + "name", + "description", + "type", + "payload", + ) + + def __init__(self, root): + self.root = root + + def __call__(self, path): + return self.parse(path) + + def _validate(self, linter): + relpath = os.path.relpath(linter["path"], self.root) + + missing_attrs = [] + for attr in self.required_attributes: + if attr not in linter: + missing_attrs.append(attr) + + if missing_attrs: + raise LinterParseError( + relpath, + "Missing required attribute(s): " "{}".format(",".join(missing_attrs)), + ) + + if linter["type"] not in supported_types: + raise LinterParseError(relpath, "Invalid type '{}'".format(linter["type"])) + + for attr in ("include", "exclude", "support-files"): + if attr not in linter: + continue + + if not isinstance(linter[attr], list) or not all( + isinstance(a, str) for a in linter[attr] + ): + raise LinterParseError( + relpath, + "The {} directive must be a " "list of strings!".format(attr), + ) + invalid_paths = set() + for path in linter[attr]: + if "*" in path: + if attr == "include": + raise LinterParseError( + relpath, + "Paths in the include directive cannot " + "contain globs:\n {}".format(path), + ) + continue + + abspath = path + if not os.path.isabs(abspath): + abspath = os.path.join(self.root, path) + + if not os.path.exists(abspath): + invalid_paths.add(" " + path) + + if invalid_paths: + raise LinterParseError( + relpath, + "The {} directive contains the following " + "paths that don't exist:\n{}".format( + attr, "\n".join(sorted(invalid_paths)) + ), + ) + + if "setup" in linter: + if linter["setup"].count(":") != 1: + raise LinterParseError( + relpath, + "The setup attribute '{!r}' must have the " + "form 'module:object'".format(linter["setup"]), + ) + + if "extensions" in linter: + linter["extensions"] = [e.strip(".") for e in linter["extensions"]] + + def parse(self, path): + """Read a linter and return its LINTER definition. + + :param path: Path to the linter. + :returns: List of linter definitions ([dict]) + :raises: LinterNotFound, LinterParseError + """ + if not os.path.isfile(path): + raise LinterNotFound(path) + + if not path.endswith(".yml"): + raise LinterParseError( + path, "Invalid filename, linters must end with '.yml'!" + ) + + with open(path) as fh: + configs = list(yaml.safe_load_all(fh)) + + if not configs: + raise LinterParseError(path, "No lint definitions found!") + + linters = [] + for config in configs: + for name, linter in config.items(): + linter["name"] = name + linter["path"] = path + self._validate(linter) + linter.setdefault("support-files", []).extend( + GLOBAL_SUPPORT_FILES + [path] + ) + linter.setdefault("include", ["."]) + linters.append(linter) + + return linters diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py new file mode 100644 index 0000000000..b1b4b644bc --- /dev/null +++ b/python/mozlint/mozlint/pathutils.py @@ -0,0 +1,313 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozpack import path as mozpath +from mozpack.files import FileFinder + + +class FilterPath(object): + """Helper class to make comparing and matching file paths easier.""" + + def __init__(self, path): + self.path = os.path.normpath(path) + self._finder = None + + @property + def finder(self): + if self._finder: + return self._finder + self._finder = FileFinder(mozpath.normsep(self.path)) + return self._finder + + @property + def ext(self): + return os.path.splitext(self.path)[1].strip(".") + + @property + def exists(self): + return os.path.exists(self.path) + + @property + def isfile(self): + return os.path.isfile(self.path) + + @property + def isdir(self): + return os.path.isdir(self.path) + + def join(self, *args): + return FilterPath(os.path.join(self.path, *args)) + + def match(self, patterns): + a = mozpath.normsep(self.path) + for p in patterns: + if isinstance(p, FilterPath): + p = p.path + p = mozpath.normsep(p) + if mozpath.match(a, p): + return True + return False + + def contains(self, other): + """Return True if other is a subdirectory of self or equals self.""" + if isinstance(other, FilterPath): + other = other.path + a = os.path.abspath(self.path) + b = os.path.normpath(os.path.abspath(other)) + + parts_a = a.split(os.sep) + parts_b = b.split(os.sep) + + if len(parts_a) > len(parts_b): + return False + + for i, part in enumerate(parts_a): + if part != parts_b[i]: + return False + return True + + def __repr__(self): + return repr(self.path) + + +def collapse(paths, base=None, dotfiles=False): + """Given an iterable of paths, collapse them into the smallest possible set + of paths that contain the original set (without containing any extra paths). + + For example, if directory 'a' contains two files b.txt and c.txt, calling: + + collapse(['a/b.txt', 'a/c.txt']) + + returns ['a']. But if a third file d.txt also exists, then it will return + ['a/b.txt', 'a/c.txt'] since ['a'] would also include that extra file. + + :param paths: An iterable of paths (files and directories) to collapse. + :returns: The smallest set of paths (files and directories) that contain + the original set of paths and only the original set. + """ + if not paths: + if not base: + return [] + + # Need to test whether directory chain is empty. If it is then bubble + # the base back up so that it counts as 'covered'. + for _, _, names in os.walk(base): + if names: + return [] + return [base] + + if not base: + paths = list(map(mozpath.abspath, paths)) + base = mozpath.commonprefix(paths).rstrip("/") + + # Make sure `commonprefix` factors in sibling directories that have the + # same prefix in their basenames. + parent = mozpath.dirname(base) + same_prefix = [ + p for p in os.listdir(parent) if p.startswith(mozpath.basename(base)) + ] + if not os.path.isdir(base) or len(same_prefix) > 1: + base = parent + + if base in paths: + return [base] + + covered = set() + full = set() + for name in os.listdir(base): + if not dotfiles and name[0] == ".": + continue + + path = mozpath.join(base, name) + full.add(path) + + if path in paths: + # This path was explicitly specified, so just bubble it back up + # without recursing down into it (if it was a directory). + covered.add(path) + elif os.path.isdir(path): + new_paths = [p for p in paths if p.startswith(path)] + covered.update(collapse(new_paths, base=path, dotfiles=dotfiles)) + + if full == covered: + # Every file under this base was covered, so we can collapse them all + # up into the base path. + return [base] + return list(covered) + + +def filterpaths(root, paths, include, exclude=None, extensions=None): + """Filters a list of paths. + + Given a list of paths and some filtering rules, return the set of paths + that should be linted. + + :param paths: A starting list of paths to possibly lint. + :param include: A list of paths that should be included (required). + :param exclude: A list of paths that should be excluded (optional). + :param extensions: A list of file extensions which should be considered (optional). + :returns: A tuple containing a list of file paths to lint and a list of + paths to exclude. + """ + + def normalize(path): + if "*" not in path and not os.path.isabs(path): + path = os.path.join(root, path) + return FilterPath(path) + + # Includes are always paths and should always exist. + include = list(map(normalize, include)) + + # Exclude paths with and without globs will be handled separately, + # pull them apart now. + exclude = list(map(normalize, exclude or [])) + excludepaths = [p for p in exclude if p.exists] + excludeglobs = [p.path for p in exclude if not p.exists] + + keep = set() + discard = set() + for path in list(map(normalize, paths)): + # Exclude bad file extensions + if extensions and path.isfile and path.ext not in extensions: + continue + + if path.match(excludeglobs): + continue + + # First handle include/exclude directives + # that exist (i.e don't have globs) + for inc in include: + # Only excludes that are subdirectories of the include + # path matter. + excs = [e for e in excludepaths if inc.contains(e)] + + if path.contains(inc): + # If specified path is an ancestor of include path, + # then lint the include path. + keep.add(inc) + + # We can't apply these exclude paths without explicitly + # including every sibling file. Rather than do that, + # just return them and hope the underlying linter will + # deal with them. + discard.update(excs) + + elif inc.contains(path): + # If the include path is an ancestor of the specified + # path, then add the specified path only if there are + # no exclude paths in-between them. + if not any(e.contains(path) for e in excs): + keep.add(path) + discard.update([e for e in excs if path.contains(e)]) + + # Next expand excludes with globs in them so we can add them to + # the set of files to discard. + for pattern in excludeglobs: + for p, f in path.finder.find(pattern): + discard.add(path.join(p)) + + return ( + [f.path for f in keep if f.exists], + collapse([f.path for f in discard if f.exists]), + ) + + +def findobject(path): + """ + Find a Python object given a path of the form <modulepath>:<objectpath>. + Conceptually equivalent to + + def find_object(modulepath, objectpath): + import <modulepath> as mod + return mod.<objectpath> + """ + if path.count(":") != 1: + raise ValueError( + 'python path {!r} does not have the form "module:object"'.format(path) + ) + + modulepath, objectpath = path.split(":") + obj = __import__(modulepath) + for a in modulepath.split(".")[1:]: + obj = getattr(obj, a) + for a in objectpath.split("."): + obj = getattr(obj, a) + return obj + + +def ancestors(path): + while path: + yield path + (path, child) = os.path.split(path) + if child == "": + break + + +def get_ancestors_by_name(name, path, root): + """Returns a list of files called `name` in `path`'s ancestors, + sorted from closest->furthest. This can be useful for finding + relevant configuration files. + """ + configs = [] + for path in ancestors(path): + config = os.path.join(path, name) + if os.path.isfile(config): + configs.append(config) + if path == root: + break + return configs + + +def expand_exclusions(paths, config, root): + """Returns all files that match patterns and aren't excluded. + + This is used by some external linters who receive 'batch' files (e.g dirs) + but aren't capable of applying their own exclusions. There is an argument + to be made that this step should just apply to all linters no matter what. + + Args: + paths (list): List of candidate paths to lint. + config (dict): Linter's config object. + root (str): Root of the repository. + + Returns: + Generator which generates list of paths that weren't excluded. + """ + extensions = [e.lstrip(".") for e in config.get("extensions", [])] + find_dotfiles = config.get("find-dotfiles", False) + + def normalize(path): + path = mozpath.normpath(path) + if os.path.isabs(path): + return path + return mozpath.join(root, path) + + exclude = list(map(normalize, config.get("exclude", []))) + for path in paths: + path = mozpath.normsep(path) + if os.path.isfile(path): + if any(path.startswith(e) for e in exclude if "*" not in e): + continue + + if any(mozpath.match(path, e) for e in exclude if "*" in e): + continue + + yield path + continue + + ignore = [ + e[len(path) :].lstrip("/") + for e in exclude + if mozpath.commonprefix((path, e)) == path + ] + finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles) + + _, ext = os.path.splitext(path) + ext.lstrip(".") + + for ext in extensions: + for p, f in finder.find("**/*.{}".format(ext)): + yield os.path.join(path, p) diff --git a/python/mozlint/mozlint/result.py b/python/mozlint/mozlint/result.py new file mode 100644 index 0000000000..01b04afee6 --- /dev/null +++ b/python/mozlint/mozlint/result.py @@ -0,0 +1,163 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from collections import defaultdict +from itertools import chain +from json import JSONEncoder + +import attr +import mozpack.path as mozpath + + +class ResultSummary(object): + """Represents overall result state from an entire lint run.""" + + root = None + + def __init__(self, root, fail_on_warnings=True): + self.fail_on_warnings = fail_on_warnings + self.reset() + + # Store the repository root folder to be able to build + # Issues relative paths to that folder + if ResultSummary.root is None: + ResultSummary.root = mozpath.normpath(root) + + def reset(self): + self.issues = defaultdict(list) + self.failed_run = set() + self.failed_setup = set() + self.suppressed_warnings = defaultdict(int) + self.fixed = 0 + + def has_issues_failure(self): + """Returns true in case issues were detected during the lint run. Do not + consider warning issues in case `self.fail_on_warnings` is set to False. + """ + if self.fail_on_warnings is False: + return any( + result.level != "warning" for result in chain(*self.issues.values()) + ) + return len(self.issues) >= 1 + + @property + def returncode(self): + if self.has_issues_failure() or self.failed: + return 1 + return 0 + + @property + def failed(self): + return self.failed_setup | self.failed_run + + @property + def total_issues(self): + return sum([len(v) for v in self.issues.values()]) + + @property + def total_suppressed_warnings(self): + return sum(self.suppressed_warnings.values()) + + @property + def total_fixed(self): + return self.fixed + + def update(self, other): + """Merge results from another ResultSummary into this one.""" + for path, obj in other.issues.items(): + self.issues[path].extend(obj) + + self.failed_run |= other.failed_run + self.failed_setup |= other.failed_setup + self.fixed += other.fixed + for k, v in other.suppressed_warnings.items(): + self.suppressed_warnings[k] += v + + +@attr.s(slots=True, kw_only=True) +class Issue(object): + """Represents a single lint issue and its related metadata. + + :param linter: name of the linter that flagged this error + :param path: path to the file containing the error + :param message: text describing the error + :param lineno: line number that contains the error + :param column: column containing the error + :param level: severity of the error, either 'warning' or 'error' (default 'error') + :param hint: suggestion for fixing the error (optional) + :param source: source code context of the error (optional) + :param rule: name of the rule that was violated (optional) + :param lineoffset: denotes an error spans multiple lines, of the form + (<lineno offset>, <num lines>) (optional) + :param diff: a diff describing the changes that need to be made to the code + """ + + linter = attr.ib() + path = attr.ib() + lineno = attr.ib( + default=None, converter=lambda lineno: int(lineno) if lineno else 0 + ) + column = attr.ib( + default=None, converter=lambda column: int(column) if column else column + ) + message = attr.ib() + hint = attr.ib(default=None) + source = attr.ib(default=None) + level = attr.ib(default=None, converter=lambda level: level or "error") + rule = attr.ib(default=None) + lineoffset = attr.ib(default=None) + diff = attr.ib(default=None) + relpath = attr.ib(init=False, default=None) + + def __attrs_post_init__(self): + root = ResultSummary.root + assert root is not None, "Missing ResultSummary.root" + if os.path.isabs(self.path): + self.path = mozpath.normpath(self.path) + self.relpath = mozpath.relpath(self.path, root) + else: + self.relpath = mozpath.normpath(self.path) + self.path = mozpath.join(root, self.path) + + +class IssueEncoder(JSONEncoder): + """Class for encoding :class:`~result.Issue` to json. + + Usage: + + .. code-block:: python + + json.dumps(results, cls=IssueEncoder) + + """ + + def default(self, o): + if isinstance(o, Issue): + return attr.asdict(o) + return JSONEncoder.default(self, o) + + +def from_config(config, **kwargs): + """Create a :class:`~result.Issue` from a linter config. + + Convenience method that pulls defaults from a linter + config and forwards them. + + :param config: linter config as defined in a .yml file + :param kwargs: same as :class:`~result.Issue` + :returns: :class:`~result.Issue` object + """ + args = {} + for arg in attr.fields(Issue): + if arg.init: + args[arg.name] = kwargs.get(arg.name, config.get(arg.name)) + + if not args["linter"]: + args["linter"] = config.get("name") + + if not args["message"]: + args["message"] = config.get("description") + + return Issue(**args) diff --git a/python/mozlint/mozlint/roller.py b/python/mozlint/mozlint/roller.py new file mode 100644 index 0000000000..1425178114 --- /dev/null +++ b/python/mozlint/mozlint/roller.py @@ -0,0 +1,421 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import atexit +import copy +import logging +import os +import signal +import sys +import time +import traceback +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import _python_exit as futures_atexit +from itertools import chain +from math import ceil +from multiprocessing import cpu_count, get_context +from multiprocessing.queues import Queue +from subprocess import CalledProcessError +from typing import Dict, Set + +import mozpack.path as mozpath +from mozversioncontrol import ( + InvalidRepoPath, + MissingUpstreamRepo, + get_repository_object, +) + +from .errors import LintersNotConfigured, NoValidLinter +from .parser import Parser +from .pathutils import findobject +from .result import ResultSummary +from .types import supported_types + +SHUTDOWN = False +orig_sigint = signal.getsignal(signal.SIGINT) + +logger = logging.getLogger("mozlint") +handler = logging.StreamHandler() +formatter = logging.Formatter( + "%(asctime)s.%(msecs)d %(lintname)s (%(pid)s) | %(message)s", "%H:%M:%S" +) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def _run_worker(config, paths, **lintargs): + log = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) + lintargs["log"] = log + result = ResultSummary(lintargs["root"]) + + if SHUTDOWN: + return result + + # Override warnings setup for code review + # Only disactivating when code_review_warnings is set to False on a linter.yml in use + if os.environ.get("CODE_REVIEW") == "1" and config.get( + "code_review_warnings", True + ): + lintargs["show_warnings"] = True + + # Override ignore thirdparty + # Only deactivating include_thirdparty is set on a linter.yml in use + if config.get("include_thirdparty", False): + lintargs["include_thirdparty"] = True + + func = supported_types[config["type"]] + start_time = time.monotonic() + try: + res = func(paths, config, **lintargs) + # Some linters support fixed operations + # dict returned - {"results":results,"fixed":fixed} + if isinstance(res, dict): + result.fixed += res["fixed"] + res = res["results"] or [] + elif isinstance(res, list): + res = res or [] + else: + print("Unexpected type received") + assert False + except Exception: + traceback.print_exc() + res = 1 + except (KeyboardInterrupt, SystemExit): + return result + finally: + end_time = time.monotonic() + log.debug("Finished in {:.2f} seconds".format(end_time - start_time)) + sys.stdout.flush() + + if not isinstance(res, (list, tuple)): + if res: + result.failed_run.add(config["name"]) + else: + for r in res: + if not lintargs.get("show_warnings") and r.level == "warning": + result.suppressed_warnings[r.path] += 1 + continue + + result.issues[r.path].append(r) + + return result + + +class InterruptableQueue(Queue): + """A multiprocessing.Queue that catches KeyboardInterrupt when a worker is + blocking on it and returns None. + + This is needed to gracefully handle KeyboardInterrupts when a worker is + blocking on ProcessPoolExecutor's call queue. + """ + + def __init__(self, *args, **kwargs): + kwargs["ctx"] = get_context() + super(InterruptableQueue, self).__init__(*args, **kwargs) + + def get(self, *args, **kwargs): + try: + return Queue.get(self, *args, **kwargs) + except KeyboardInterrupt: + return None + + +def _worker_sigint_handler(signum, frame): + """Sigint handler for the worker subprocesses. + + Tells workers not to process the extra jobs on the call queue that couldn't + be canceled by the parent process. + """ + global SHUTDOWN + SHUTDOWN = True + orig_sigint(signum, frame) + + +def wrap_futures_atexit(): + """Sometimes futures' atexit handler can spew tracebacks. This wrapper + suppresses them.""" + try: + futures_atexit() + except Exception: + # Generally `atexit` handlers aren't supposed to raise exceptions, but the + # futures' handler can sometimes raise when the user presses `CTRL-C`. We + # suppress all possible exceptions here so users have a nice experience + # when canceling their lint run. Any exceptions raised by this function + # won't be useful anyway. + pass + + +atexit.unregister(futures_atexit) +atexit.register(wrap_futures_atexit) + + +class LintRoller(object): + """Registers and runs linters. + + :param root: Path to which relative paths will be joined. If + unspecified, root will either be determined from + version control or cwd. + :param lintargs: Arguments to pass to the underlying linter(s). + """ + + MAX_PATHS_PER_JOB = ( + 50 # set a max size to prevent command lines that are too long on Windows + ) + + def __init__(self, root, exclude=None, setupargs=None, **lintargs): + self.parse = Parser(root) + try: + self.vcs = get_repository_object(root) + except InvalidRepoPath: + self.vcs = None + + self.linters = [] + self.lintargs = lintargs + self.lintargs["root"] = root + self._setupargs = setupargs or {} + + # result state + self.result = ResultSummary( + root, + # Prevent failing on warnings when the --warnings parameter is set to "soft" + fail_on_warnings=lintargs.get("show_warnings") != "soft", + ) + + self.root = root + self.exclude = exclude or [] + + if lintargs.get("show_verbose"): + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.WARNING) + + self.log = logging.LoggerAdapter( + logger, {"lintname": "mozlint", "pid": os.getpid()} + ) + + def read(self, paths): + """Parse one or more linters and add them to the registry. + + :param paths: A path or iterable of paths to linter definitions. + """ + if isinstance(paths, str): + paths = (paths,) + + for linter in chain(*[self.parse(p) for p in paths]): + # Add only the excludes present in paths + linter["local_exclude"] = linter.get("exclude", [])[:] + # Add in our global excludes + linter.setdefault("exclude", []).extend(self.exclude) + self.linters.append(linter) + + def setup(self, virtualenv_manager=None): + """Run setup for applicable linters""" + if not self.linters: + raise NoValidLinter + + for linter in self.linters: + if "setup" not in linter: + continue + + try: + setupargs = copy.deepcopy(self.lintargs) + setupargs.update(self._setupargs) + setupargs["name"] = linter["name"] + setupargs["log"] = logging.LoggerAdapter( + self.log, {"lintname": linter["name"]} + ) + if virtualenv_manager is not None: + setupargs["virtualenv_manager"] = virtualenv_manager + start_time = time.monotonic() + res = findobject(linter["setup"])( + **setupargs, + ) + self.log.debug( + f"setup for {linter['name']} finished in " + f"{round(time.monotonic() - start_time, 2)} seconds" + ) + except Exception: + traceback.print_exc() + res = 1 + + if res: + self.result.failed_setup.add(linter["name"]) + + if self.result.failed_setup: + print( + "error: problem with lint setup, skipping {}".format( + ", ".join(sorted(self.result.failed_setup)) + ) + ) + self.linters = [ + l for l in self.linters if l["name"] not in self.result.failed_setup + ] + return 1 + return 0 + + def should_lint_entire_tree(self, vcs_paths: Set[str], linter: Dict) -> bool: + """Return `True` if the linter should be run on the entire tree.""" + # Don't lint the entire tree when `--fix` is passed to linters. + if "fix" in self.lintargs and self.lintargs["fix"]: + return False + + # Lint the whole tree when a `support-file` is modified. + return any( + os.path.isfile(p) and mozpath.match(p, pattern) + for pattern in linter.get("support-files", []) + for p in vcs_paths + ) + + def _generate_jobs(self, paths, vcs_paths, num_procs): + def __get_current_paths(path=self.root): + return [os.path.join(path, p) for p in os.listdir(path)] + + """A job is of the form (<linter:dict>, <paths:list>).""" + for linter in self.linters: + if self.should_lint_entire_tree(vcs_paths, linter): + lpaths = __get_current_paths() + print( + "warning: {} support-file modified, linting entire tree " + "(press ctrl-c to cancel)".format(linter["name"]) + ) + elif paths == {self.root}: + # If the command line is ".", the path will match with the root + # directory. In this case, get all the paths, so that we can + # benefit from chunking below. + lpaths = __get_current_paths() + else: + lpaths = paths.union(vcs_paths) + + lpaths = list(lpaths) or __get_current_paths(os.getcwd()) + chunk_size = ( + min(self.MAX_PATHS_PER_JOB, int(ceil(len(lpaths) / num_procs))) or 1 + ) + if linter["type"] == "global": + # Global linters lint the entire tree in one job. + chunk_size = len(lpaths) or 1 + assert chunk_size > 0 + + while lpaths: + yield linter, lpaths[:chunk_size] + lpaths = lpaths[chunk_size:] + + def _collect_results(self, future): + if future.cancelled(): + return + + # Merge this job's results with our global ones. + self.result.update(future.result()) + + def roll(self, paths=None, outgoing=None, workdir=None, rev=None, num_procs=None): + """Run all of the registered linters against the specified file paths. + + :param paths: An iterable of files and/or directories to lint. + :param outgoing: Lint files touched by commits that are not on the remote repository. + :param workdir: Lint all files touched in the working directory. + :param num_procs: The number of processes to use. Default: cpu count + :return: A :class:`~result.ResultSummary` instance. + """ + if not self.linters: + raise LintersNotConfigured + + self.result.reset() + + # Need to use a set in case vcs operations specify the same file + # more than once. + paths = paths or set() + if isinstance(paths, str): + paths = set([paths]) + elif isinstance(paths, (list, tuple)): + paths = set(paths) + + if not self.vcs and (workdir or outgoing): + print( + "error: '{}' is not a known repository, can't use " + "--workdir or --outgoing".format(self.lintargs["root"]) + ) + + # Calculate files from VCS + vcs_paths = set() + try: + if workdir: + vcs_paths.update(self.vcs.get_changed_files("AM", mode=workdir)) + if rev: + vcs_paths.update(self.vcs.get_changed_files("AM", rev=rev)) + if outgoing: + upstream = outgoing if isinstance(outgoing, str) else None + try: + vcs_paths.update( + self.vcs.get_outgoing_files("AM", upstream=upstream) + ) + except MissingUpstreamRepo: + print( + "warning: could not find default push, specify a remote for --outgoing" + ) + except CalledProcessError as e: + print("error running: {}".format(" ".join(e.cmd))) + if e.output: + print(e.output) + + if not (paths or vcs_paths) and (workdir or outgoing): + if os.environ.get("MOZ_AUTOMATION") and not os.environ.get( + "PYTEST_CURRENT_TEST" + ): + raise Exception( + "Despite being a CI lint job, no files were linted. Is the task " + "missing explicit paths?" + ) + + print("warning: no files linted") + return self.result + + # Make sure all paths are absolute. Join `paths` to cwd and `vcs_paths` to root. + paths = set(map(os.path.abspath, paths)) + vcs_paths = set( + [ + os.path.join(self.root, p) if not os.path.isabs(p) else p + for p in vcs_paths + ] + ) + + num_procs = num_procs or cpu_count() + jobs = list(self._generate_jobs(paths, vcs_paths, num_procs)) + + # Make sure we never spawn more processes than we have jobs. + num_procs = min(len(jobs), num_procs) or 1 + if sys.platform == "win32": + # https://github.com/python/cpython/pull/13132 + num_procs = min(num_procs, 61) + + signal.signal(signal.SIGINT, _worker_sigint_handler) + executor = ProcessPoolExecutor(num_procs) + executor._call_queue = InterruptableQueue(executor._call_queue._maxsize) + + # Submit jobs to the worker pool. The _collect_results method will be + # called when a job is finished. We store the futures so that they can + # be canceled in the event of a KeyboardInterrupt. + futures = [] + for job in jobs: + future = executor.submit(_run_worker, *job, **self.lintargs) + future.add_done_callback(self._collect_results) + futures.append(future) + + def _parent_sigint_handler(signum, frame): + """Sigint handler for the parent process. + + Cancels all jobs that have not yet been placed on the call queue. + The parent process won't exit until all workers have terminated. + Assuming the linters are implemented properly, this shouldn't take + more than a couple seconds. + """ + [f.cancel() for f in futures] + executor.shutdown(wait=True) + print("\nwarning: not all files were linted") + signal.signal(signal.SIGINT, signal.SIG_IGN) + + signal.signal(signal.SIGINT, _parent_sigint_handler) + executor.shutdown() + signal.signal(signal.SIGINT, orig_sigint) + return self.result diff --git a/python/mozlint/mozlint/types.py b/python/mozlint/mozlint/types.py new file mode 100644 index 0000000000..1a9a0bd473 --- /dev/null +++ b/python/mozlint/mozlint/types.py @@ -0,0 +1,214 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import sys +from abc import ABCMeta, abstractmethod + +from mozlog import commandline, get_default_logger, structuredlog +from mozlog.reader import LogHandler +from mozpack.files import FileFinder + +from . import result +from .pathutils import expand_exclusions, filterpaths, findobject + + +class BaseType(object): + """Abstract base class for all types of linters.""" + + __metaclass__ = ABCMeta + batch = False + + def __call__(self, paths, config, **lintargs): + """Run linter defined by `config` against `paths` with `lintargs`. + + :param paths: Paths to lint. Can be a file or directory. + :param config: Linter config the paths are being linted against. + :param lintargs: External arguments to the linter not defined in + the definition, but passed in by a consumer. + :returns: A list of :class:`~result.Issue` objects. + """ + log = lintargs["log"] + + if lintargs.get("use_filters", True): + paths, exclude = filterpaths( + lintargs["root"], + paths, + config["include"], + config.get("exclude", []), + config.get("extensions", []), + ) + config["exclude"] = exclude + elif config.get("exclude"): + del config["exclude"] + + if not paths: + return {"results": [], "fixed": 0} + + log.debug( + "Passing the following paths:\n{paths}".format( + paths=" \n".join(paths), + ) + ) + + if self.batch: + return self._lint(paths, config, **lintargs) + + errors = [] + + try: + for p in paths: + result = self._lint(p, config, **lintargs) + if result: + errors.extend(result) + except KeyboardInterrupt: + pass + return errors + + def _lint_dir(self, path, config, **lintargs): + if not config.get("extensions"): + patterns = ["**"] + else: + patterns = ["**/*.{}".format(e) for e in config["extensions"]] + + exclude = [os.path.relpath(e, path) for e in config.get("exclude", [])] + finder = FileFinder(path, ignore=exclude) + + errors = [] + for pattern in patterns: + for p, f in finder.find(pattern): + errors.extend(self._lint(os.path.join(path, p), config, **lintargs)) + return errors + + @abstractmethod + def _lint(self, path, config, **lintargs): + pass + + +class LineType(BaseType): + """Abstract base class for linter types that check each line individually. + + Subclasses of this linter type will read each file and check the provided + payload against each line one by one. + """ + + __metaclass__ = ABCMeta + + @abstractmethod + def condition(payload, line, config): + pass + + def _lint(self, path, config, **lintargs): + if os.path.isdir(path): + return self._lint_dir(path, config, **lintargs) + + payload = config["payload"] + with open(path, "r", errors="replace") as fh: + lines = fh.readlines() + + errors = [] + for i, line in enumerate(lines): + if self.condition(payload, line, config): + errors.append(result.from_config(config, path=path, lineno=i + 1)) + + return errors + + +class StringType(LineType): + """Linter type that checks whether a substring is found.""" + + def condition(self, payload, line, config): + return payload in line + + +class RegexType(LineType): + """Linter type that checks whether a regex match is found.""" + + def condition(self, payload, line, config): + flags = 0 + if config.get("ignore-case"): + flags |= re.IGNORECASE + + return re.search(payload, line, flags) + + +class ExternalType(BaseType): + """Linter type that runs an external function. + + The function is responsible for properly formatting the results + into a list of :class:`~result.Issue` objects. + """ + + batch = True + + def _lint(self, files, config, **lintargs): + func = findobject(config["payload"]) + return func(files, config, **lintargs) + + +class ExternalFileType(ExternalType): + batch = False + + +class GlobalType(ExternalType): + """Linter type that runs an external global linting function just once. + + The function is responsible for properly formatting the results + into a list of :class:`~result.Issue` objects. + """ + + batch = True + + def _lint(self, files, config, **lintargs): + # Global lints are expensive to invoke. Try to avoid running + # them based on extensions and exclusions. + try: + next(expand_exclusions(files, config, lintargs["root"])) + except StopIteration: + return [] + + func = findobject(config["payload"]) + return func(config, **lintargs) + + +class LintHandler(LogHandler): + def __init__(self, config): + self.config = config + self.results = [] + + def lint(self, data): + self.results.append(result.from_config(self.config, **data)) + + +class StructuredLogType(BaseType): + batch = True + + def _lint(self, files, config, **lintargs): + handler = LintHandler(config) + logger = config.get("logger") + if logger is None: + logger = get_default_logger() + if logger is None: + logger = structuredlog.StructuredLogger(config["name"]) + commandline.setup_logging(logger, {}, {"mach": sys.stdout}) + logger.add_handler(handler) + + func = findobject(config["payload"]) + try: + func(files, config, logger, **lintargs) + except KeyboardInterrupt: + pass + return handler.results + + +supported_types = { + "string": StringType(), + "regex": RegexType(), + "external": ExternalType(), + "external-file": ExternalFileType(), + "global": GlobalType(), + "structured_log": StructuredLogType(), +} +"""Mapping of type string to an associated instance.""" diff --git a/python/mozlint/mozlint/util/__init__.py b/python/mozlint/mozlint/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozlint/mozlint/util/__init__.py diff --git a/python/mozlint/mozlint/util/implementation.py b/python/mozlint/mozlint/util/implementation.py new file mode 100644 index 0000000000..9c72c0ea0f --- /dev/null +++ b/python/mozlint/mozlint/util/implementation.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import signal +from abc import ABC, abstractmethod + +from mozprocess import ProcessHandlerMixin + + +class LintProcess(ProcessHandlerMixin, ABC): + def __init__(self, config, *args, **kwargs): + self.config = config + self.results = [] + + kwargs["universal_newlines"] = True + kwargs["processOutputLine"] = [self.process_line] + ProcessHandlerMixin.__init__(self, *args, **kwargs) + + @abstractmethod + def process_line(self, line): + """Process a single line of output. + + The implementation is responsible for creating one or more :class:`~mozlint.result.Issue` + and storing them somewhere accessible. + + Args: + line (str): The line of output to process. + """ + pass + + def run(self, *args, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + ProcessHandlerMixin.run(self, *args, **kwargs) + signal.signal(signal.SIGINT, orig) diff --git a/python/mozlint/mozlint/util/string.py b/python/mozlint/mozlint/util/string.py new file mode 100644 index 0000000000..9c1c7c99c2 --- /dev/null +++ b/python/mozlint/mozlint/util/string.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def pluralize(s, num): + if num != 1: + s += "s" + return str(num) + " " + s |