summaryrefslogtreecommitdiffstats
path: root/python/mozlint
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /python/mozlint
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python/mozlint')
-rw-r--r--python/mozlint/.ruff.toml4
-rw-r--r--python/mozlint/mozlint/__init__.py7
-rw-r--r--python/mozlint/mozlint/cli.py448
-rw-r--r--python/mozlint/mozlint/editor.py57
-rw-r--r--python/mozlint/mozlint/errors.py33
-rw-r--r--python/mozlint/mozlint/formatters/__init__.py31
-rw-r--r--python/mozlint/mozlint/formatters/compact.py41
-rw-r--r--python/mozlint/mozlint/formatters/stylish.py156
-rw-r--r--python/mozlint/mozlint/formatters/summary.py50
-rw-r--r--python/mozlint/mozlint/formatters/treeherder.py34
-rw-r--r--python/mozlint/mozlint/formatters/unix.py33
-rw-r--r--python/mozlint/mozlint/parser.py130
-rw-r--r--python/mozlint/mozlint/pathutils.py316
-rw-r--r--python/mozlint/mozlint/result.py163
-rw-r--r--python/mozlint/mozlint/roller.py421
-rw-r--r--python/mozlint/mozlint/types.py214
-rw-r--r--python/mozlint/mozlint/util/__init__.py0
-rw-r--r--python/mozlint/mozlint/util/implementation.py36
-rw-r--r--python/mozlint/mozlint/util/string.py9
-rw-r--r--python/mozlint/setup.py26
-rw-r--r--python/mozlint/test/__init__.py0
-rw-r--r--python/mozlint/test/conftest.py66
-rw-r--r--python/mozlint/test/files/foobar.js2
-rw-r--r--python/mozlint/test/files/foobar.py3
-rw-r--r--python/mozlint/test/files/irrelevant/file.txt1
-rw-r--r--python/mozlint/test/files/no_foobar.js2
-rw-r--r--python/mozlint/test/filter/a.js0
-rw-r--r--python/mozlint/test/filter/a.py0
-rw-r--r--python/mozlint/test/filter/foo/empty.txt0
-rw-r--r--python/mozlint/test/filter/foobar/empty.txt0
-rw-r--r--python/mozlint/test/filter/subdir1/b.js0
-rw-r--r--python/mozlint/test/filter/subdir1/b.py0
-rw-r--r--python/mozlint/test/filter/subdir1/subdir3/d.js0
-rw-r--r--python/mozlint/test/filter/subdir1/subdir3/d.py0
-rw-r--r--python/mozlint/test/filter/subdir2/c.js0
-rw-r--r--python/mozlint/test/filter/subdir2/c.py0
-rw-r--r--python/mozlint/test/linters/badreturncode.yml8
-rw-r--r--python/mozlint/test/linters/excludes.yml10
-rw-r--r--python/mozlint/test/linters/excludes_empty.yml8
-rw-r--r--python/mozlint/test/linters/explicit_path.yml8
-rw-r--r--python/mozlint/test/linters/external.py74
-rw-r--r--python/mozlint/test/linters/external.yml8
-rw-r--r--python/mozlint/test/linters/global.yml8
-rw-r--r--python/mozlint/test/linters/global_payload.py38
-rw-r--r--python/mozlint/test/linters/global_skipped.yml8
-rw-r--r--python/mozlint/test/linters/invalid_exclude.yml6
-rw-r--r--python/mozlint/test/linters/invalid_extension.ym5
-rw-r--r--python/mozlint/test/linters/invalid_include.yml6
-rw-r--r--python/mozlint/test/linters/invalid_include_with_glob.yml6
-rw-r--r--python/mozlint/test/linters/invalid_support_files.yml6
-rw-r--r--python/mozlint/test/linters/invalid_type.yml5
-rw-r--r--python/mozlint/test/linters/missing_attrs.yml3
-rw-r--r--python/mozlint/test/linters/missing_definition.yml1
-rw-r--r--python/mozlint/test/linters/multiple.yml19
-rw-r--r--python/mozlint/test/linters/non_existing_exclude.yml7
-rw-r--r--python/mozlint/test/linters/non_existing_include.yml7
-rw-r--r--python/mozlint/test/linters/non_existing_support_files.yml7
-rw-r--r--python/mozlint/test/linters/raises.yml6
-rw-r--r--python/mozlint/test/linters/regex.yml10
-rw-r--r--python/mozlint/test/linters/setup.yml9
-rw-r--r--python/mozlint/test/linters/setupfailed.yml9
-rw-r--r--python/mozlint/test/linters/setupraised.yml9
-rw-r--r--python/mozlint/test/linters/slow.yml8
-rw-r--r--python/mozlint/test/linters/string.yml9
-rw-r--r--python/mozlint/test/linters/structured.yml8
-rw-r--r--python/mozlint/test/linters/support_files.yml10
-rw-r--r--python/mozlint/test/linters/warning.yml11
-rw-r--r--python/mozlint/test/linters/warning_no_code_review.yml12
-rw-r--r--python/mozlint/test/python.toml18
-rw-r--r--python/mozlint/test/runcli.py17
-rw-r--r--python/mozlint/test/test_cli.py126
-rw-r--r--python/mozlint/test/test_editor.py92
-rw-r--r--python/mozlint/test/test_formatters.py141
-rw-r--r--python/mozlint/test/test_parser.py80
-rw-r--r--python/mozlint/test/test_pathutils.py166
-rw-r--r--python/mozlint/test/test_result.py26
-rw-r--r--python/mozlint/test/test_roller.py396
-rw-r--r--python/mozlint/test/test_types.py84
78 files changed, 3778 insertions, 0 deletions
diff --git a/python/mozlint/.ruff.toml b/python/mozlint/.ruff.toml
new file mode 100644
index 0000000000..11e713da73
--- /dev/null
+++ b/python/mozlint/.ruff.toml
@@ -0,0 +1,4 @@
+extend = "../../pyproject.toml"
+
+[isort]
+known-first-party = ["mozlint"]
diff --git a/python/mozlint/mozlint/__init__.py b/python/mozlint/mozlint/__init__.py
new file mode 100644
index 0000000000..bcab4a48b1
--- /dev/null
+++ b/python/mozlint/mozlint/__init__.py
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# flake8: noqa
+
+from .result import Issue
+from .roller import LintRoller
diff --git a/python/mozlint/mozlint/cli.py b/python/mozlint/mozlint/cli.py
new file mode 100644
index 0000000000..36ff7b4f9a
--- /dev/null
+++ b/python/mozlint/mozlint/cli.py
@@ -0,0 +1,448 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+from argparse import REMAINDER, SUPPRESS, ArgumentParser
+from pathlib import Path
+
+from mozlint.errors import NoValidLinter
+from mozlint.formatters import all_formatters
+
+
+class MozlintParser(ArgumentParser):
+ arguments = [
+ [
+ ["paths"],
+ {
+ "nargs": "*",
+ "default": None,
+ "help": "Paths to file or directories to lint, like "
+ "'browser/components/loop' or 'mobile/android'. "
+ "If not provided, defaults to the files changed according "
+ "to --outgoing and --workdir.",
+ },
+ ],
+ [
+ ["-l", "--linter"],
+ {
+ "dest": "linters",
+ "default": [],
+ "action": "append",
+ "help": "Linters to run, e.g 'eslint'. By default all linters "
+ "are run for all the appropriate files.",
+ },
+ ],
+ [
+ ["--list"],
+ {
+ "dest": "list_linters",
+ "default": False,
+ "action": "store_true",
+ "help": "List all available linters and exit.",
+ },
+ ],
+ [
+ ["-W", "--warnings"],
+ {
+ "const": True,
+ "nargs": "?",
+ "choices": ["soft"],
+ "dest": "show_warnings",
+ "help": "Display and fail on warnings in addition to errors. "
+ "--warnings=soft can be used to report warnings but only fail "
+ "on errors.",
+ },
+ ],
+ [
+ ["-v", "--verbose"],
+ {
+ "dest": "show_verbose",
+ "default": False,
+ "action": "store_true",
+ "help": "Enable verbose logging.",
+ },
+ ],
+ [
+ ["-f", "--format"],
+ {
+ "dest": "formats",
+ "action": "append",
+ "help": "Formatter to use. Defaults to 'stylish' on stdout. "
+ "You can specify an optional path as --format formatter:path "
+ "that will be used instead of stdout. "
+ "You can also use multiple formatters at the same time. "
+ "Formatters available: {}.".format(", ".join(all_formatters.keys())),
+ },
+ ],
+ [
+ ["-n", "--no-filter"],
+ {
+ "dest": "use_filters",
+ "default": True,
+ "action": "store_false",
+ "help": "Ignore all filtering. This is useful for quickly "
+ "testing a directory that otherwise wouldn't be run, "
+ "without needing to modify the config file.",
+ },
+ ],
+ [
+ ["--include-third-party"],
+ {
+ "dest": "include_third-party",
+ "default": False,
+ "action": "store_true",
+ "help": "Also run the linter(s) on third-party code",
+ },
+ ],
+ [
+ ["-o", "--outgoing"],
+ {
+ "const": True,
+ "nargs": "?",
+ "help": "Lint files touched by commits that are not on the remote repository. "
+ "Without arguments, finds the default remote that would be pushed to. "
+ "The remote branch can also be specified manually. Works with "
+ "mercurial or git.",
+ },
+ ],
+ [
+ ["-w", "--workdir"],
+ {
+ "const": "all",
+ "nargs": "?",
+ "choices": ["staged", "all"],
+ "help": "Lint files touched by changes in the working directory "
+ "(i.e haven't been committed yet). On git, --workdir=staged "
+ "can be used to only consider staged files. Works with "
+ "mercurial or git.",
+ },
+ ],
+ [
+ ["-r", "--rev"],
+ {
+ "default": None,
+ "type": str,
+ "help": "Lint files touched by changes in revisions described by REV. "
+ "For mercurial, it may be any revset. For git, it is a single tree-ish.",
+ },
+ ],
+ [
+ ["--fix"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Fix lint errors if possible. Any errors that could not be fixed "
+ "will be printed as normal.",
+ },
+ ],
+ [
+ ["--edit"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Each file containing lint errors will be opened in $EDITOR one after "
+ "the other.",
+ },
+ ],
+ [
+ ["--setup"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Bootstrap linter dependencies without running any of the linters.",
+ },
+ ],
+ [
+ ["-j", "--jobs"],
+ {
+ "default": None,
+ "dest": "num_procs",
+ "type": int,
+ "help": "Number of worker processes to spawn when running linters. "
+ "Defaults to the number of cores in your CPU.",
+ },
+ ],
+ # Paths to check for linter configurations.
+ # Default: tools/lint set in tools/lint/mach_commands.py
+ [
+ ["--config-path"],
+ {
+ "action": "append",
+ "default": [],
+ "dest": "config_paths",
+ "help": SUPPRESS,
+ },
+ ],
+ [
+ ["--check-exclude-list"],
+ {
+ "dest": "check_exclude_list",
+ "default": False,
+ "action": "store_true",
+ "help": "Run linters for all the paths in the exclude list.",
+ },
+ ],
+ [
+ ["extra_args"],
+ {
+ "nargs": REMAINDER,
+ "help": "Extra arguments that will be forwarded to the underlying linter.",
+ },
+ ],
+ ]
+
+ def __init__(self, **kwargs):
+ ArgumentParser.__init__(self, usage=self.__doc__, **kwargs)
+
+ for cli, args in self.arguments:
+ self.add_argument(*cli, **args)
+
+ def parse_known_args(self, *args, **kwargs):
+ # Allow '-wo' or '-ow' as shorthand for both --workdir and --outgoing.
+ for token in ("-wo", "-ow"):
+ if token in args[0]:
+ i = args[0].index(token)
+ args[0].pop(i)
+ args[0][i:i] = [token[:2], "-" + token[2]]
+
+ # This is here so the eslint mach command doesn't lose 'extra_args'
+ # when using mach's dispatch functionality.
+ args, extra = ArgumentParser.parse_known_args(self, *args, **kwargs)
+ args.extra_args = extra
+
+ self.validate(args)
+ return args, extra
+
+ def validate(self, args):
+ if args.edit and not os.environ.get("EDITOR"):
+ self.error("must set the $EDITOR environment variable to use --edit")
+
+ if args.paths:
+ invalid = [p for p in args.paths if not os.path.exists(p)]
+ if invalid:
+ self.error(
+ "the following paths do not exist:\n{}".format("\n".join(invalid))
+ )
+
+ if args.formats:
+ formats = []
+ for fmt in args.formats:
+ if isinstance(fmt, tuple): # format is already processed
+ formats.append(fmt)
+ continue
+
+ path = None
+ if ":" in fmt:
+ # Detect optional formatter path
+ pos = fmt.index(":")
+ fmt, path = fmt[:pos], os.path.realpath(fmt[pos + 1 :])
+
+ # Check path is writable
+ fmt_dir = os.path.dirname(path)
+ if not os.access(fmt_dir, os.W_OK | os.X_OK):
+ self.error(
+ "the following directory is not writable: {}".format(
+ fmt_dir
+ )
+ )
+
+ if fmt not in all_formatters.keys():
+ self.error(
+ "the following formatter is not available: {}".format(fmt)
+ )
+
+ formats.append((fmt, path))
+ args.formats = formats
+ else:
+ # Can't use argparse default or this choice will be always present
+ args.formats = [("stylish", None)]
+
+
+def find_linters(config_paths, linters=None):
+ lints = {}
+ for search_path in config_paths:
+ if not os.path.isdir(search_path):
+ continue
+
+ sys.path.insert(0, search_path)
+ files = os.listdir(search_path)
+ for f in files:
+ name = os.path.basename(f)
+
+ if not name.endswith(".yml"):
+ continue
+
+ name = name.rsplit(".", 1)[0]
+
+ if linters and name not in linters:
+ continue
+
+ lints[name] = os.path.join(search_path, f)
+
+ linters_not_found = list(set(linters).difference(set(lints.keys())))
+ return {"lint_paths": lints.values(), "linters_not_found": linters_not_found}
+
+
+def get_exclude_list_output(result, paths):
+ # Store the paths of all the subdirectories leading to the error files
+ error_file_paths = set()
+ for issues in result.issues.values():
+ error_file = issues[0].relpath
+ error_file_paths.add(error_file)
+ parent_dir = os.path.dirname(error_file)
+ while parent_dir:
+ error_file_paths.add(parent_dir)
+ parent_dir = os.path.dirname(parent_dir)
+
+ paths = [os.path.dirname(path) if path[-1] == "/" else path for path in paths]
+ # Remove all the error paths to get the list of green paths
+ green_paths = sorted(set(paths).difference(error_file_paths))
+
+ if green_paths:
+ out = (
+ "The following list of paths are now green "
+ "and can be removed from the exclude list:\n\n"
+ )
+ out += "\n".join(green_paths)
+
+ else:
+ out = "No path in the exclude list is green."
+
+ return out
+
+
+def run(
+ paths,
+ linters,
+ formats,
+ outgoing,
+ workdir,
+ rev,
+ edit,
+ check_exclude_list,
+ setup=False,
+ list_linters=False,
+ num_procs=None,
+ virtualenv_manager=None,
+ setupargs=None,
+ **lintargs
+):
+ from mozlint import LintRoller, formatters
+ from mozlint.editor import edit_issues
+
+ lintargs["config_paths"] = [
+ os.path.join(lintargs["root"], p) for p in lintargs["config_paths"]
+ ]
+
+ # Always perform exhaustive linting for exclude list paths
+ lintargs["use_filters"] = lintargs["use_filters"] and not check_exclude_list
+
+ if list_linters:
+ lint_paths = find_linters(lintargs["config_paths"], linters)
+ linters = [
+ os.path.splitext(os.path.basename(l))[0] for l in lint_paths["lint_paths"]
+ ]
+ print("\n".join(sorted(linters)))
+ print(
+ "\nNote that clang-tidy checks are not run as part of this "
+ "command, but using the static-analysis command."
+ )
+ return 0
+
+ lint = LintRoller(setupargs=setupargs or {}, **lintargs)
+ linters_info = find_linters(lintargs["config_paths"], linters)
+
+ result = None
+
+ try:
+ lint.read(linters_info["lint_paths"])
+
+ if check_exclude_list:
+ if len(lint.linters) > 1:
+ print("error: specify a single linter to check with `-l/--linter`")
+ return 1
+ paths = lint.linters[0]["local_exclude"]
+
+ if (
+ not paths
+ and Path.cwd() == Path(lint.root)
+ and not (outgoing or workdir or rev)
+ ):
+ print(
+ "warning: linting the entire repo takes a long time, using --outgoing and "
+ "--workdir instead. If you want to lint the entire repo, run `./mach lint .`"
+ )
+ # Setting the default values
+ outgoing = True
+ workdir = "all"
+
+ # Always run bootstrapping, but return early if --setup was passed in.
+ ret = lint.setup(virtualenv_manager=virtualenv_manager)
+ if setup:
+ return ret
+
+ if linters_info["linters_not_found"] != []:
+ raise NoValidLinter
+
+ # run all linters
+ result = lint.roll(
+ paths, outgoing=outgoing, workdir=workdir, rev=rev, num_procs=num_procs
+ )
+ except NoValidLinter as e:
+ result = lint.result
+ print(str(e))
+
+ if edit and result.issues:
+ edit_issues(result)
+ result = lint.roll(result.issues.keys(), num_procs=num_procs)
+
+ for every in linters_info["linters_not_found"]:
+ result.failed_setup.add(every)
+
+ if check_exclude_list:
+ # Get and display all those paths in the exclude list which are
+ # now green and can be safely removed from the list
+ out = get_exclude_list_output(result, paths)
+ print(out, file=sys.stdout)
+ return result.returncode
+
+ for formatter_name, path in formats:
+ formatter = formatters.get(formatter_name)
+
+ out = formatter(result)
+ # We do this only for `json` that is mostly used in automation
+ if not out and formatter_name == "json":
+ out = "{}"
+
+ if out:
+ fh = open(path, "w") if path else sys.stdout
+
+ if not path and fh.encoding == "ascii":
+ # If sys.stdout.encoding is ascii, printing output will fail
+ # due to the stylish formatter's use of unicode characters.
+ # Ideally the user should fix their environment by setting
+ # `LC_ALL=C.UTF-8` or similar. But this is a common enough
+ # problem that we help them out a little here by manually
+ # encoding and writing to the stdout buffer directly.
+ out += "\n"
+ fh.buffer.write(out.encode("utf-8", errors="replace"))
+ fh.buffer.flush()
+ else:
+ print(out, file=fh)
+
+ if path:
+ fh.close()
+
+ return result.returncode
+
+
+def main() -> int:
+ parser = MozlintParser()
+ args = vars(parser.parse_args())
+ return run(**args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/python/mozlint/mozlint/editor.py b/python/mozlint/mozlint/editor.py
new file mode 100644
index 0000000000..1738892f93
--- /dev/null
+++ b/python/mozlint/mozlint/editor.py
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import tempfile
+
+from mozlint import formatters
+
+
+def get_editor():
+ return os.environ.get("EDITOR")
+
+
+def edit_issues(result):
+ if not result.issues:
+ return
+
+ editor = get_editor()
+ if not editor:
+ print("warning: could not find a default editor")
+ return
+
+ name = os.path.basename(editor)
+ if name in ("vim", "nvim", "gvim"):
+ cmd = [
+ editor,
+ # need errorformat to match both Error and Warning, with or without a column
+ "--cmd",
+ "set errorformat+=%f:\\ line\\ %l\\\\,\\ col\\ %c\\\\,\\ %trror\\ -\\ %m",
+ "--cmd",
+ "set errorformat+=%f:\\ line\\ %l\\\\,\\ col\\ %c\\\\,\\ %tarning\\ -\\ %m",
+ "--cmd",
+ "set errorformat+=%f:\\ line\\ %l\\\\,\\ %trror\\ -\\ %m",
+ "--cmd",
+ "set errorformat+=%f:\\ line\\ %l\\\\,\\ %tarning\\ -\\ %m",
+ # start with quickfix window opened
+ "-c",
+ "copen",
+ # running with -q seems to open an empty buffer in addition to the
+ # first file, this removes that empty buffer
+ "-c",
+ "1bd",
+ ]
+
+ with tempfile.NamedTemporaryFile(mode="w") as fh:
+ s = formatters.get("compact", summary=False)(result)
+ fh.write(s)
+ fh.flush()
+
+ cmd.extend(["-q", fh.name])
+ subprocess.call(cmd)
+
+ else:
+ for path, errors in result.issues.items():
+ subprocess.call([editor, path])
diff --git a/python/mozlint/mozlint/errors.py b/python/mozlint/mozlint/errors.py
new file mode 100644
index 0000000000..4b36f00f69
--- /dev/null
+++ b/python/mozlint/mozlint/errors.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+class LintException(Exception):
+ pass
+
+
+class LinterNotFound(LintException):
+ def __init__(self, path):
+ LintException.__init__(self, "Could not find lint file '{}'".format(path))
+
+
+class NoValidLinter(LintException):
+ def __init__(self):
+ LintException.__init__(
+ self,
+ "Invalid linters given, run again using valid linters or no linters",
+ )
+
+
+class LinterParseError(LintException):
+ def __init__(self, path, message):
+ LintException.__init__(self, "{}: {}".format(path, message))
+
+
+class LintersNotConfigured(LintException):
+ def __init__(self):
+ LintException.__init__(
+ self,
+ "No linters registered! Use `LintRoller.read` " "to register a linter.",
+ )
diff --git a/python/mozlint/mozlint/formatters/__init__.py b/python/mozlint/mozlint/formatters/__init__.py
new file mode 100644
index 0000000000..e50616216f
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/__init__.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from ..result import IssueEncoder
+from .compact import CompactFormatter
+from .stylish import StylishFormatter
+from .summary import SummaryFormatter
+from .treeherder import TreeherderFormatter
+from .unix import UnixFormatter
+
+
+class JSONFormatter(object):
+ def __call__(self, result):
+ return json.dumps(result.issues, cls=IssueEncoder)
+
+
+all_formatters = {
+ "compact": CompactFormatter,
+ "json": JSONFormatter,
+ "stylish": StylishFormatter,
+ "summary": SummaryFormatter,
+ "treeherder": TreeherderFormatter,
+ "unix": UnixFormatter,
+}
+
+
+def get(name, **fmtargs):
+ return all_formatters[name](**fmtargs)
diff --git a/python/mozlint/mozlint/formatters/compact.py b/python/mozlint/mozlint/formatters/compact.py
new file mode 100644
index 0000000000..54ee194215
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/compact.py
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import attr
+
+from ..result import Issue
+
+
+class CompactFormatter(object):
+ """Formatter for compact output.
+
+ This formatter prints one error per line, mimicking the
+ eslint 'compact' formatter.
+ """
+
+ # If modifying this format, please also update the vim errorformats in editor.py
+ fmt = "{path}: line {lineno}{column}, {level} - {message} ({rule})"
+
+ def __init__(self, summary=True):
+ self.summary = summary
+
+ def __call__(self, result):
+ message = []
+ num_problems = 0
+ for path, errors in sorted(result.issues.items()):
+ num_problems += len(errors)
+ for err in errors:
+ assert isinstance(err, Issue)
+
+ d = attr.asdict(err)
+ d["column"] = ", col %s" % d["column"] if d["column"] else ""
+ d["level"] = d["level"].capitalize()
+ d["rule"] = d["rule"] or d["linter"]
+ message.append(self.fmt.format(**d))
+
+ if self.summary and num_problems:
+ message.append(
+ "\n{} problem{}".format(num_problems, "" if num_problems == 1 else "s")
+ )
+ return "\n".join(message)
diff --git a/python/mozlint/mozlint/formatters/stylish.py b/python/mozlint/mozlint/formatters/stylish.py
new file mode 100644
index 0000000000..3f80bc7ad2
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/stylish.py
@@ -0,0 +1,156 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozterm import Terminal
+
+from ..result import Issue
+from ..util.string import pluralize
+
+
+class StylishFormatter(object):
+ """Formatter based on the eslint default."""
+
+ _indent_ = " "
+
+ # Colors later on in the list are fallbacks in case the terminal
+ # doesn't support colors earlier in the list.
+ # See http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html
+ _colors = {
+ "grey": [247, 8, 7],
+ "red": [1],
+ "green": [2],
+ "yellow": [3],
+ "brightred": [9, 1],
+ "brightyellow": [11, 3],
+ }
+
+ fmt = """
+ {c1}{lineno}{column} {c2}{level}{normal} {message} {c1}{rule}({linter}){normal}
+{diff}""".lstrip(
+ "\n"
+ )
+ fmt_summary = (
+ "{t.bold}{c}\u2716 {problem} ({error}, {warning}{failure}, {fixed}){t.normal}"
+ )
+
+ def __init__(self, disable_colors=False):
+ self.term = Terminal(disable_styling=disable_colors)
+ self.num_colors = self.term.number_of_colors
+
+ def color(self, color):
+ for num in self._colors[color]:
+ if num < self.num_colors:
+ return self.term.color(num)
+ return ""
+
+ def _reset_max(self):
+ self.max_lineno = 0
+ self.max_column = 0
+ self.max_level = 0
+ self.max_message = 0
+
+ def _update_max(self, err):
+ """Calculates the longest length of each token for spacing."""
+ self.max_lineno = max(self.max_lineno, len(str(err.lineno)))
+ if err.column:
+ self.max_column = max(self.max_column, len(str(err.column)))
+ self.max_level = max(self.max_level, len(str(err.level)))
+ self.max_message = max(self.max_message, len(err.message))
+
+ def _get_colored_diff(self, diff):
+ if not diff:
+ return ""
+
+ new_diff = ""
+ for line in diff.split("\n"):
+ if line.startswith("+"):
+ new_diff += self.color("green")
+ elif line.startswith("-"):
+ new_diff += self.color("red")
+ else:
+ new_diff += self.term.normal
+ new_diff += self._indent_ + line + "\n"
+ return new_diff
+
+ def __call__(self, result):
+ message = []
+ failed = result.failed
+
+ num_errors = 0
+ num_warnings = 0
+ num_fixed = result.fixed
+ for path, errors in sorted(result.issues.items()):
+ self._reset_max()
+
+ message.append(self.term.underline(path))
+ # Do a first pass to calculate required padding
+ for err in errors:
+ assert isinstance(err, Issue)
+ self._update_max(err)
+ if err.level == "error":
+ num_errors += 1
+ else:
+ num_warnings += 1
+
+ for err in sorted(
+ errors, key=lambda e: (int(e.lineno), int(e.column or 0))
+ ):
+ if err.column:
+ col = ":" + str(err.column).ljust(self.max_column)
+ else:
+ col = "".ljust(self.max_column + 1)
+
+ args = {
+ "normal": self.term.normal,
+ "c1": self.color("grey"),
+ "c2": self.color("red")
+ if err.level == "error"
+ else self.color("yellow"),
+ "lineno": str(err.lineno).rjust(self.max_lineno),
+ "column": col,
+ "level": err.level.ljust(self.max_level),
+ "rule": "{} ".format(err.rule) if err.rule else "",
+ "linter": err.linter.lower(),
+ "message": err.message.ljust(self.max_message),
+ "diff": self._get_colored_diff(err.diff).ljust(self.max_message),
+ }
+ message.append(self.fmt.format(**args).rstrip().rstrip("\n"))
+
+ message.append("") # newline
+
+ # If there were failures, make it clear which linters failed
+ for fail in failed:
+ message.append(
+ "{c}A failure occurred in the {name} linter.".format(
+ c=self.color("brightred"), name=fail
+ )
+ )
+
+ # Print a summary
+ message.append(
+ self.fmt_summary.format(
+ t=self.term,
+ c=self.color("brightred")
+ if num_errors or failed
+ else self.color("brightyellow"),
+ problem=pluralize("problem", num_errors + num_warnings + len(failed)),
+ error=pluralize("error", num_errors),
+ warning=pluralize(
+ "warning", num_warnings or result.total_suppressed_warnings
+ ),
+ failure=", {}".format(pluralize("failure", len(failed)))
+ if failed
+ else "",
+ fixed="{} fixed".format(num_fixed),
+ )
+ )
+
+ if result.total_suppressed_warnings > 0 and num_errors == 0:
+ message.append(
+ "(pass {c1}-W/--warnings{c2} to see warnings.)".format(
+ c1=self.color("grey"), c2=self.term.normal
+ )
+ )
+
+ return "\n".join(message)
diff --git a/python/mozlint/mozlint/formatters/summary.py b/python/mozlint/mozlint/formatters/summary.py
new file mode 100644
index 0000000000..e6ecf37508
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/summary.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from collections import defaultdict
+
+import mozpack.path as mozpath
+
+from ..util.string import pluralize
+
+
+class SummaryFormatter(object):
+ def __init__(self, depth=None):
+ self.depth = depth or int(os.environ.get("MOZLINT_SUMMARY_DEPTH", 1))
+
+ def __call__(self, result):
+ paths = set(
+ list(result.issues.keys()) + list(result.suppressed_warnings.keys())
+ )
+
+ commonprefix = mozpath.commonprefix([mozpath.abspath(p) for p in paths])
+ commonprefix = commonprefix.rsplit("/", 1)[0] + "/"
+
+ summary = defaultdict(lambda: [0, 0])
+ for path in paths:
+ abspath = mozpath.abspath(path)
+ assert abspath.startswith(commonprefix)
+
+ if abspath != commonprefix:
+ parts = mozpath.split(mozpath.relpath(abspath, commonprefix))[
+ : self.depth
+ ]
+ abspath = mozpath.join(commonprefix, *parts)
+
+ summary[abspath][0] += len(
+ [r for r in result.issues[path] if r.level == "error"]
+ )
+ summary[abspath][1] += len(
+ [r for r in result.issues[path] if r.level == "warning"]
+ )
+ summary[abspath][1] += result.suppressed_warnings[path]
+
+ msg = []
+ for path, (errors, warnings) in sorted(summary.items()):
+ warning_str = (
+ ", {}".format(pluralize("warning", warnings)) if warnings else ""
+ )
+ msg.append("{}: {}{}".format(path, pluralize("error", errors), warning_str))
+ return "\n".join(msg)
diff --git a/python/mozlint/mozlint/formatters/treeherder.py b/python/mozlint/mozlint/formatters/treeherder.py
new file mode 100644
index 0000000000..66c7c59eee
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/treeherder.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import attr
+
+from ..result import Issue
+
+
+class TreeherderFormatter(object):
+ """Formatter for treeherder friendly output.
+
+ This formatter looks ugly, but prints output such that
+ treeherder is able to highlight the errors and warnings.
+ This is a stop-gap until bug 1276486 is fixed.
+ """
+
+ fmt = "TEST-UNEXPECTED-{level} | {path}:{lineno}{column} | {message} ({rule})"
+
+ def __call__(self, result):
+ message = []
+ for path, errors in sorted(result.issues.items()):
+ for err in errors:
+ assert isinstance(err, Issue)
+
+ d = attr.asdict(err)
+ d["column"] = ":%s" % d["column"] if d["column"] else ""
+ d["level"] = d["level"].upper()
+ d["rule"] = d["rule"] or d["linter"]
+ message.append(self.fmt.format(**d))
+
+ if not message:
+ message.append("No lint issues found.")
+ return "\n".join(message)
diff --git a/python/mozlint/mozlint/formatters/unix.py b/python/mozlint/mozlint/formatters/unix.py
new file mode 100644
index 0000000000..ae096f3e2e
--- /dev/null
+++ b/python/mozlint/mozlint/formatters/unix.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import attr
+
+from ..result import Issue
+
+
+class UnixFormatter(object):
+ """Formatter that respects Unix output conventions frequently
+ employed by preprocessors and compilers. The format is
+ `<FILENAME>:<LINE>[:<COL>]: <RULE> <LEVEL>: <MESSAGE>`.
+
+ """
+
+ fmt = "{path}:{lineno}:{column} {rule} {level}: {message}"
+
+ def __call__(self, result):
+ msg = []
+
+ for path, errors in sorted(result.issues.items()):
+ for err in errors:
+ assert isinstance(err, Issue)
+
+ slots = attr.asdict(err)
+ slots["path"] = slots["relpath"]
+ slots["column"] = "%d:" % slots["column"] if slots["column"] else ""
+ slots["rule"] = slots["rule"] or slots["linter"]
+
+ msg.append(self.fmt.format(**slots))
+
+ return "\n".join(msg)
diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py
new file mode 100644
index 0000000000..eac502495b
--- /dev/null
+++ b/python/mozlint/mozlint/parser.py
@@ -0,0 +1,130 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import yaml
+
+from .errors import LinterNotFound, LinterParseError
+from .types import supported_types
+
+GLOBAL_SUPPORT_FILES = []
+
+
+class Parser(object):
+ """Reads and validates lint configuration files."""
+
+ required_attributes = (
+ "name",
+ "description",
+ "type",
+ "payload",
+ )
+
+ def __init__(self, root):
+ self.root = root
+
+ def __call__(self, path):
+ return self.parse(path)
+
+ def _validate(self, linter):
+ relpath = os.path.relpath(linter["path"], self.root)
+
+ missing_attrs = []
+ for attr in self.required_attributes:
+ if attr not in linter:
+ missing_attrs.append(attr)
+
+ if missing_attrs:
+ raise LinterParseError(
+ relpath,
+ "Missing required attribute(s): " "{}".format(",".join(missing_attrs)),
+ )
+
+ if linter["type"] not in supported_types:
+ raise LinterParseError(relpath, "Invalid type '{}'".format(linter["type"]))
+
+ for attr in ("include", "exclude", "support-files"):
+ if attr not in linter:
+ continue
+
+ if not isinstance(linter[attr], list) or not all(
+ isinstance(a, str) for a in linter[attr]
+ ):
+ raise LinterParseError(
+ relpath,
+ "The {} directive must be a " "list of strings!".format(attr),
+ )
+ invalid_paths = set()
+ for path in linter[attr]:
+ if "*" in path:
+ if attr == "include":
+ raise LinterParseError(
+ relpath,
+ "Paths in the include directive cannot "
+ "contain globs:\n {}".format(path),
+ )
+ continue
+
+ abspath = path
+ if not os.path.isabs(abspath):
+ abspath = os.path.join(self.root, path)
+
+ if not os.path.exists(abspath):
+ invalid_paths.add(" " + path)
+
+ if invalid_paths:
+ raise LinterParseError(
+ relpath,
+ "The {} directive contains the following "
+ "paths that don't exist:\n{}".format(
+ attr, "\n".join(sorted(invalid_paths))
+ ),
+ )
+
+ if "setup" in linter:
+ if linter["setup"].count(":") != 1:
+ raise LinterParseError(
+ relpath,
+ "The setup attribute '{!r}' must have the "
+ "form 'module:object'".format(linter["setup"]),
+ )
+
+ if "extensions" in linter:
+ linter["extensions"] = [e.strip(".") for e in linter["extensions"]]
+
+ def parse(self, path):
+ """Read a linter and return its LINTER definition.
+
+ :param path: Path to the linter.
+ :returns: List of linter definitions ([dict])
+ :raises: LinterNotFound, LinterParseError
+ """
+ if not os.path.isfile(path):
+ raise LinterNotFound(path)
+
+ if not path.endswith(".yml"):
+ raise LinterParseError(
+ path, "Invalid filename, linters must end with '.yml'!"
+ )
+
+ with open(path) as fh:
+ configs = list(yaml.safe_load_all(fh))
+
+ if not configs:
+ raise LinterParseError(path, "No lint definitions found!")
+
+ linters = []
+ for config in configs:
+ for name, linter in config.items():
+ linter["name"] = name
+ linter["path"] = path
+ self._validate(linter)
+ linter.setdefault("support-files", []).extend(
+ GLOBAL_SUPPORT_FILES + [path]
+ )
+ linter.setdefault("include", ["."])
+ linters.append(linter)
+
+ return linters
diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py
new file mode 100644
index 0000000000..9b46fa6d41
--- /dev/null
+++ b/python/mozlint/mozlint/pathutils.py
@@ -0,0 +1,316 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+from mozpack import path as mozpath
+from mozpack.files import FileFinder
+
+
+class FilterPath(object):
+ """Helper class to make comparing and matching file paths easier."""
+
+ def __init__(self, path):
+ self.path = os.path.normpath(path)
+ self._finder = None
+
+ @property
+ def finder(self):
+ if self._finder:
+ return self._finder
+ self._finder = FileFinder(mozpath.normsep(self.path))
+ return self._finder
+
+ @property
+ def ext(self):
+ return os.path.splitext(self.path)[1].strip(".")
+
+ @property
+ def exists(self):
+ return os.path.exists(self.path)
+
+ @property
+ def isfile(self):
+ return os.path.isfile(self.path)
+
+ @property
+ def isdir(self):
+ return os.path.isdir(self.path)
+
+ def join(self, *args):
+ return FilterPath(os.path.join(self.path, *args))
+
+ def match(self, patterns):
+ a = mozpath.normsep(self.path)
+ for p in patterns:
+ if isinstance(p, FilterPath):
+ p = p.path
+ p = mozpath.normsep(p)
+ if mozpath.match(a, p):
+ return True
+ return False
+
+ def contains(self, other):
+ """Return True if other is a subdirectory of self or equals self."""
+ if isinstance(other, FilterPath):
+ other = other.path
+ a = os.path.abspath(self.path)
+ b = os.path.normpath(os.path.abspath(other))
+
+ parts_a = a.split(os.sep)
+ parts_b = b.split(os.sep)
+
+ if len(parts_a) > len(parts_b):
+ return False
+
+ for i, part in enumerate(parts_a):
+ if part != parts_b[i]:
+ return False
+ return True
+
+ def __repr__(self):
+ return repr(self.path)
+
+
+def collapse(paths, base=None, dotfiles=False):
+ """Given an iterable of paths, collapse them into the smallest possible set
+ of paths that contain the original set (without containing any extra paths).
+
+ For example, if directory 'a' contains two files b.txt and c.txt, calling:
+
+ collapse(['a/b.txt', 'a/c.txt'])
+
+ returns ['a']. But if a third file d.txt also exists, then it will return
+ ['a/b.txt', 'a/c.txt'] since ['a'] would also include that extra file.
+
+ :param paths: An iterable of paths (files and directories) to collapse.
+ :returns: The smallest set of paths (files and directories) that contain
+ the original set of paths and only the original set.
+ """
+ if not paths:
+ if not base:
+ return []
+
+ # Need to test whether directory chain is empty. If it is then bubble
+ # the base back up so that it counts as 'covered'.
+ for _, _, names in os.walk(base):
+ if names:
+ return []
+ return [base]
+
+ if not base:
+ paths = list(map(mozpath.abspath, paths))
+ base = mozpath.commonprefix(paths).rstrip("/")
+
+ # Make sure `commonprefix` factors in sibling directories that have the
+ # same prefix in their basenames.
+ parent = mozpath.dirname(base)
+ same_prefix = [
+ p for p in os.listdir(parent) if p.startswith(mozpath.basename(base))
+ ]
+ if not os.path.isdir(base) or len(same_prefix) > 1:
+ base = parent
+
+ if base in paths:
+ return [base]
+
+ covered = set()
+ full = set()
+ for name in os.listdir(base):
+ if not dotfiles and name[0] == ".":
+ continue
+
+ path = mozpath.join(base, name)
+ full.add(path)
+
+ if path in paths:
+ # This path was explicitly specified, so just bubble it back up
+ # without recursing down into it (if it was a directory).
+ covered.add(path)
+ elif os.path.isdir(path):
+ new_paths = [p for p in paths if p.startswith(path)]
+ covered.update(collapse(new_paths, base=path, dotfiles=dotfiles))
+
+ if full == covered:
+ # Every file under this base was covered, so we can collapse them all
+ # up into the base path.
+ return [base]
+ return list(covered)
+
+
+def filterpaths(root, paths, include, exclude=None, extensions=None):
+ """Filters a list of paths.
+
+ Given a list of paths and some filtering rules, return the set of paths
+ that should be linted.
+
+ :param paths: A starting list of paths to possibly lint.
+ :param include: A list of paths that should be included (required).
+ :param exclude: A list of paths that should be excluded (optional).
+ :param extensions: A list of file extensions which should be considered (optional).
+ :returns: A tuple containing a list of file paths to lint and a list of
+ paths to exclude.
+ """
+
+ def normalize(path):
+ if "*" not in path and not os.path.isabs(path):
+ path = os.path.join(root, path)
+ return FilterPath(path)
+
+ # Includes are always paths and should always exist.
+ include = list(map(normalize, include))
+
+ # Exclude paths with and without globs will be handled separately,
+ # pull them apart now.
+ exclude = list(map(normalize, exclude or []))
+ excludepaths = [p for p in exclude if p.exists]
+ excludeglobs = [p.path for p in exclude if not p.exists]
+
+ keep = set()
+ discard = set()
+ for path in list(map(normalize, paths)):
+ # Exclude bad file extensions
+ if extensions and path.isfile and path.ext not in extensions:
+ continue
+
+ if path.match(excludeglobs):
+ continue
+
+ # First handle include/exclude directives
+ # that exist (i.e don't have globs)
+ for inc in include:
+ if inc.isfile:
+ keep.add(inc)
+
+ # Only excludes that are subdirectories of the include
+ # path matter.
+ excs = [e for e in excludepaths if inc.contains(e)]
+
+ if path.contains(inc):
+ # If specified path is an ancestor of include path,
+ # then lint the include path.
+ keep.add(inc)
+
+ # We can't apply these exclude paths without explicitly
+ # including every sibling file. Rather than do that,
+ # just return them and hope the underlying linter will
+ # deal with them.
+ discard.update(excs)
+
+ elif inc.contains(path):
+ # If the include path is an ancestor of the specified
+ # path, then add the specified path only if there are
+ # no exclude paths in-between them.
+ if not any(e.contains(path) for e in excs):
+ keep.add(path)
+ discard.update([e for e in excs if path.contains(e)])
+
+ # Next expand excludes with globs in them so we can add them to
+ # the set of files to discard.
+ for pattern in excludeglobs:
+ for p, f in path.finder.find(pattern):
+ discard.add(path.join(p))
+
+ return (
+ [f.path for f in keep if f.exists],
+ collapse([f.path for f in discard if f.exists]),
+ )
+
+
+def findobject(path):
+ """
+ Find a Python object given a path of the form <modulepath>:<objectpath>.
+ Conceptually equivalent to
+
+ def find_object(modulepath, objectpath):
+ import <modulepath> as mod
+ return mod.<objectpath>
+ """
+ if path.count(":") != 1:
+ raise ValueError(
+ 'python path {!r} does not have the form "module:object"'.format(path)
+ )
+
+ modulepath, objectpath = path.split(":")
+ obj = __import__(modulepath)
+ for a in modulepath.split(".")[1:]:
+ obj = getattr(obj, a)
+ for a in objectpath.split("."):
+ obj = getattr(obj, a)
+ return obj
+
+
+def ancestors(path):
+ while path:
+ yield path
+ (path, child) = os.path.split(path)
+ if child == "":
+ break
+
+
+def get_ancestors_by_name(name, path, root):
+ """Returns a list of files called `name` in `path`'s ancestors,
+ sorted from closest->furthest. This can be useful for finding
+ relevant configuration files.
+ """
+ configs = []
+ for path in ancestors(path):
+ config = os.path.join(path, name)
+ if os.path.isfile(config):
+ configs.append(config)
+ if path == root:
+ break
+ return configs
+
+
+def expand_exclusions(paths, config, root):
+ """Returns all files that match patterns and aren't excluded.
+
+ This is used by some external linters who receive 'batch' files (e.g dirs)
+ but aren't capable of applying their own exclusions. There is an argument
+ to be made that this step should just apply to all linters no matter what.
+
+ Args:
+ paths (list): List of candidate paths to lint.
+ config (dict): Linter's config object.
+ root (str): Root of the repository.
+
+ Returns:
+ Generator which generates list of paths that weren't excluded.
+ """
+ extensions = [e.lstrip(".") for e in config.get("extensions", [])]
+ find_dotfiles = config.get("find-dotfiles", False)
+
+ def normalize(path):
+ path = mozpath.normpath(path)
+ if os.path.isabs(path):
+ return path
+ return mozpath.join(root, path)
+
+ exclude = list(map(normalize, config.get("exclude", [])))
+ for path in paths:
+ path = mozpath.normsep(path)
+ if os.path.isfile(path):
+ if any(path.startswith(e) for e in exclude if "*" not in e):
+ continue
+
+ if any(mozpath.match(path, e) for e in exclude if "*" in e):
+ continue
+
+ yield path
+ continue
+
+ ignore = [
+ e[len(path) :].lstrip("/")
+ for e in exclude
+ if mozpath.commonprefix((path, e)) == path
+ ]
+ finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles)
+
+ _, ext = os.path.splitext(path)
+ ext.lstrip(".")
+
+ for ext in extensions:
+ for p, f in finder.find("**/*.{}".format(ext)):
+ yield os.path.join(path, p)
diff --git a/python/mozlint/mozlint/result.py b/python/mozlint/mozlint/result.py
new file mode 100644
index 0000000000..01b04afee6
--- /dev/null
+++ b/python/mozlint/mozlint/result.py
@@ -0,0 +1,163 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from collections import defaultdict
+from itertools import chain
+from json import JSONEncoder
+
+import attr
+import mozpack.path as mozpath
+
+
+class ResultSummary(object):
+ """Represents overall result state from an entire lint run."""
+
+ root = None
+
+ def __init__(self, root, fail_on_warnings=True):
+ self.fail_on_warnings = fail_on_warnings
+ self.reset()
+
+ # Store the repository root folder to be able to build
+ # Issues relative paths to that folder
+ if ResultSummary.root is None:
+ ResultSummary.root = mozpath.normpath(root)
+
+ def reset(self):
+ self.issues = defaultdict(list)
+ self.failed_run = set()
+ self.failed_setup = set()
+ self.suppressed_warnings = defaultdict(int)
+ self.fixed = 0
+
+ def has_issues_failure(self):
+ """Returns true in case issues were detected during the lint run. Do not
+ consider warning issues in case `self.fail_on_warnings` is set to False.
+ """
+ if self.fail_on_warnings is False:
+ return any(
+ result.level != "warning" for result in chain(*self.issues.values())
+ )
+ return len(self.issues) >= 1
+
+ @property
+ def returncode(self):
+ if self.has_issues_failure() or self.failed:
+ return 1
+ return 0
+
+ @property
+ def failed(self):
+ return self.failed_setup | self.failed_run
+
+ @property
+ def total_issues(self):
+ return sum([len(v) for v in self.issues.values()])
+
+ @property
+ def total_suppressed_warnings(self):
+ return sum(self.suppressed_warnings.values())
+
+ @property
+ def total_fixed(self):
+ return self.fixed
+
+ def update(self, other):
+ """Merge results from another ResultSummary into this one."""
+ for path, obj in other.issues.items():
+ self.issues[path].extend(obj)
+
+ self.failed_run |= other.failed_run
+ self.failed_setup |= other.failed_setup
+ self.fixed += other.fixed
+ for k, v in other.suppressed_warnings.items():
+ self.suppressed_warnings[k] += v
+
+
+@attr.s(slots=True, kw_only=True)
+class Issue(object):
+ """Represents a single lint issue and its related metadata.
+
+ :param linter: name of the linter that flagged this error
+ :param path: path to the file containing the error
+ :param message: text describing the error
+ :param lineno: line number that contains the error
+ :param column: column containing the error
+ :param level: severity of the error, either 'warning' or 'error' (default 'error')
+ :param hint: suggestion for fixing the error (optional)
+ :param source: source code context of the error (optional)
+ :param rule: name of the rule that was violated (optional)
+ :param lineoffset: denotes an error spans multiple lines, of the form
+ (<lineno offset>, <num lines>) (optional)
+ :param diff: a diff describing the changes that need to be made to the code
+ """
+
+ linter = attr.ib()
+ path = attr.ib()
+ lineno = attr.ib(
+ default=None, converter=lambda lineno: int(lineno) if lineno else 0
+ )
+ column = attr.ib(
+ default=None, converter=lambda column: int(column) if column else column
+ )
+ message = attr.ib()
+ hint = attr.ib(default=None)
+ source = attr.ib(default=None)
+ level = attr.ib(default=None, converter=lambda level: level or "error")
+ rule = attr.ib(default=None)
+ lineoffset = attr.ib(default=None)
+ diff = attr.ib(default=None)
+ relpath = attr.ib(init=False, default=None)
+
+ def __attrs_post_init__(self):
+ root = ResultSummary.root
+ assert root is not None, "Missing ResultSummary.root"
+ if os.path.isabs(self.path):
+ self.path = mozpath.normpath(self.path)
+ self.relpath = mozpath.relpath(self.path, root)
+ else:
+ self.relpath = mozpath.normpath(self.path)
+ self.path = mozpath.join(root, self.path)
+
+
+class IssueEncoder(JSONEncoder):
+ """Class for encoding :class:`~result.Issue` to json.
+
+ Usage:
+
+ .. code-block:: python
+
+ json.dumps(results, cls=IssueEncoder)
+
+ """
+
+ def default(self, o):
+ if isinstance(o, Issue):
+ return attr.asdict(o)
+ return JSONEncoder.default(self, o)
+
+
+def from_config(config, **kwargs):
+ """Create a :class:`~result.Issue` from a linter config.
+
+ Convenience method that pulls defaults from a linter
+ config and forwards them.
+
+ :param config: linter config as defined in a .yml file
+ :param kwargs: same as :class:`~result.Issue`
+ :returns: :class:`~result.Issue` object
+ """
+ args = {}
+ for arg in attr.fields(Issue):
+ if arg.init:
+ args[arg.name] = kwargs.get(arg.name, config.get(arg.name))
+
+ if not args["linter"]:
+ args["linter"] = config.get("name")
+
+ if not args["message"]:
+ args["message"] = config.get("description")
+
+ return Issue(**args)
diff --git a/python/mozlint/mozlint/roller.py b/python/mozlint/mozlint/roller.py
new file mode 100644
index 0000000000..1425178114
--- /dev/null
+++ b/python/mozlint/mozlint/roller.py
@@ -0,0 +1,421 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import atexit
+import copy
+import logging
+import os
+import signal
+import sys
+import time
+import traceback
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures.process import _python_exit as futures_atexit
+from itertools import chain
+from math import ceil
+from multiprocessing import cpu_count, get_context
+from multiprocessing.queues import Queue
+from subprocess import CalledProcessError
+from typing import Dict, Set
+
+import mozpack.path as mozpath
+from mozversioncontrol import (
+ InvalidRepoPath,
+ MissingUpstreamRepo,
+ get_repository_object,
+)
+
+from .errors import LintersNotConfigured, NoValidLinter
+from .parser import Parser
+from .pathutils import findobject
+from .result import ResultSummary
+from .types import supported_types
+
+SHUTDOWN = False
+orig_sigint = signal.getsignal(signal.SIGINT)
+
+logger = logging.getLogger("mozlint")
+handler = logging.StreamHandler()
+formatter = logging.Formatter(
+ "%(asctime)s.%(msecs)d %(lintname)s (%(pid)s) | %(message)s", "%H:%M:%S"
+)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+
+def _run_worker(config, paths, **lintargs):
+ log = logging.LoggerAdapter(
+ logger, {"lintname": config.get("name"), "pid": os.getpid()}
+ )
+ lintargs["log"] = log
+ result = ResultSummary(lintargs["root"])
+
+ if SHUTDOWN:
+ return result
+
+ # Override warnings setup for code review
+ # Only disactivating when code_review_warnings is set to False on a linter.yml in use
+ if os.environ.get("CODE_REVIEW") == "1" and config.get(
+ "code_review_warnings", True
+ ):
+ lintargs["show_warnings"] = True
+
+ # Override ignore thirdparty
+ # Only deactivating include_thirdparty is set on a linter.yml in use
+ if config.get("include_thirdparty", False):
+ lintargs["include_thirdparty"] = True
+
+ func = supported_types[config["type"]]
+ start_time = time.monotonic()
+ try:
+ res = func(paths, config, **lintargs)
+ # Some linters support fixed operations
+ # dict returned - {"results":results,"fixed":fixed}
+ if isinstance(res, dict):
+ result.fixed += res["fixed"]
+ res = res["results"] or []
+ elif isinstance(res, list):
+ res = res or []
+ else:
+ print("Unexpected type received")
+ assert False
+ except Exception:
+ traceback.print_exc()
+ res = 1
+ except (KeyboardInterrupt, SystemExit):
+ return result
+ finally:
+ end_time = time.monotonic()
+ log.debug("Finished in {:.2f} seconds".format(end_time - start_time))
+ sys.stdout.flush()
+
+ if not isinstance(res, (list, tuple)):
+ if res:
+ result.failed_run.add(config["name"])
+ else:
+ for r in res:
+ if not lintargs.get("show_warnings") and r.level == "warning":
+ result.suppressed_warnings[r.path] += 1
+ continue
+
+ result.issues[r.path].append(r)
+
+ return result
+
+
+class InterruptableQueue(Queue):
+ """A multiprocessing.Queue that catches KeyboardInterrupt when a worker is
+ blocking on it and returns None.
+
+ This is needed to gracefully handle KeyboardInterrupts when a worker is
+ blocking on ProcessPoolExecutor's call queue.
+ """
+
+ def __init__(self, *args, **kwargs):
+ kwargs["ctx"] = get_context()
+ super(InterruptableQueue, self).__init__(*args, **kwargs)
+
+ def get(self, *args, **kwargs):
+ try:
+ return Queue.get(self, *args, **kwargs)
+ except KeyboardInterrupt:
+ return None
+
+
+def _worker_sigint_handler(signum, frame):
+ """Sigint handler for the worker subprocesses.
+
+ Tells workers not to process the extra jobs on the call queue that couldn't
+ be canceled by the parent process.
+ """
+ global SHUTDOWN
+ SHUTDOWN = True
+ orig_sigint(signum, frame)
+
+
+def wrap_futures_atexit():
+ """Sometimes futures' atexit handler can spew tracebacks. This wrapper
+ suppresses them."""
+ try:
+ futures_atexit()
+ except Exception:
+ # Generally `atexit` handlers aren't supposed to raise exceptions, but the
+ # futures' handler can sometimes raise when the user presses `CTRL-C`. We
+ # suppress all possible exceptions here so users have a nice experience
+ # when canceling their lint run. Any exceptions raised by this function
+ # won't be useful anyway.
+ pass
+
+
+atexit.unregister(futures_atexit)
+atexit.register(wrap_futures_atexit)
+
+
+class LintRoller(object):
+ """Registers and runs linters.
+
+ :param root: Path to which relative paths will be joined. If
+ unspecified, root will either be determined from
+ version control or cwd.
+ :param lintargs: Arguments to pass to the underlying linter(s).
+ """
+
+ MAX_PATHS_PER_JOB = (
+ 50 # set a max size to prevent command lines that are too long on Windows
+ )
+
+ def __init__(self, root, exclude=None, setupargs=None, **lintargs):
+ self.parse = Parser(root)
+ try:
+ self.vcs = get_repository_object(root)
+ except InvalidRepoPath:
+ self.vcs = None
+
+ self.linters = []
+ self.lintargs = lintargs
+ self.lintargs["root"] = root
+ self._setupargs = setupargs or {}
+
+ # result state
+ self.result = ResultSummary(
+ root,
+ # Prevent failing on warnings when the --warnings parameter is set to "soft"
+ fail_on_warnings=lintargs.get("show_warnings") != "soft",
+ )
+
+ self.root = root
+ self.exclude = exclude or []
+
+ if lintargs.get("show_verbose"):
+ logger.setLevel(logging.DEBUG)
+ else:
+ logger.setLevel(logging.WARNING)
+
+ self.log = logging.LoggerAdapter(
+ logger, {"lintname": "mozlint", "pid": os.getpid()}
+ )
+
+ def read(self, paths):
+ """Parse one or more linters and add them to the registry.
+
+ :param paths: A path or iterable of paths to linter definitions.
+ """
+ if isinstance(paths, str):
+ paths = (paths,)
+
+ for linter in chain(*[self.parse(p) for p in paths]):
+ # Add only the excludes present in paths
+ linter["local_exclude"] = linter.get("exclude", [])[:]
+ # Add in our global excludes
+ linter.setdefault("exclude", []).extend(self.exclude)
+ self.linters.append(linter)
+
+ def setup(self, virtualenv_manager=None):
+ """Run setup for applicable linters"""
+ if not self.linters:
+ raise NoValidLinter
+
+ for linter in self.linters:
+ if "setup" not in linter:
+ continue
+
+ try:
+ setupargs = copy.deepcopy(self.lintargs)
+ setupargs.update(self._setupargs)
+ setupargs["name"] = linter["name"]
+ setupargs["log"] = logging.LoggerAdapter(
+ self.log, {"lintname": linter["name"]}
+ )
+ if virtualenv_manager is not None:
+ setupargs["virtualenv_manager"] = virtualenv_manager
+ start_time = time.monotonic()
+ res = findobject(linter["setup"])(
+ **setupargs,
+ )
+ self.log.debug(
+ f"setup for {linter['name']} finished in "
+ f"{round(time.monotonic() - start_time, 2)} seconds"
+ )
+ except Exception:
+ traceback.print_exc()
+ res = 1
+
+ if res:
+ self.result.failed_setup.add(linter["name"])
+
+ if self.result.failed_setup:
+ print(
+ "error: problem with lint setup, skipping {}".format(
+ ", ".join(sorted(self.result.failed_setup))
+ )
+ )
+ self.linters = [
+ l for l in self.linters if l["name"] not in self.result.failed_setup
+ ]
+ return 1
+ return 0
+
+ def should_lint_entire_tree(self, vcs_paths: Set[str], linter: Dict) -> bool:
+ """Return `True` if the linter should be run on the entire tree."""
+ # Don't lint the entire tree when `--fix` is passed to linters.
+ if "fix" in self.lintargs and self.lintargs["fix"]:
+ return False
+
+ # Lint the whole tree when a `support-file` is modified.
+ return any(
+ os.path.isfile(p) and mozpath.match(p, pattern)
+ for pattern in linter.get("support-files", [])
+ for p in vcs_paths
+ )
+
+ def _generate_jobs(self, paths, vcs_paths, num_procs):
+ def __get_current_paths(path=self.root):
+ return [os.path.join(path, p) for p in os.listdir(path)]
+
+ """A job is of the form (<linter:dict>, <paths:list>)."""
+ for linter in self.linters:
+ if self.should_lint_entire_tree(vcs_paths, linter):
+ lpaths = __get_current_paths()
+ print(
+ "warning: {} support-file modified, linting entire tree "
+ "(press ctrl-c to cancel)".format(linter["name"])
+ )
+ elif paths == {self.root}:
+ # If the command line is ".", the path will match with the root
+ # directory. In this case, get all the paths, so that we can
+ # benefit from chunking below.
+ lpaths = __get_current_paths()
+ else:
+ lpaths = paths.union(vcs_paths)
+
+ lpaths = list(lpaths) or __get_current_paths(os.getcwd())
+ chunk_size = (
+ min(self.MAX_PATHS_PER_JOB, int(ceil(len(lpaths) / num_procs))) or 1
+ )
+ if linter["type"] == "global":
+ # Global linters lint the entire tree in one job.
+ chunk_size = len(lpaths) or 1
+ assert chunk_size > 0
+
+ while lpaths:
+ yield linter, lpaths[:chunk_size]
+ lpaths = lpaths[chunk_size:]
+
+ def _collect_results(self, future):
+ if future.cancelled():
+ return
+
+ # Merge this job's results with our global ones.
+ self.result.update(future.result())
+
+ def roll(self, paths=None, outgoing=None, workdir=None, rev=None, num_procs=None):
+ """Run all of the registered linters against the specified file paths.
+
+ :param paths: An iterable of files and/or directories to lint.
+ :param outgoing: Lint files touched by commits that are not on the remote repository.
+ :param workdir: Lint all files touched in the working directory.
+ :param num_procs: The number of processes to use. Default: cpu count
+ :return: A :class:`~result.ResultSummary` instance.
+ """
+ if not self.linters:
+ raise LintersNotConfigured
+
+ self.result.reset()
+
+ # Need to use a set in case vcs operations specify the same file
+ # more than once.
+ paths = paths or set()
+ if isinstance(paths, str):
+ paths = set([paths])
+ elif isinstance(paths, (list, tuple)):
+ paths = set(paths)
+
+ if not self.vcs and (workdir or outgoing):
+ print(
+ "error: '{}' is not a known repository, can't use "
+ "--workdir or --outgoing".format(self.lintargs["root"])
+ )
+
+ # Calculate files from VCS
+ vcs_paths = set()
+ try:
+ if workdir:
+ vcs_paths.update(self.vcs.get_changed_files("AM", mode=workdir))
+ if rev:
+ vcs_paths.update(self.vcs.get_changed_files("AM", rev=rev))
+ if outgoing:
+ upstream = outgoing if isinstance(outgoing, str) else None
+ try:
+ vcs_paths.update(
+ self.vcs.get_outgoing_files("AM", upstream=upstream)
+ )
+ except MissingUpstreamRepo:
+ print(
+ "warning: could not find default push, specify a remote for --outgoing"
+ )
+ except CalledProcessError as e:
+ print("error running: {}".format(" ".join(e.cmd)))
+ if e.output:
+ print(e.output)
+
+ if not (paths or vcs_paths) and (workdir or outgoing):
+ if os.environ.get("MOZ_AUTOMATION") and not os.environ.get(
+ "PYTEST_CURRENT_TEST"
+ ):
+ raise Exception(
+ "Despite being a CI lint job, no files were linted. Is the task "
+ "missing explicit paths?"
+ )
+
+ print("warning: no files linted")
+ return self.result
+
+ # Make sure all paths are absolute. Join `paths` to cwd and `vcs_paths` to root.
+ paths = set(map(os.path.abspath, paths))
+ vcs_paths = set(
+ [
+ os.path.join(self.root, p) if not os.path.isabs(p) else p
+ for p in vcs_paths
+ ]
+ )
+
+ num_procs = num_procs or cpu_count()
+ jobs = list(self._generate_jobs(paths, vcs_paths, num_procs))
+
+ # Make sure we never spawn more processes than we have jobs.
+ num_procs = min(len(jobs), num_procs) or 1
+ if sys.platform == "win32":
+ # https://github.com/python/cpython/pull/13132
+ num_procs = min(num_procs, 61)
+
+ signal.signal(signal.SIGINT, _worker_sigint_handler)
+ executor = ProcessPoolExecutor(num_procs)
+ executor._call_queue = InterruptableQueue(executor._call_queue._maxsize)
+
+ # Submit jobs to the worker pool. The _collect_results method will be
+ # called when a job is finished. We store the futures so that they can
+ # be canceled in the event of a KeyboardInterrupt.
+ futures = []
+ for job in jobs:
+ future = executor.submit(_run_worker, *job, **self.lintargs)
+ future.add_done_callback(self._collect_results)
+ futures.append(future)
+
+ def _parent_sigint_handler(signum, frame):
+ """Sigint handler for the parent process.
+
+ Cancels all jobs that have not yet been placed on the call queue.
+ The parent process won't exit until all workers have terminated.
+ Assuming the linters are implemented properly, this shouldn't take
+ more than a couple seconds.
+ """
+ [f.cancel() for f in futures]
+ executor.shutdown(wait=True)
+ print("\nwarning: not all files were linted")
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+ signal.signal(signal.SIGINT, _parent_sigint_handler)
+ executor.shutdown()
+ signal.signal(signal.SIGINT, orig_sigint)
+ return self.result
diff --git a/python/mozlint/mozlint/types.py b/python/mozlint/mozlint/types.py
new file mode 100644
index 0000000000..1a9a0bd473
--- /dev/null
+++ b/python/mozlint/mozlint/types.py
@@ -0,0 +1,214 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+import sys
+from abc import ABCMeta, abstractmethod
+
+from mozlog import commandline, get_default_logger, structuredlog
+from mozlog.reader import LogHandler
+from mozpack.files import FileFinder
+
+from . import result
+from .pathutils import expand_exclusions, filterpaths, findobject
+
+
+class BaseType(object):
+ """Abstract base class for all types of linters."""
+
+ __metaclass__ = ABCMeta
+ batch = False
+
+ def __call__(self, paths, config, **lintargs):
+ """Run linter defined by `config` against `paths` with `lintargs`.
+
+ :param paths: Paths to lint. Can be a file or directory.
+ :param config: Linter config the paths are being linted against.
+ :param lintargs: External arguments to the linter not defined in
+ the definition, but passed in by a consumer.
+ :returns: A list of :class:`~result.Issue` objects.
+ """
+ log = lintargs["log"]
+
+ if lintargs.get("use_filters", True):
+ paths, exclude = filterpaths(
+ lintargs["root"],
+ paths,
+ config["include"],
+ config.get("exclude", []),
+ config.get("extensions", []),
+ )
+ config["exclude"] = exclude
+ elif config.get("exclude"):
+ del config["exclude"]
+
+ if not paths:
+ return {"results": [], "fixed": 0}
+
+ log.debug(
+ "Passing the following paths:\n{paths}".format(
+ paths=" \n".join(paths),
+ )
+ )
+
+ if self.batch:
+ return self._lint(paths, config, **lintargs)
+
+ errors = []
+
+ try:
+ for p in paths:
+ result = self._lint(p, config, **lintargs)
+ if result:
+ errors.extend(result)
+ except KeyboardInterrupt:
+ pass
+ return errors
+
+ def _lint_dir(self, path, config, **lintargs):
+ if not config.get("extensions"):
+ patterns = ["**"]
+ else:
+ patterns = ["**/*.{}".format(e) for e in config["extensions"]]
+
+ exclude = [os.path.relpath(e, path) for e in config.get("exclude", [])]
+ finder = FileFinder(path, ignore=exclude)
+
+ errors = []
+ for pattern in patterns:
+ for p, f in finder.find(pattern):
+ errors.extend(self._lint(os.path.join(path, p), config, **lintargs))
+ return errors
+
+ @abstractmethod
+ def _lint(self, path, config, **lintargs):
+ pass
+
+
+class LineType(BaseType):
+ """Abstract base class for linter types that check each line individually.
+
+ Subclasses of this linter type will read each file and check the provided
+ payload against each line one by one.
+ """
+
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def condition(payload, line, config):
+ pass
+
+ def _lint(self, path, config, **lintargs):
+ if os.path.isdir(path):
+ return self._lint_dir(path, config, **lintargs)
+
+ payload = config["payload"]
+ with open(path, "r", errors="replace") as fh:
+ lines = fh.readlines()
+
+ errors = []
+ for i, line in enumerate(lines):
+ if self.condition(payload, line, config):
+ errors.append(result.from_config(config, path=path, lineno=i + 1))
+
+ return errors
+
+
+class StringType(LineType):
+ """Linter type that checks whether a substring is found."""
+
+ def condition(self, payload, line, config):
+ return payload in line
+
+
+class RegexType(LineType):
+ """Linter type that checks whether a regex match is found."""
+
+ def condition(self, payload, line, config):
+ flags = 0
+ if config.get("ignore-case"):
+ flags |= re.IGNORECASE
+
+ return re.search(payload, line, flags)
+
+
+class ExternalType(BaseType):
+ """Linter type that runs an external function.
+
+ The function is responsible for properly formatting the results
+ into a list of :class:`~result.Issue` objects.
+ """
+
+ batch = True
+
+ def _lint(self, files, config, **lintargs):
+ func = findobject(config["payload"])
+ return func(files, config, **lintargs)
+
+
+class ExternalFileType(ExternalType):
+ batch = False
+
+
+class GlobalType(ExternalType):
+ """Linter type that runs an external global linting function just once.
+
+ The function is responsible for properly formatting the results
+ into a list of :class:`~result.Issue` objects.
+ """
+
+ batch = True
+
+ def _lint(self, files, config, **lintargs):
+ # Global lints are expensive to invoke. Try to avoid running
+ # them based on extensions and exclusions.
+ try:
+ next(expand_exclusions(files, config, lintargs["root"]))
+ except StopIteration:
+ return []
+
+ func = findobject(config["payload"])
+ return func(config, **lintargs)
+
+
+class LintHandler(LogHandler):
+ def __init__(self, config):
+ self.config = config
+ self.results = []
+
+ def lint(self, data):
+ self.results.append(result.from_config(self.config, **data))
+
+
+class StructuredLogType(BaseType):
+ batch = True
+
+ def _lint(self, files, config, **lintargs):
+ handler = LintHandler(config)
+ logger = config.get("logger")
+ if logger is None:
+ logger = get_default_logger()
+ if logger is None:
+ logger = structuredlog.StructuredLogger(config["name"])
+ commandline.setup_logging(logger, {}, {"mach": sys.stdout})
+ logger.add_handler(handler)
+
+ func = findobject(config["payload"])
+ try:
+ func(files, config, logger, **lintargs)
+ except KeyboardInterrupt:
+ pass
+ return handler.results
+
+
+supported_types = {
+ "string": StringType(),
+ "regex": RegexType(),
+ "external": ExternalType(),
+ "external-file": ExternalFileType(),
+ "global": GlobalType(),
+ "structured_log": StructuredLogType(),
+}
+"""Mapping of type string to an associated instance."""
diff --git a/python/mozlint/mozlint/util/__init__.py b/python/mozlint/mozlint/util/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/mozlint/util/__init__.py
diff --git a/python/mozlint/mozlint/util/implementation.py b/python/mozlint/mozlint/util/implementation.py
new file mode 100644
index 0000000000..72e160a9b0
--- /dev/null
+++ b/python/mozlint/mozlint/util/implementation.py
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import signal
+import subprocess
+from abc import ABC, abstractmethod
+
+
+class LintProcess(subprocess.Popen, ABC):
+ def __init__(self, config, *args, **kwargs):
+ self.config = config
+ self.results = []
+
+ kwargs["text"] = True
+ kwargs["stdout"] = subprocess.PIPE
+ kwargs["stderr"] = subprocess.STDOUT
+ orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+ subprocess.Popen.__init__(self, *args, **kwargs)
+ signal.signal(signal.SIGINT, orig)
+
+ @abstractmethod
+ def process_line(self, line):
+ """Process a single line of output.
+
+ The implementation is responsible for creating one or more :class:`~mozlint.result.Issue`
+ and storing them somewhere accessible.
+
+ Args:
+ line (str): The line of output to process.
+ """
+ pass
+
+ def run(self):
+ for line in self.stdout:
+ self.process_line(line)
diff --git a/python/mozlint/mozlint/util/string.py b/python/mozlint/mozlint/util/string.py
new file mode 100644
index 0000000000..9c1c7c99c2
--- /dev/null
+++ b/python/mozlint/mozlint/util/string.py
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def pluralize(s, num):
+ if num != 1:
+ s += "s"
+ return str(num) + " " + s
diff --git a/python/mozlint/setup.py b/python/mozlint/setup.py
new file mode 100644
index 0000000000..c16e15fd7f
--- /dev/null
+++ b/python/mozlint/setup.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = 0.1
+DEPS = ["mozlog >= 6.0"]
+
+setup(
+ name="mozlint",
+ description="Framework for registering and running micro lints",
+ license="MPL 2.0",
+ author="Andrew Halberstadt",
+ author_email="ahalberstadt@mozilla.com",
+ url="",
+ packages=["mozlint"],
+ version=VERSION,
+ classifiers=[
+ "Environment :: Console",
+ "Development Status :: 3 - Alpha",
+ "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+ "Natural Language :: English",
+ ],
+ install_requires=DEPS,
+)
diff --git a/python/mozlint/test/__init__.py b/python/mozlint/test/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/__init__.py
diff --git a/python/mozlint/test/conftest.py b/python/mozlint/test/conftest.py
new file mode 100644
index 0000000000..9683c23b13
--- /dev/null
+++ b/python/mozlint/test/conftest.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+from argparse import Namespace
+
+import pytest
+
+from mozlint import LintRoller
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture
+def lint(request):
+ lintargs = getattr(request.module, "lintargs", {})
+ lint = LintRoller(root=here, **lintargs)
+
+ # Add a few super powers to our lint instance
+ def mock_vcs(files):
+ def _fake_vcs_files(*args, **kwargs):
+ return files
+
+ setattr(lint.vcs, "get_changed_files", _fake_vcs_files)
+ setattr(lint.vcs, "get_outgoing_files", _fake_vcs_files)
+
+ setattr(lint, "vcs", Namespace())
+ setattr(lint, "mock_vcs", mock_vcs)
+ return lint
+
+
+@pytest.fixture(scope="session")
+def filedir():
+ return os.path.join(here, "files")
+
+
+@pytest.fixture(scope="module")
+def files(filedir, request):
+ suffix_filter = getattr(request.module, "files", [""])
+ return [
+ os.path.join(filedir, p)
+ for p in os.listdir(filedir)
+ if any(p.endswith(suffix) for suffix in suffix_filter)
+ ]
+
+
+@pytest.fixture(scope="session")
+def lintdir():
+ lintdir = os.path.join(here, "linters")
+ sys.path.insert(0, lintdir)
+ return lintdir
+
+
+@pytest.fixture(scope="module")
+def linters(lintdir):
+ def inner(*names):
+ return [
+ os.path.join(lintdir, p)
+ for p in os.listdir(lintdir)
+ if any(os.path.splitext(p)[0] == name for name in names)
+ if os.path.splitext(p)[1] == ".yml"
+ ]
+
+ return inner
diff --git a/python/mozlint/test/files/foobar.js b/python/mozlint/test/files/foobar.js
new file mode 100644
index 0000000000..d9754d0a2f
--- /dev/null
+++ b/python/mozlint/test/files/foobar.js
@@ -0,0 +1,2 @@
+// Oh no.. we called this variable foobar, bad!
+var foobar = "a string";
diff --git a/python/mozlint/test/files/foobar.py b/python/mozlint/test/files/foobar.py
new file mode 100644
index 0000000000..3b6416d211
--- /dev/null
+++ b/python/mozlint/test/files/foobar.py
@@ -0,0 +1,3 @@
+# Oh no.. we called this variable foobar, bad!
+
+foobar = "a string"
diff --git a/python/mozlint/test/files/irrelevant/file.txt b/python/mozlint/test/files/irrelevant/file.txt
new file mode 100644
index 0000000000..323fae03f4
--- /dev/null
+++ b/python/mozlint/test/files/irrelevant/file.txt
@@ -0,0 +1 @@
+foobar
diff --git a/python/mozlint/test/files/no_foobar.js b/python/mozlint/test/files/no_foobar.js
new file mode 100644
index 0000000000..6b95d646c0
--- /dev/null
+++ b/python/mozlint/test/files/no_foobar.js
@@ -0,0 +1,2 @@
+// What a relief
+var properlyNamed = "a string";
diff --git a/python/mozlint/test/filter/a.js b/python/mozlint/test/filter/a.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/a.js
diff --git a/python/mozlint/test/filter/a.py b/python/mozlint/test/filter/a.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/a.py
diff --git a/python/mozlint/test/filter/foo/empty.txt b/python/mozlint/test/filter/foo/empty.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/foo/empty.txt
diff --git a/python/mozlint/test/filter/foobar/empty.txt b/python/mozlint/test/filter/foobar/empty.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/foobar/empty.txt
diff --git a/python/mozlint/test/filter/subdir1/b.js b/python/mozlint/test/filter/subdir1/b.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir1/b.js
diff --git a/python/mozlint/test/filter/subdir1/b.py b/python/mozlint/test/filter/subdir1/b.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir1/b.py
diff --git a/python/mozlint/test/filter/subdir1/subdir3/d.js b/python/mozlint/test/filter/subdir1/subdir3/d.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir1/subdir3/d.js
diff --git a/python/mozlint/test/filter/subdir1/subdir3/d.py b/python/mozlint/test/filter/subdir1/subdir3/d.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir1/subdir3/d.py
diff --git a/python/mozlint/test/filter/subdir2/c.js b/python/mozlint/test/filter/subdir2/c.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir2/c.js
diff --git a/python/mozlint/test/filter/subdir2/c.py b/python/mozlint/test/filter/subdir2/c.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozlint/test/filter/subdir2/c.py
diff --git a/python/mozlint/test/linters/badreturncode.yml b/python/mozlint/test/linters/badreturncode.yml
new file mode 100644
index 0000000000..72abf83cc7
--- /dev/null
+++ b/python/mozlint/test/linters/badreturncode.yml
@@ -0,0 +1,8 @@
+---
+BadReturnCodeLinter:
+ description: Returns an error code no matter what
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:badreturncode
diff --git a/python/mozlint/test/linters/excludes.yml b/python/mozlint/test/linters/excludes.yml
new file mode 100644
index 0000000000..1fc1068735
--- /dev/null
+++ b/python/mozlint/test/linters/excludes.yml
@@ -0,0 +1,10 @@
+---
+ExcludesLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad
+ rule: no-foobar
+ exclude: ['**/foobar.js']
+ extensions: ['.js', 'jsm']
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/excludes_empty.yml b/python/mozlint/test/linters/excludes_empty.yml
new file mode 100644
index 0000000000..03cd1aecab
--- /dev/null
+++ b/python/mozlint/test/linters/excludes_empty.yml
@@ -0,0 +1,8 @@
+---
+ExcludesEmptyLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: foobar
diff --git a/python/mozlint/test/linters/explicit_path.yml b/python/mozlint/test/linters/explicit_path.yml
new file mode 100644
index 0000000000..1e7e8f4bf1
--- /dev/null
+++ b/python/mozlint/test/linters/explicit_path.yml
@@ -0,0 +1,8 @@
+---
+ExplicitPathLinter:
+ description: Only lint a specific file name
+ rule: no-foobar
+ include:
+ - files/no_foobar.js
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/external.py b/python/mozlint/test/linters/external.py
new file mode 100644
index 0000000000..9c2e58909d
--- /dev/null
+++ b/python/mozlint/test/linters/external.py
@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import time
+
+from mozlint import result
+from mozlint.errors import LintException
+
+
+def badreturncode(files, config, **lintargs):
+ return 1
+
+
+def external(files, config, **lintargs):
+ if lintargs.get("fix"):
+ # mimics no results because they got fixed
+ return []
+
+ results = []
+ for path in files:
+ if os.path.isdir(path):
+ continue
+
+ with open(path, "r") as fh:
+ for i, line in enumerate(fh.readlines()):
+ if "foobar" in line:
+ results.append(
+ result.from_config(
+ config, path=path, lineno=i + 1, column=1, rule="no-foobar"
+ )
+ )
+ return results
+
+
+def raises(files, config, **lintargs):
+ raise LintException("Oh no something bad happened!")
+
+
+def slow(files, config, **lintargs):
+ time.sleep(2)
+ return []
+
+
+def structured(files, config, logger, **kwargs):
+ for path in files:
+ if os.path.isdir(path):
+ continue
+
+ with open(path, "r") as fh:
+ for i, line in enumerate(fh.readlines()):
+ if "foobar" in line:
+ logger.lint_error(
+ path=path, lineno=i + 1, column=1, rule="no-foobar"
+ )
+
+
+def passes(files, config, **lintargs):
+ return []
+
+
+def setup(**lintargs):
+ print("setup passed")
+
+
+def setupfailed(**lintargs):
+ print("setup failed")
+ return 1
+
+
+def setupraised(**lintargs):
+ print("setup raised")
+ raise LintException("oh no setup failed")
diff --git a/python/mozlint/test/linters/external.yml b/python/mozlint/test/linters/external.yml
new file mode 100644
index 0000000000..574b8df4cb
--- /dev/null
+++ b/python/mozlint/test/linters/external.yml
@@ -0,0 +1,8 @@
+---
+ExternalLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:external
diff --git a/python/mozlint/test/linters/global.yml b/python/mozlint/test/linters/global.yml
new file mode 100644
index 0000000000..47d5ce81e4
--- /dev/null
+++ b/python/mozlint/test/linters/global.yml
@@ -0,0 +1,8 @@
+---
+GlobalLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: global
+ extensions: ['.js', '.jsm']
+ payload: global_payload:global_payload
diff --git a/python/mozlint/test/linters/global_payload.py b/python/mozlint/test/linters/global_payload.py
new file mode 100644
index 0000000000..ec620b6af1
--- /dev/null
+++ b/python/mozlint/test/linters/global_payload.py
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozpack.path as mozpath
+from external import external
+from mozpack.files import FileFinder
+
+from mozlint import result
+
+
+def global_payload(config, **lintargs):
+ # A global linter that runs the external linter to actually lint.
+ finder = FileFinder(lintargs["root"])
+ files = [mozpath.join(lintargs["root"], p) for p, _ in finder.find("files/**")]
+ issues = external(files, config, **lintargs)
+ for issue in issues:
+ # Make issue look like it comes from this linter.
+ issue.linter = "global_payload"
+ return issues
+
+
+def global_skipped(config, **lintargs):
+ # A global linter that always registers a lint error. Absence of
+ # this error shows that the path exclusion mechanism can cause
+ # global lint payloads to not be invoked at all. In particular,
+ # the `extensions` field means that nothing under `files/**` will
+ # match.
+
+ finder = FileFinder(lintargs["root"])
+ files = [mozpath.join(lintargs["root"], p) for p, _ in finder.find("files/**")]
+
+ issues = []
+ issues.append(
+ result.from_config(
+ config, path=files[0], lineno=1, column=1, rule="not-skipped"
+ )
+ )
diff --git a/python/mozlint/test/linters/global_skipped.yml b/python/mozlint/test/linters/global_skipped.yml
new file mode 100644
index 0000000000..99b784e8be
--- /dev/null
+++ b/python/mozlint/test/linters/global_skipped.yml
@@ -0,0 +1,8 @@
+---
+GlobalSkippedLinter:
+ description: It's bad to run global linters when nothing matches.
+ include:
+ - files
+ type: global
+ extensions: ['.non.existent.extension']
+ payload: global_payload:global_skipped
diff --git a/python/mozlint/test/linters/invalid_exclude.yml b/python/mozlint/test/linters/invalid_exclude.yml
new file mode 100644
index 0000000000..7231d2c146
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_exclude.yml
@@ -0,0 +1,6 @@
+---
+BadExcludeLinter:
+ description: Has an invalid exclude directive.
+ exclude: [0, 1] # should be a list of strings
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/invalid_extension.ym b/python/mozlint/test/linters/invalid_extension.ym
new file mode 100644
index 0000000000..435fa10320
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_extension.ym
@@ -0,0 +1,5 @@
+---
+BadExtensionLinter:
+ description: Has an invalid file extension.
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/invalid_include.yml b/python/mozlint/test/linters/invalid_include.yml
new file mode 100644
index 0000000000..b76b3e6a61
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_include.yml
@@ -0,0 +1,6 @@
+---
+BadIncludeLinter:
+ description: Has an invalid include directive.
+ include: should be a list
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/invalid_include_with_glob.yml b/python/mozlint/test/linters/invalid_include_with_glob.yml
new file mode 100644
index 0000000000..857bb1376b
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_include_with_glob.yml
@@ -0,0 +1,6 @@
+---
+BadIncludeLinterWithGlob:
+ description: Has an invalid include directive.
+ include: ['**/*.js']
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/invalid_support_files.yml b/python/mozlint/test/linters/invalid_support_files.yml
new file mode 100644
index 0000000000..db39597d68
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_support_files.yml
@@ -0,0 +1,6 @@
+---
+BadSupportFilesLinter:
+ description: Has an invalid support files directive.
+ support-files: should be a list
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/invalid_type.yml b/python/mozlint/test/linters/invalid_type.yml
new file mode 100644
index 0000000000..29d82e541e
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_type.yml
@@ -0,0 +1,5 @@
+---
+BadTypeLinter:
+ description: Has an invalid type.
+ type: invalid
+ payload: foobar
diff --git a/python/mozlint/test/linters/missing_attrs.yml b/python/mozlint/test/linters/missing_attrs.yml
new file mode 100644
index 0000000000..5abe15fcfc
--- /dev/null
+++ b/python/mozlint/test/linters/missing_attrs.yml
@@ -0,0 +1,3 @@
+---
+MissingAttrsLinter:
+ description: Missing type and payload
diff --git a/python/mozlint/test/linters/missing_definition.yml b/python/mozlint/test/linters/missing_definition.yml
new file mode 100644
index 0000000000..d66b2cb781
--- /dev/null
+++ b/python/mozlint/test/linters/missing_definition.yml
@@ -0,0 +1 @@
+# No definition
diff --git a/python/mozlint/test/linters/multiple.yml b/python/mozlint/test/linters/multiple.yml
new file mode 100644
index 0000000000..5b880b3691
--- /dev/null
+++ b/python/mozlint/test/linters/multiple.yml
@@ -0,0 +1,19 @@
+---
+StringLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad
+ rule: no-foobar
+ extensions: ['.js', 'jsm']
+ type: string
+ payload: foobar
+
+---
+RegexLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad
+ rule: no-foobar
+ extensions: ['.js', 'jsm']
+ type: regex
+ payload: foobar
diff --git a/python/mozlint/test/linters/non_existing_exclude.yml b/python/mozlint/test/linters/non_existing_exclude.yml
new file mode 100644
index 0000000000..8190123027
--- /dev/null
+++ b/python/mozlint/test/linters/non_existing_exclude.yml
@@ -0,0 +1,7 @@
+---
+BadExcludeLinter:
+ description: Has an invalid exclude directive.
+ exclude:
+ - files/does_not_exist
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/non_existing_include.yml b/python/mozlint/test/linters/non_existing_include.yml
new file mode 100644
index 0000000000..5443d751ed
--- /dev/null
+++ b/python/mozlint/test/linters/non_existing_include.yml
@@ -0,0 +1,7 @@
+---
+BadIncludeLinter:
+ description: Has an invalid include directive.
+ include:
+ - files/does_not_exist
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/non_existing_support_files.yml b/python/mozlint/test/linters/non_existing_support_files.yml
new file mode 100644
index 0000000000..e636fadf93
--- /dev/null
+++ b/python/mozlint/test/linters/non_existing_support_files.yml
@@ -0,0 +1,7 @@
+---
+BadSupportFilesLinter:
+ description: Has an invalid support-files directive.
+ support-files:
+ - files/does_not_exist
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/raises.yml b/python/mozlint/test/linters/raises.yml
new file mode 100644
index 0000000000..9c0b234779
--- /dev/null
+++ b/python/mozlint/test/linters/raises.yml
@@ -0,0 +1,6 @@
+---
+RaisesLinter:
+ description: Raises an exception
+ include: ['.']
+ type: external
+ payload: external:raises
diff --git a/python/mozlint/test/linters/regex.yml b/python/mozlint/test/linters/regex.yml
new file mode 100644
index 0000000000..2c9c812428
--- /dev/null
+++ b/python/mozlint/test/linters/regex.yml
@@ -0,0 +1,10 @@
+---
+RegexLinter:
+ description: >-
+ Make sure the string foobar never appears in a js variable
+ file because it is bad.
+ rule: no-foobar
+ include: ['.']
+ extensions: ['js', '.jsm']
+ type: regex
+ payload: foobar
diff --git a/python/mozlint/test/linters/setup.yml b/python/mozlint/test/linters/setup.yml
new file mode 100644
index 0000000000..ac75d72c70
--- /dev/null
+++ b/python/mozlint/test/linters/setup.yml
@@ -0,0 +1,9 @@
+---
+SetupLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:external
+ setup: external:setup
diff --git a/python/mozlint/test/linters/setupfailed.yml b/python/mozlint/test/linters/setupfailed.yml
new file mode 100644
index 0000000000..1e3543286f
--- /dev/null
+++ b/python/mozlint/test/linters/setupfailed.yml
@@ -0,0 +1,9 @@
+---
+SetupFailedLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:external
+ setup: external:setupfailed
diff --git a/python/mozlint/test/linters/setupraised.yml b/python/mozlint/test/linters/setupraised.yml
new file mode 100644
index 0000000000..8c987f2d3c
--- /dev/null
+++ b/python/mozlint/test/linters/setupraised.yml
@@ -0,0 +1,9 @@
+---
+SetupRaisedLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:external
+ setup: external:setupraised
diff --git a/python/mozlint/test/linters/slow.yml b/python/mozlint/test/linters/slow.yml
new file mode 100644
index 0000000000..2c47679010
--- /dev/null
+++ b/python/mozlint/test/linters/slow.yml
@@ -0,0 +1,8 @@
+---
+SlowLinter:
+ description: A linter that takes awhile to run
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:slow
diff --git a/python/mozlint/test/linters/string.yml b/python/mozlint/test/linters/string.yml
new file mode 100644
index 0000000000..836d866ae2
--- /dev/null
+++ b/python/mozlint/test/linters/string.yml
@@ -0,0 +1,9 @@
+---
+StringLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad
+ rule: no-foobar
+ extensions: ['.js', 'jsm']
+ type: string
+ payload: foobar
diff --git a/python/mozlint/test/linters/structured.yml b/python/mozlint/test/linters/structured.yml
new file mode 100644
index 0000000000..01ef447ee3
--- /dev/null
+++ b/python/mozlint/test/linters/structured.yml
@@ -0,0 +1,8 @@
+---
+StructuredLinter:
+ description: "It's bad to have the string foobar in js files."
+ include:
+ - files
+ type: structured_log
+ extensions: ['.js', '.jsm']
+ payload: external:structured
diff --git a/python/mozlint/test/linters/support_files.yml b/python/mozlint/test/linters/support_files.yml
new file mode 100644
index 0000000000..0c278d51fa
--- /dev/null
+++ b/python/mozlint/test/linters/support_files.yml
@@ -0,0 +1,10 @@
+---
+SupportFilesLinter:
+ description: Linter that has a few support files
+ include:
+ - files
+ support-files:
+ - '**/*.py'
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:passes
diff --git a/python/mozlint/test/linters/warning.yml b/python/mozlint/test/linters/warning.yml
new file mode 100644
index 0000000000..b86bfd07c7
--- /dev/null
+++ b/python/mozlint/test/linters/warning.yml
@@ -0,0 +1,11 @@
+---
+WarningLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad, but not too bad (just a warning)
+ rule: no-foobar
+ level: warning
+ include: ['.']
+ type: string
+ extensions: ['.js', 'jsm']
+ payload: foobar
diff --git a/python/mozlint/test/linters/warning_no_code_review.yml b/python/mozlint/test/linters/warning_no_code_review.yml
new file mode 100644
index 0000000000..20bfc0503b
--- /dev/null
+++ b/python/mozlint/test/linters/warning_no_code_review.yml
@@ -0,0 +1,12 @@
+---
+WarningNoCodeReviewLinter:
+ description: >-
+ Make sure the string foobar never appears in browser js
+ files because it is bad, but not too bad (just a warning)
+ rule: no-foobar-no-code-review
+ level: warning
+ include: ['.']
+ type: string
+ extensions: ['.js', 'jsm']
+ payload: foobar
+ code_review_warnings: false
diff --git a/python/mozlint/test/python.toml b/python/mozlint/test/python.toml
new file mode 100644
index 0000000000..3876a4aaa4
--- /dev/null
+++ b/python/mozlint/test/python.toml
@@ -0,0 +1,18 @@
+[DEFAULT]
+subsuite = "mozlint"
+
+["test_cli.py"]
+
+["test_editor.py"]
+
+["test_formatters.py"]
+
+["test_parser.py"]
+
+["test_pathutils.py"]
+
+["test_result.py"]
+
+["test_roller.py"]
+
+["test_types.py"]
diff --git a/python/mozlint/test/runcli.py b/python/mozlint/test/runcli.py
new file mode 100644
index 0000000000..be60a1da19
--- /dev/null
+++ b/python/mozlint/test/runcli.py
@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+from mozlint import cli
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+if __name__ == "__main__":
+ parser = cli.MozlintParser()
+ args = vars(parser.parse_args(sys.argv[1:]))
+ args["root"] = here
+ args["config_paths"] = [os.path.join(here, "linters")]
+ sys.exit(cli.run(**args))
diff --git a/python/mozlint/test/test_cli.py b/python/mozlint/test/test_cli.py
new file mode 100644
index 0000000000..4e9219a2ea
--- /dev/null
+++ b/python/mozlint/test/test_cli.py
@@ -0,0 +1,126 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import sys
+from distutils.spawn import find_executable
+
+import mozunit
+import pytest
+
+from mozlint import cli
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture
+def parser():
+ return cli.MozlintParser()
+
+
+@pytest.fixture
+def run(parser, lintdir, files):
+ def inner(args=None):
+ args = args or []
+ args.extend(files)
+ lintargs = vars(parser.parse_args(args))
+ lintargs["root"] = here
+ lintargs["config_paths"] = [os.path.join(here, "linters")]
+ return cli.run(**lintargs)
+
+ return inner
+
+
+def test_cli_with_ascii_encoding(run, monkeypatch, capfd):
+ cmd = [sys.executable, "runcli.py", "-l=string", "-f=stylish", "files/foobar.js"]
+ env = os.environ.copy()
+ env["PYTHONPATH"] = os.pathsep.join(sys.path)
+ env["PYTHONIOENCODING"] = "ascii"
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ cwd=here,
+ env=env,
+ universal_newlines=True,
+ )
+ out = proc.communicate()[0]
+ assert "Traceback" not in out
+
+
+def test_cli_run_with_fix(run, capfd):
+ ret = run(["-f", "json", "--fix", "--linter", "external"])
+ out, err = capfd.readouterr()
+ assert ret == 0
+ assert out.endswith("{}\n")
+
+
+@pytest.mark.skipif(not find_executable("echo"), reason="No `echo` executable found.")
+def test_cli_run_with_edit(run, parser, capfd):
+ os.environ["EDITOR"] = "echo"
+
+ ret = run(["-f", "compact", "--edit", "--linter", "external"])
+ out, err = capfd.readouterr()
+ out = out.splitlines()
+ assert ret == 1
+ assert out[0].endswith("foobar.js") # from the `echo` editor
+ assert "foobar.js: line 1, col 1, Error" in out[1]
+ assert "foobar.js: line 2, col 1, Error" in out[2]
+ assert "2 problems" in out[-1]
+ assert len(out) == 5
+
+ del os.environ["EDITOR"]
+ with pytest.raises(SystemExit):
+ parser.parse_args(["--edit"])
+
+
+def test_cli_run_with_setup(run, capfd):
+ # implicitly call setup
+ ret = run(["-l", "setup", "-l", "setupfailed", "-l", "setupraised"])
+ out, err = capfd.readouterr()
+ assert "setup passed" in out
+ assert "setup failed" in out
+ assert "setup raised" in out
+ assert ret == 1
+
+ # explicitly call setup
+ ret = run(["-l", "setup", "-l", "setupfailed", "-l", "setupraised", "--setup"])
+ out, err = capfd.readouterr()
+ assert "setup passed" in out
+ assert "setup failed" in out
+ assert "setup raised" in out
+ assert ret == 1
+
+
+def test_cli_for_exclude_list(run, monkeypatch, capfd):
+ ret = run(["-l", "excludes", "--check-exclude-list"])
+ out, err = capfd.readouterr()
+
+ assert "**/foobar.js" in out
+ assert (
+ "The following list of paths are now green and can be removed from the exclude list:"
+ in out
+ )
+
+ ret = run(["-l", "excludes_empty", "--check-exclude-list"])
+ out, err = capfd.readouterr()
+
+ assert "No path in the exclude list is green." in out
+ assert ret == 1
+
+
+def test_cli_run_with_wrong_linters(run, capfd):
+ run(["-l", "external", "-l", "foobar"])
+ out, err = capfd.readouterr()
+
+ # Check if it identifies foobar as invalid linter
+ assert "A failure occurred in the foobar linter." in out
+
+ # Check for exception message
+ assert "Invalid linters given, run again using valid linters or no linters" in out
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_editor.py b/python/mozlint/test/test_editor.py
new file mode 100644
index 0000000000..7a15a613a6
--- /dev/null
+++ b/python/mozlint/test/test_editor.py
@@ -0,0 +1,92 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+
+import mozunit
+import pytest
+
+from mozlint import editor
+from mozlint.result import Issue, ResultSummary
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture
+def capture_commands(monkeypatch):
+ def inner(commands):
+ def fake_subprocess_call(*args, **kwargs):
+ commands.append(args[0])
+
+ monkeypatch.setattr(subprocess, "call", fake_subprocess_call)
+
+ return inner
+
+
+@pytest.fixture
+def result():
+ result = ResultSummary("/fake/root")
+ result.issues["foo.py"].extend(
+ [
+ Issue(
+ linter="no-foobar",
+ path="foo.py",
+ lineno=1,
+ message="Oh no!",
+ ),
+ Issue(
+ linter="no-foobar",
+ path="foo.py",
+ lineno=3,
+ column=10,
+ message="To Yuma!",
+ ),
+ ]
+ )
+ return result
+
+
+def test_no_editor(monkeypatch, capture_commands, result):
+ commands = []
+ capture_commands(commands)
+
+ monkeypatch.delenv("EDITOR", raising=False)
+ editor.edit_issues(result)
+ assert commands == []
+
+
+def test_no_issues(monkeypatch, capture_commands, result):
+ commands = []
+ capture_commands(commands)
+
+ monkeypatch.setenv("EDITOR", "generic")
+ result.issues = {}
+ editor.edit_issues(result)
+ assert commands == []
+
+
+def test_vim(monkeypatch, capture_commands, result):
+ commands = []
+ capture_commands(commands)
+
+ monkeypatch.setenv("EDITOR", "vim")
+ editor.edit_issues(result)
+ assert len(commands) == 1
+ assert commands[0][0] == "vim"
+
+
+def test_generic(monkeypatch, capture_commands, result):
+ commands = []
+ capture_commands(commands)
+
+ monkeypatch.setenv("EDITOR", "generic")
+ editor.edit_issues(result)
+ assert len(commands) == len(result.issues)
+ assert all(c[0] == "generic" for c in commands)
+ assert all("foo.py" in c for c in commands)
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_formatters.py b/python/mozlint/test/test_formatters.py
new file mode 100644
index 0000000000..5a276a1c23
--- /dev/null
+++ b/python/mozlint/test/test_formatters.py
@@ -0,0 +1,141 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+import attr
+import mozpack.path as mozpath
+import mozunit
+import pytest
+
+from mozlint import formatters
+from mozlint.result import Issue, ResultSummary
+
+NORMALISED_PATHS = {
+ "abc": mozpath.normpath("a/b/c.txt"),
+ "def": mozpath.normpath("d/e/f.txt"),
+ "root": mozpath.abspath("/fake/root"),
+}
+
+EXPECTED = {
+ "compact": {
+ "kwargs": {},
+ "format": """
+/fake/root/a/b/c.txt: line 1, Error - oh no foo (foo)
+/fake/root/a/b/c.txt: line 4, col 10, Error - oh no baz (baz)
+/fake/root/a/b/c.txt: line 5, Error - oh no foo-diff (foo-diff)
+/fake/root/d/e/f.txt: line 4, col 2, Warning - oh no bar (bar-not-allowed)
+
+4 problems
+""".strip(),
+ },
+ "stylish": {
+ "kwargs": {"disable_colors": True},
+ "format": """
+/fake/root/a/b/c.txt
+ 1 error oh no foo (foo)
+ 4:10 error oh no baz (baz)
+ 5 error oh no foo-diff (foo-diff)
+ diff 1
+ - hello
+ + hello2
+
+/fake/root/d/e/f.txt
+ 4:2 warning oh no bar bar-not-allowed (bar)
+
+\u2716 4 problems (3 errors, 1 warning, 0 fixed)
+""".strip(),
+ },
+ "treeherder": {
+ "kwargs": {},
+ "format": """
+TEST-UNEXPECTED-ERROR | /fake/root/a/b/c.txt:1 | oh no foo (foo)
+TEST-UNEXPECTED-ERROR | /fake/root/a/b/c.txt:4:10 | oh no baz (baz)
+TEST-UNEXPECTED-ERROR | /fake/root/a/b/c.txt:5 | oh no foo-diff (foo-diff)
+TEST-UNEXPECTED-WARNING | /fake/root/d/e/f.txt:4:2 | oh no bar (bar-not-allowed)
+""".strip(),
+ },
+ "unix": {
+ "kwargs": {},
+ "format": """
+{abc}:1: foo error: oh no foo
+{abc}:4:10: baz error: oh no baz
+{abc}:5: foo-diff error: oh no foo-diff
+{def}:4:2: bar-not-allowed warning: oh no bar
+""".format(
+ **NORMALISED_PATHS
+ ).strip(),
+ },
+ "summary": {
+ "kwargs": {},
+ "format": """
+{root}/a: 3 errors
+{root}/d: 0 errors, 1 warning
+""".format(
+ **NORMALISED_PATHS
+ ).strip(),
+ },
+}
+
+
+@pytest.fixture
+def result(scope="module"):
+ result = ResultSummary("/fake/root")
+ containers = (
+ Issue(linter="foo", path="a/b/c.txt", message="oh no foo", lineno=1),
+ Issue(
+ linter="bar",
+ path="d/e/f.txt",
+ message="oh no bar",
+ hint="try baz instead",
+ level="warning",
+ lineno="4",
+ column="2",
+ rule="bar-not-allowed",
+ ),
+ Issue(
+ linter="baz",
+ path="a/b/c.txt",
+ message="oh no baz",
+ lineno=4,
+ column=10,
+ source="if baz:",
+ ),
+ Issue(
+ linter="foo-diff",
+ path="a/b/c.txt",
+ message="oh no foo-diff",
+ lineno=5,
+ source="if baz:",
+ diff="diff 1\n- hello\n+ hello2",
+ ),
+ )
+ result = ResultSummary("/fake/root")
+ for c in containers:
+ result.issues[c.path].append(c)
+ return result
+
+
+@pytest.mark.parametrize("name", EXPECTED.keys())
+def test_formatters(result, name):
+ opts = EXPECTED[name]
+ fmt = formatters.get(name, **opts["kwargs"])
+ # encoding to str bypasses a UnicodeEncodeError in pytest
+ assert fmt(result) == opts["format"]
+
+
+def test_json_formatter(result):
+ fmt = formatters.get("json")
+ formatted = json.loads(fmt(result))
+
+ assert set(formatted.keys()) == set(result.issues.keys())
+
+ attrs = attr.fields(Issue)
+ for errors in formatted.values():
+ for err in errors:
+ assert all(a.name in err for a in attrs)
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_parser.py b/python/mozlint/test/test_parser.py
new file mode 100644
index 0000000000..2fbf26c8e5
--- /dev/null
+++ b/python/mozlint/test/test_parser.py
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import mozunit
+import pytest
+
+from mozlint.errors import LinterNotFound, LinterParseError
+from mozlint.parser import Parser
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture(scope="module")
+def parse(lintdir):
+ parser = Parser(here)
+
+ def _parse(name):
+ path = os.path.join(lintdir, name)
+ return parser(path)
+
+ return _parse
+
+
+def test_parse_valid_linter(parse):
+ lintobj = parse("string.yml")
+ assert isinstance(lintobj, list)
+ assert len(lintobj) == 1
+
+ lintobj = lintobj[0]
+ assert isinstance(lintobj, dict)
+ assert "name" in lintobj
+ assert "description" in lintobj
+ assert "type" in lintobj
+ assert "payload" in lintobj
+ assert "extensions" in lintobj
+ assert "include" in lintobj
+ assert lintobj["include"] == ["."]
+ assert set(lintobj["extensions"]) == set(["js", "jsm"])
+
+
+def test_parser_valid_multiple(parse):
+ lintobj = parse("multiple.yml")
+ assert isinstance(lintobj, list)
+ assert len(lintobj) == 2
+
+ assert lintobj[0]["name"] == "StringLinter"
+ assert lintobj[1]["name"] == "RegexLinter"
+
+
+@pytest.mark.parametrize(
+ "linter",
+ [
+ "invalid_type.yml",
+ "invalid_extension.ym",
+ "invalid_include.yml",
+ "invalid_include_with_glob.yml",
+ "invalid_exclude.yml",
+ "invalid_support_files.yml",
+ "missing_attrs.yml",
+ "missing_definition.yml",
+ "non_existing_include.yml",
+ "non_existing_exclude.yml",
+ "non_existing_support_files.yml",
+ ],
+)
+def test_parse_invalid_linter(parse, linter):
+ with pytest.raises(LinterParseError):
+ parse(linter)
+
+
+def test_parse_non_existent_linter(parse):
+ with pytest.raises(LinterNotFound):
+ parse("missing_file.lint")
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_pathutils.py b/python/mozlint/test/test_pathutils.py
new file mode 100644
index 0000000000..78f7883e88
--- /dev/null
+++ b/python/mozlint/test/test_pathutils.py
@@ -0,0 +1,166 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from fnmatch import fnmatch
+
+import mozunit
+import pytest
+
+from mozlint import pathutils
+
+here = os.path.abspath(os.path.dirname(__file__))
+root = os.path.join(here, "filter")
+
+
+def assert_paths(a, b):
+ def normalize(p):
+ if not os.path.isabs(p):
+ p = os.path.join(root, p)
+ return os.path.normpath(p)
+
+ assert set(map(normalize, a)) == set(map(normalize, b))
+
+
+@pytest.mark.parametrize(
+ "test",
+ (
+ {
+ "paths": ["a.js", "subdir1/subdir3/d.js"],
+ "include": ["."],
+ "exclude": ["subdir1"],
+ "expected": ["a.js"],
+ },
+ {
+ "paths": ["a.js", "subdir1/subdir3/d.js"],
+ "include": ["subdir1/subdir3"],
+ "exclude": ["subdir1"],
+ "expected": ["subdir1/subdir3/d.js"],
+ },
+ {
+ "paths": ["."],
+ "include": ["."],
+ "exclude": ["**/c.py", "subdir1/subdir3"],
+ "extensions": ["py"],
+ "expected": ["."],
+ "expected_exclude": ["subdir2/c.py", "subdir1/subdir3"],
+ },
+ {
+ "paths": [
+ "a.py",
+ "a.js",
+ "subdir1/b.py",
+ "subdir2/c.py",
+ "subdir1/subdir3/d.py",
+ ],
+ "include": ["."],
+ "exclude": ["**/c.py", "subdir1/subdir3"],
+ "extensions": ["py"],
+ "expected": ["a.py", "subdir1/b.py"],
+ },
+ {
+ "paths": ["a.py", "a.js", "subdir2"],
+ "include": ["."],
+ "exclude": [],
+ "extensions": ["py"],
+ "expected": ["a.py", "subdir2"],
+ },
+ {
+ "paths": ["subdir1"],
+ "include": ["."],
+ "exclude": ["subdir1/subdir3"],
+ "extensions": ["py"],
+ "expected": ["subdir1"],
+ "expected_exclude": ["subdir1/subdir3"],
+ },
+ {
+ "paths": ["docshell"],
+ "include": ["docs"],
+ "exclude": [],
+ "expected": [],
+ },
+ {
+ "paths": ["does/not/exist"],
+ "include": ["."],
+ "exclude": [],
+ "expected": [],
+ },
+ ),
+)
+def test_filterpaths(test):
+ expected = test.pop("expected")
+ expected_exclude = test.pop("expected_exclude", [])
+
+ paths, exclude = pathutils.filterpaths(root, **test)
+ assert_paths(paths, expected)
+ assert_paths(exclude, expected_exclude)
+
+
+@pytest.mark.parametrize(
+ "test",
+ (
+ {
+ "paths": ["subdir1/b.js"],
+ "config": {
+ "exclude": ["subdir1"],
+ "extensions": "js",
+ },
+ "expected": [],
+ },
+ {
+ "paths": ["subdir1/subdir3"],
+ "config": {
+ "exclude": ["subdir1"],
+ "extensions": "js",
+ },
+ "expected": [],
+ },
+ ),
+)
+def test_expand_exclusions(test):
+ expected = test.pop("expected", [])
+
+ paths = list(pathutils.expand_exclusions(test["paths"], test["config"], root))
+ assert_paths(paths, expected)
+
+
+@pytest.mark.parametrize(
+ "paths,expected",
+ [
+ (["subdir1/*"], ["subdir1"]),
+ (["subdir2/*"], ["subdir2"]),
+ (["subdir1/*.*", "subdir1/subdir3/*", "subdir2/*"], ["subdir1", "subdir2"]),
+ ([root + "/*", "subdir1/*.*", "subdir1/subdir3/*", "subdir2/*"], [root]),
+ (["subdir1/b.py", "subdir1/subdir3"], ["subdir1/b.py", "subdir1/subdir3"]),
+ (["subdir1/b.py", "subdir1/b.js"], ["subdir1/b.py", "subdir1/b.js"]),
+ (["subdir1/subdir3"], ["subdir1/subdir3"]),
+ (
+ [
+ "foo",
+ "foobar",
+ ],
+ ["foo", "foobar"],
+ ),
+ ],
+)
+def test_collapse(paths, expected):
+ os.chdir(root)
+
+ inputs = []
+ for path in paths:
+ base, name = os.path.split(path)
+ if "*" in name:
+ for n in os.listdir(base):
+ if not fnmatch(n, name):
+ continue
+ inputs.append(os.path.join(base, n))
+ else:
+ inputs.append(path)
+
+ print("inputs: {}".format(inputs))
+ assert_paths(pathutils.collapse(inputs), expected)
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_result.py b/python/mozlint/test/test_result.py
new file mode 100644
index 0000000000..02e8156b3c
--- /dev/null
+++ b/python/mozlint/test/test_result.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+
+from mozlint.result import Issue, ResultSummary
+
+
+def test_issue_defaults():
+ ResultSummary.root = "/fake/root"
+
+ issue = Issue(linter="linter", path="path", message="message", lineno=None)
+ assert issue.lineno == 0
+ assert issue.column is None
+ assert issue.level == "error"
+
+ issue = Issue(
+ linter="linter", path="path", message="message", lineno="1", column="2"
+ )
+ assert issue.lineno == 1
+ assert issue.column == 2
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_roller.py b/python/mozlint/test/test_roller.py
new file mode 100644
index 0000000000..2918047cd2
--- /dev/null
+++ b/python/mozlint/test/test_roller.py
@@ -0,0 +1,396 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import platform
+import signal
+import subprocess
+import sys
+import time
+from itertools import chain
+
+import mozunit
+import pytest
+
+from mozlint.errors import LintersNotConfigured, NoValidLinter
+from mozlint.result import Issue, ResultSummary
+from mozlint.roller import LintRoller
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def test_roll_no_linters_configured(lint, files):
+ with pytest.raises(LintersNotConfigured):
+ lint.roll(files)
+
+
+def test_roll_successful(lint, linters, files):
+ lint.read(linters("string", "regex", "external"))
+
+ result = lint.roll(files)
+ assert len(result.issues) == 1
+ assert result.failed == set([])
+
+ path = list(result.issues.keys())[0]
+ assert os.path.basename(path) == "foobar.js"
+
+ errors = result.issues[path]
+ assert isinstance(errors, list)
+ assert len(errors) == 6
+
+ container = errors[0]
+ assert isinstance(container, Issue)
+ assert container.rule == "no-foobar"
+
+
+def test_roll_from_subdir(lint, linters):
+ lint.read(linters("string", "regex", "external"))
+
+ oldcwd = os.getcwd()
+ try:
+ os.chdir(os.path.join(lint.root, "files"))
+
+ # Path relative to cwd works
+ result = lint.roll("foobar.js")
+ assert len(result.issues) == 1
+ assert len(result.failed) == 0
+ assert result.returncode == 1
+
+ # Path relative to root doesn't work
+ result = lint.roll(os.path.join("files", "foobar.js"))
+ assert len(result.issues) == 0
+ assert len(result.failed) == 0
+ assert result.returncode == 0
+
+ # Paths from vcs are always joined to root instead of cwd
+ lint.mock_vcs([os.path.join("files", "foobar.js")])
+ result = lint.roll(outgoing=True)
+ assert len(result.issues) == 1
+ assert len(result.failed) == 0
+ assert result.returncode == 1
+
+ result = lint.roll(workdir=True)
+ assert len(result.issues) == 1
+ assert len(result.failed) == 0
+ assert result.returncode == 1
+
+ result = lint.roll(rev='not public() and keyword("dummy revset expression")')
+ assert len(result.issues) == 1
+ assert len(result.failed) == 0
+ assert result.returncode == 1
+ finally:
+ os.chdir(oldcwd)
+
+
+def test_roll_catch_exception(lint, linters, files, capfd):
+ lint.read(linters("raises"))
+
+ lint.roll(files) # assert not raises
+ out, err = capfd.readouterr()
+ assert "LintException" in err
+
+
+def test_roll_with_global_excluded_path(lint, linters, files):
+ lint.exclude = ["**/foobar.js"]
+ lint.read(linters("string", "regex", "external"))
+ result = lint.roll(files)
+
+ assert len(result.issues) == 0
+ assert result.failed == set([])
+
+
+def test_roll_with_local_excluded_path(lint, linters, files):
+ lint.read(linters("excludes"))
+ result = lint.roll(files)
+
+ assert "**/foobar.js" in lint.linters[0]["local_exclude"]
+ assert len(result.issues) == 0
+ assert result.failed == set([])
+
+
+def test_roll_with_no_files_to_lint(lint, linters, capfd):
+ lint.read(linters("string", "regex", "external"))
+ lint.mock_vcs([])
+ result = lint.roll([], workdir=True)
+ assert isinstance(result, ResultSummary)
+ assert len(result.issues) == 0
+ assert len(result.failed) == 0
+
+ out, err = capfd.readouterr()
+ assert "warning: no files linted" in out
+
+
+def test_roll_with_invalid_extension(lint, linters, filedir):
+ lint.read(linters("external"))
+ result = lint.roll(os.path.join(filedir, "foobar.py"))
+ assert len(result.issues) == 0
+ assert result.failed == set([])
+
+
+def test_roll_with_failure_code(lint, linters, files):
+ lint.read(linters("badreturncode"))
+
+ result = lint.roll(files, num_procs=1)
+ assert len(result.issues) == 0
+ assert result.failed == set(["BadReturnCodeLinter"])
+
+
+def test_roll_warnings(lint, linters, files):
+ lint.read(linters("warning"))
+ result = lint.roll(files)
+ assert len(result.issues) == 0
+ assert result.total_issues == 0
+ assert len(result.suppressed_warnings) == 1
+ assert result.total_suppressed_warnings == 2
+
+ lint.lintargs["show_warnings"] = True
+ result = lint.roll(files)
+ assert len(result.issues) == 1
+ assert result.total_issues == 2
+ assert len(result.suppressed_warnings) == 0
+ assert result.total_suppressed_warnings == 0
+
+
+def test_roll_code_review(monkeypatch, linters, files):
+ monkeypatch.setenv("CODE_REVIEW", "1")
+ lint = LintRoller(root=here, show_warnings=False)
+ lint.read(linters("warning"))
+ result = lint.roll(files)
+ assert len(result.issues) == 1
+ assert result.total_issues == 2
+ assert len(result.suppressed_warnings) == 0
+ assert result.total_suppressed_warnings == 0
+ assert result.returncode == 1
+
+
+def test_roll_code_review_warnings_disabled(monkeypatch, linters, files):
+ monkeypatch.setenv("CODE_REVIEW", "1")
+ lint = LintRoller(root=here, show_warnings=False)
+ lint.read(linters("warning_no_code_review"))
+ result = lint.roll(files)
+ assert len(result.issues) == 0
+ assert result.total_issues == 0
+ assert lint.result.fail_on_warnings is True
+ assert len(result.suppressed_warnings) == 1
+ assert result.total_suppressed_warnings == 2
+ assert result.returncode == 0
+
+
+def test_roll_code_review_warnings_soft(linters, files):
+ lint = LintRoller(root=here, show_warnings="soft")
+ lint.read(linters("warning_no_code_review"))
+ result = lint.roll(files)
+ assert len(result.issues) == 1
+ assert result.total_issues == 2
+ assert lint.result.fail_on_warnings is False
+ assert len(result.suppressed_warnings) == 0
+ assert result.total_suppressed_warnings == 0
+ assert result.returncode == 0
+
+
+def fake_run_worker(config, paths, **lintargs):
+ result = ResultSummary(lintargs["root"])
+ result.issues["count"].append(1)
+ return result
+
+
+@pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason="monkeypatch issues with multiprocessing on Windows",
+)
+@pytest.mark.parametrize("num_procs", [1, 4, 8, 16])
+def test_number_of_jobs(monkeypatch, lint, linters, files, num_procs):
+ monkeypatch.setattr(sys.modules[lint.__module__], "_run_worker", fake_run_worker)
+
+ linters = linters("string", "regex", "external")
+ lint.read(linters)
+ num_jobs = len(lint.roll(files, num_procs=num_procs).issues["count"])
+
+ if len(files) >= num_procs:
+ assert num_jobs == num_procs * len(linters)
+ else:
+ assert num_jobs == len(files) * len(linters)
+
+
+@pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason="monkeypatch issues with multiprocessing on Windows",
+)
+@pytest.mark.parametrize("max_paths,expected_jobs", [(1, 12), (4, 6), (16, 6)])
+def test_max_paths_per_job(monkeypatch, lint, linters, files, max_paths, expected_jobs):
+ monkeypatch.setattr(sys.modules[lint.__module__], "_run_worker", fake_run_worker)
+
+ files = files[:4]
+ assert len(files) == 4
+
+ linters = linters("string", "regex", "external")[:3]
+ assert len(linters) == 3
+
+ lint.MAX_PATHS_PER_JOB = max_paths
+ lint.read(linters)
+ num_jobs = len(lint.roll(files, num_procs=2).issues["count"])
+ assert num_jobs == expected_jobs
+
+
+@pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason="monkeypatch issues with multiprocessing on Windows",
+)
+@pytest.mark.parametrize("num_procs", [1, 4, 8, 16])
+def test_number_of_jobs_global(monkeypatch, lint, linters, files, num_procs):
+ monkeypatch.setattr(sys.modules[lint.__module__], "_run_worker", fake_run_worker)
+
+ linters = linters("global")
+ lint.read(linters)
+ num_jobs = len(lint.roll(files, num_procs=num_procs).issues["count"])
+
+ assert num_jobs == 1
+
+
+@pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason="monkeypatch issues with multiprocessing on Windows",
+)
+@pytest.mark.parametrize("max_paths", [1, 4, 16])
+def test_max_paths_per_job_global(monkeypatch, lint, linters, files, max_paths):
+ monkeypatch.setattr(sys.modules[lint.__module__], "_run_worker", fake_run_worker)
+
+ files = files[:4]
+ assert len(files) == 4
+
+ linters = linters("global")[:1]
+ assert len(linters) == 1
+
+ lint.MAX_PATHS_PER_JOB = max_paths
+ lint.read(linters)
+ num_jobs = len(lint.roll(files, num_procs=2).issues["count"])
+ assert num_jobs == 1
+
+
+@pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason="signal.CTRL_C_EVENT isn't causing a KeyboardInterrupt on Windows",
+)
+def test_keyboard_interrupt():
+ # We use two linters so we'll have two jobs. One (string.yml) will complete
+ # quickly. The other (slow.yml) will run slowly. This way the first worker
+ # will be be stuck blocking on the ProcessPoolExecutor._call_queue when the
+ # signal arrives and the other still be doing work.
+ cmd = [sys.executable, "runcli.py", "-l=string", "-l=slow", "files/foobar.js"]
+ env = os.environ.copy()
+ env["PYTHONPATH"] = os.pathsep.join(sys.path)
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ cwd=here,
+ env=env,
+ universal_newlines=True,
+ )
+ time.sleep(1)
+ proc.send_signal(signal.SIGINT)
+
+ out = proc.communicate()[0]
+ print(out)
+ assert "warning: not all files were linted" in out
+ assert "2 problems" in out
+ assert "Traceback" not in out
+
+
+def test_support_files(lint, linters, filedir, monkeypatch, files):
+ jobs = []
+
+ # Replace the original _generate_jobs with a new one that simply
+ # adds jobs to a list (and then doesn't return anything).
+ orig_generate_jobs = lint._generate_jobs
+
+ def fake_generate_jobs(*args, **kwargs):
+ jobs.extend([job[1] for job in orig_generate_jobs(*args, **kwargs)])
+ return []
+
+ monkeypatch.setattr(lint, "_generate_jobs", fake_generate_jobs)
+
+ linter_path = linters("support_files")[0]
+ lint.read(linter_path)
+ lint.root = filedir
+
+ # Modified support files only lint entire root if --outgoing or --workdir
+ # are used.
+ path = os.path.join(filedir, "foobar.js")
+ vcs_path = os.path.join(filedir, "foobar.py")
+
+ lint.mock_vcs([vcs_path])
+ lint.roll(path)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == [path]
+
+ expected_files = sorted(files)
+
+ jobs = []
+ lint.roll(path, workdir=True)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == expected_files
+
+ jobs = []
+ lint.roll(path, outgoing=True)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == expected_files
+
+ jobs = []
+ lint.roll(path, rev='draft() and keyword("dummy revset expression")')
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == expected_files
+
+ # Lint config file is implicitly added as a support file
+ lint.mock_vcs([linter_path])
+ jobs = []
+ lint.roll(path, outgoing=True, workdir=True)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == expected_files
+
+ # Avoid linting the entire root when `--fix` is passed.
+ lint.mock_vcs([vcs_path])
+ lint.lintargs["fix"] = True
+
+ jobs = []
+ lint.roll(path, outgoing=True)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == sorted([path, vcs_path]), (
+ "`--fix` with `--outgoing` on a `support-files` change should "
+ "avoid linting the entire root."
+ )
+
+ jobs = []
+ lint.roll(path, workdir=True)
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == sorted([path, vcs_path]), (
+ "`--fix` with `--workdir` on a `support-files` change should "
+ "avoid linting the entire root."
+ )
+
+ jobs = []
+ lint.roll(path, rev='draft() and keyword("dummy revset expression")')
+ actual_files = sorted(chain(*jobs))
+ assert actual_files == sorted([path, vcs_path]), (
+ "`--fix` with `--rev` on a `support-files` change should "
+ "avoid linting the entire root."
+ )
+
+
+def test_setup(lint, linters, filedir, capfd):
+ with pytest.raises(NoValidLinter):
+ lint.setup()
+
+ lint.read(linters("setup", "setupfailed", "setupraised"))
+ lint.setup()
+ out, err = capfd.readouterr()
+ assert "setup passed" in out
+ assert "setup failed" in out
+ assert "setup raised" in out
+ assert "error: problem with lint setup, skipping" in out
+ assert lint.result.failed_setup == set(["SetupFailedLinter", "SetupRaisedLinter"])
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/python/mozlint/test/test_types.py b/python/mozlint/test/test_types.py
new file mode 100644
index 0000000000..6ed78747b7
--- /dev/null
+++ b/python/mozlint/test/test_types.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import mozpack.path as mozpath
+import mozunit
+import pytest
+
+from mozlint.result import Issue, ResultSummary
+
+
+@pytest.fixture
+def path(filedir):
+ def _path(name):
+ return mozpath.join(filedir, name)
+
+ return _path
+
+
+@pytest.fixture(
+ params=[
+ "external.yml",
+ "global.yml",
+ "regex.yml",
+ "string.yml",
+ "structured.yml",
+ ]
+)
+def linter(lintdir, request):
+ return os.path.join(lintdir, request.param)
+
+
+def test_linter_types(lint, linter, files, path):
+ lint.read(linter)
+ result = lint.roll(files)
+ assert isinstance(result, ResultSummary)
+ assert isinstance(result.issues, dict)
+ assert path("foobar.js") in result.issues
+ assert path("no_foobar.js") not in result.issues
+
+ issue = result.issues[path("foobar.js")][0]
+ assert isinstance(issue, Issue)
+
+ name = os.path.basename(linter).split(".")[0]
+ assert issue.linter.lower().startswith(name)
+
+
+def test_linter_missing_files(lint, linter, filedir):
+ # Missing files should be caught by `mozlint.cli`, so the only way they
+ # could theoretically happen is if they show up from versioncontrol. So
+ # let's just make sure they get ignored.
+ lint.read(linter)
+ files = [
+ os.path.join(filedir, "missing.js"),
+ os.path.join(filedir, "missing.py"),
+ ]
+ result = lint.roll(files)
+ assert result.returncode == 0
+
+ lint.mock_vcs(files)
+ result = lint.roll(outgoing=True)
+ assert result.returncode == 0
+
+
+def test_no_filter(lint, lintdir, files):
+ lint.read(os.path.join(lintdir, "explicit_path.yml"))
+ result = lint.roll(files)
+ assert len(result.issues) == 0
+
+ lint.lintargs["use_filters"] = False
+ result = lint.roll(files)
+ assert len(result.issues) == 3
+
+
+def test_global_skipped(lint, lintdir, files):
+ lint.read(os.path.join(lintdir, "global_skipped.yml"))
+ result = lint.roll(files)
+ assert len(result.issues) == 0
+
+
+if __name__ == "__main__":
+ mozunit.main()