diff options
Diffstat (limited to 'src/ansiblelint/runner.py')
| -rw-r--r-- | src/ansiblelint/runner.py | 326 |
1 files changed, 225 insertions, 101 deletions
diff --git a/src/ansiblelint/runner.py b/src/ansiblelint/runner.py index 9d3500d..f487329 100644 --- a/src/ansiblelint/runner.py +++ b/src/ansiblelint/runner.py @@ -1,8 +1,10 @@ """Runner implementation.""" + from __future__ import annotations import json import logging +import math import multiprocessing import multiprocessing.pool import os @@ -12,7 +14,9 @@ import tempfile import warnings from dataclasses import dataclass from fnmatch import fnmatch +from functools import cache from pathlib import Path +from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any from ansible.errors import AnsibleError @@ -23,31 +27,24 @@ from ansible_compat.runtime import AnsibleWarning import ansiblelint.skip_utils import ansiblelint.utils -from ansiblelint._internal.rules import ( - BaseRule, - LoadingFailureRule, - RuntimeErrorRule, - WarningRule, -) from ansiblelint.app import App, get_app from ansiblelint.constants import States from ansiblelint.errors import LintWarning, MatchError, WarnSource from ansiblelint.file_utils import Lintable, expand_dirs_in_lintables from ansiblelint.logger import timed_info -from ansiblelint.rules.syntax_check import OUTPUT_PATTERNS, AnsibleSyntaxCheckRule +from ansiblelint.rules.syntax_check import OUTPUT_PATTERNS from ansiblelint.text import strip_ansi_escape from ansiblelint.utils import ( PLAYBOOK_DIR, - _include_children, - _roles_children, - _taskshandlers_children, + HandleChildren, + parse_examples_from_plugin, template, ) if TYPE_CHECKING: - from collections.abc import Generator - from typing import Callable + from collections.abc import Callable, Generator + from ansiblelint._internal.rules import BaseRule from ansiblelint.config import Options from ansiblelint.constants import FileType from ansiblelint.rules import RulesCollection @@ -77,11 +74,13 @@ class Runner: verbosity: int = 0, checked_files: set[Lintable] | None = None, project_dir: str | None = None, + _skip_ansible_syntax_check: bool = False, ) -> None: """Initialize a Runner instance.""" self.rules = rules self.lintables: set[Lintable] = set() self.project_dir = os.path.abspath(project_dir) if project_dir else None + self.skip_ansible_syntax_check = _skip_ansible_syntax_check if skip_list is None: skip_list = [] @@ -107,6 +106,8 @@ class Runner: checked_files = set() self.checked_files = checked_files + self.app = get_app(cached=True) + def _update_exclude_paths(self, exclude_paths: list[str]) -> None: if exclude_paths: # These will be (potentially) relative paths @@ -172,19 +173,19 @@ class Runner: if isinstance(warn.source, WarnSource): match = MatchError( message=warn.source.message or warn.category.__name__, - rule=WarningRule(), - filename=warn.source.filename.filename, + rule=self.rules["warning"], + lintable=Lintable(warn.source.filename.filename), tag=warn.source.tag, lineno=warn.source.lineno, ) else: filename = warn.source match = MatchError( - message=warn.message - if isinstance(warn.message, str) - else "?", - rule=WarningRule(), - filename=str(filename), + message=( + warn.message if isinstance(warn.message, str) else "?" + ), + rule=self.rules["warning"], + lintable=Lintable(str(filename)), ) matches.append(match) continue @@ -215,7 +216,7 @@ class Runner: lintable=lintable, message=str(lintable.exc), details=str(lintable.exc.__cause__), - rule=LoadingFailureRule(), + rule=self.rules["load-failure"], tag=f"load-failure[{lintable.exc.__class__.__name__.lower()}]", ), ) @@ -226,60 +227,63 @@ class Runner: MatchError( lintable=lintable, message="File or directory not found.", - rule=LoadingFailureRule(), + rule=self.rules["load-failure"], tag="load-failure[not-found]", ), ) # -- phase 1 : syntax check in parallel -- - app = get_app(offline=True) + if not self.skip_ansible_syntax_check: + # app = get_app(cached=True) - def worker(lintable: Lintable) -> list[MatchError]: - # pylint: disable=protected-access - return self._get_ansible_syntax_check_matches( - lintable=lintable, - app=app, - ) + def worker(lintable: Lintable) -> list[MatchError]: + return self._get_ansible_syntax_check_matches( + lintable=lintable, + app=self.app, + ) - for lintable in self.lintables: - if lintable.kind not in ("playbook", "role") or lintable.stop_processing: - continue - files.append(lintable) + for lintable in self.lintables: + if ( + lintable.kind not in ("playbook", "role") + or lintable.stop_processing + ): + continue + files.append(lintable) - # avoid resource leak warning, https://github.com/python/cpython/issues/90549 - # pylint: disable=unused-variable - global_resource = multiprocessing.Semaphore() # noqa: F841 + # avoid resource leak warning, https://github.com/python/cpython/issues/90549 + # pylint: disable=unused-variable + global_resource = multiprocessing.Semaphore() # noqa: F841 - pool = multiprocessing.pool.ThreadPool(processes=multiprocessing.cpu_count()) - return_list = pool.map(worker, files, chunksize=1) - pool.close() - pool.join() - for data in return_list: - matches.extend(data) + pool = multiprocessing.pool.ThreadPool(processes=threads()) + return_list = pool.map(worker, files, chunksize=1) + pool.close() + pool.join() + for data in return_list: + matches.extend(data) + + matches = self._filter_excluded_matches(matches) - matches = self._filter_excluded_matches(matches) # -- phase 2 --- - if not matches: - # do our processing only when ansible syntax check passed in order - # to avoid causing runtime exceptions. Our processing is not as - # resilient to be able process garbage. - matches.extend(self._emit_matches(files)) + # do our processing only when ansible syntax check passed in order + # to avoid causing runtime exceptions. Our processing is not as + # resilient to be able process garbage. + matches.extend(self._emit_matches(files)) - # remove duplicates from files list - files = [value for n, value in enumerate(files) if value not in files[:n]] + # remove duplicates from files list + files = [value for n, value in enumerate(files) if value not in files[:n]] - for file in self.lintables: - if file in self.checked_files or not file.kind: - continue - _logger.debug( - "Examining %s of type %s", - ansiblelint.file_utils.normpath(file.path), - file.kind, - ) + for file in self.lintables: + if file in self.checked_files or not file.kind or file.failed(): + continue + _logger.debug( + "Examining %s of type %s", + ansiblelint.file_utils.normpath(file.path), + file.kind, + ) - matches.extend( - self.rules.run(file, tags=set(self.tags), skip_list=self.skip_list), - ) + matches.extend( + self.rules.run(file, tags=set(self.tags), skip_list=self.skip_list), + ) # update list of checked files self.checked_files.update(self.lintables) @@ -296,7 +300,12 @@ class Runner: app: App, ) -> list[MatchError]: """Run ansible syntax check and return a list of MatchError(s).""" - default_rule: BaseRule = AnsibleSyntaxCheckRule() + try: + default_rule: BaseRule = self.rules["syntax-check"] + except ValueError: + # if syntax-check is not loaded, we do not perform any syntax check, + # that might happen during testing + return [] fh = None results = [] if lintable.kind not in ("playbook", "role"): @@ -341,6 +350,9 @@ class Runner: # https://github.com/paramiko/paramiko/issues/2038 env = app.runtime.environ.copy() env["PYTHONWARNINGS"] = "ignore" + # Avoid execution failure if user customized any_unparsed_is_failed setting + # https://github.com/ansible/ansible-lint/issues/3650 + env["ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED"] = "False" run = subprocess.run( cmd, @@ -357,6 +369,7 @@ class Runner: filename = lintable lineno = 1 column = None + ignore_rc = False stderr = strip_ansi_escape(run.stderr) stdout = strip_ansi_escape(run.stdout) @@ -376,25 +389,40 @@ class Runner: details = groups.get("details", "") lineno = int(groups.get("line", 1)) - if "filename" in groups: + if ( + "filename" in groups + and str(lintable.path.absolute()) != groups["filename"] + and lintable.filename != groups["filename"] + ): + # avoids creating a new lintable object if the filename + # is matching as this might prevent Lintable.failed() + # feature from working well. filename = Lintable(groups["filename"]) else: filename = lintable column = int(groups.get("column", 1)) - results.append( - MatchError( - message=title, - lintable=filename, - lineno=lineno, - column=column, - rule=rule, - details=details, - tag=f"{rule.id}[{pattern.tag}]", - ), - ) - if not results: - rule = RuntimeErrorRule() + if ( + pattern.tag in ("unknown-module", "specific") + and app.options.nodeps + ): + ignore_rc = True + else: + results.append( + MatchError( + message=title, + lintable=filename, + lineno=lineno, + column=column, + rule=rule, + details=details, + tag=f"{rule.id}[{pattern.tag}]", + ), + ) + break + + if not results and not ignore_rc: + rule = self.rules["internal-error"] message = ( f"Unexpected error code {run.returncode} from " f"execution of: {' '.join(cmd)}" @@ -427,6 +455,9 @@ class Runner: visited: set[Lintable] = set() while visited != self.lintables: for lintable in self.lintables - visited: + visited.add(lintable) + if not lintable.path.exists(): + continue try: children = self.find_children(lintable) for child in children: @@ -437,11 +468,13 @@ class Runner: except MatchError as exc: if not exc.filename: # pragma: no branch exc.filename = str(lintable.path) - exc.rule = LoadingFailureRule() + exc.rule = self.rules["load-failure"] yield exc except AttributeError: - yield MatchError(lintable=lintable, rule=LoadingFailureRule()) - visited.add(lintable) + yield MatchError( + lintable=lintable, + rule=self.rules["load-failure"], + ) def find_children(self, lintable: Lintable) -> list[Lintable]: """Traverse children of a single file or folder.""" @@ -452,21 +485,25 @@ class Runner: add_all_plugin_dirs(playbook_dir or ".") if lintable.kind == "role": playbook_ds = AnsibleMapping({"roles": [{"role": str(lintable.path)}]}) + elif lintable.kind == "plugin": + return self.plugin_children(lintable) elif lintable.kind not in ("playbook", "tasks"): return [] else: try: playbook_ds = ansiblelint.utils.parse_yaml_from_file(str(lintable.path)) except AnsibleError as exc: - raise SystemExit(exc) from exc + msg = f"Loading {lintable.filename} caused an {type(exc).__name__} exception: {exc}, file was ignored." + logging.exception(msg) + return [] results = [] # playbook_ds can be an AnsibleUnicode string, which we consider invalid if isinstance(playbook_ds, str): - raise MatchError(lintable=lintable, rule=LoadingFailureRule()) + raise MatchError(lintable=lintable, rule=self.rules["load-failure"]) for item in ansiblelint.utils.playbook_items(playbook_ds): # if lintable.kind not in ["playbook"]: for child in self.play_children( - lintable.path.parent, + lintable, item, lintable.kind, playbook_dir, @@ -487,35 +524,40 @@ class Runner: if path != path_str: child.path = Path(path) child.name = child.path.name - results.append(child) return results def play_children( self, - basedir: Path, + lintable: Lintable, item: tuple[str, Any], parent_type: FileType, playbook_dir: str, ) -> list[Lintable]: """Flatten the traversed play tasks.""" # pylint: disable=unused-argument - delegate_map: dict[str, Callable[[str, Any, Any, FileType], list[Lintable]]] = { - "tasks": _taskshandlers_children, - "pre_tasks": _taskshandlers_children, - "post_tasks": _taskshandlers_children, - "block": _taskshandlers_children, - "include": _include_children, - "ansible.builtin.include": _include_children, - "import_playbook": _include_children, - "ansible.builtin.import_playbook": _include_children, - "roles": _roles_children, - "dependencies": _roles_children, - "handlers": _taskshandlers_children, - "include_tasks": _include_children, - "ansible.builtin.include_tasks": _include_children, - "import_tasks": _include_children, - "ansible.builtin.import_tasks": _include_children, + basedir = lintable.path.parent + handlers = HandleChildren(self.rules, app=self.app) + + delegate_map: dict[ + str, + Callable[[Lintable, Any, Any, FileType], list[Lintable]], + ] = { + "tasks": handlers.taskshandlers_children, + "pre_tasks": handlers.taskshandlers_children, + "post_tasks": handlers.taskshandlers_children, + "block": handlers.taskshandlers_children, + "include": handlers.include_children, + "ansible.builtin.include": handlers.include_children, + "import_playbook": handlers.include_children, + "ansible.builtin.import_playbook": handlers.include_children, + "roles": handlers.roles_children, + "dependencies": handlers.roles_children, + "handlers": handlers.taskshandlers_children, + "include_tasks": handlers.include_children, + "ansible.builtin.include_tasks": handlers.include_children, + "import_tasks": handlers.include_children, + "ansible.builtin.import_tasks": handlers.include_children, } (k, v) = item add_all_plugin_dirs(str(basedir.resolve())) @@ -527,11 +569,92 @@ class Runner: {"playbook_dir": PLAYBOOK_DIR or str(basedir.resolve())}, fail_on_undefined=False, ) - return delegate_map[k](str(basedir), k, v, parent_type) + return delegate_map[k](lintable, k, v, parent_type) return [] + def plugin_children(self, lintable: Lintable) -> list[Lintable]: + """Collect lintable sections from plugin file.""" + offset, content = parse_examples_from_plugin(lintable) + if not content: + # No examples, nothing to see here + return [] + examples = Lintable( + name=lintable.name, + content=content, + kind="yaml", + base_kind="text/yaml", + parent=lintable, + ) + examples.line_offset = offset -def _get_matches(rules: RulesCollection, options: Options) -> LintResult: + # pylint: disable=consider-using-with + examples.file = NamedTemporaryFile( + mode="w+", + suffix=f"_{lintable.path.name}.yaml", + ) + examples.file.write(content) + examples.file.flush() + examples.filename = examples.file.name + examples.path = Path(examples.file.name) + return [examples] + + +@cache +def threads() -> int: + """Determine how many threads to use. + + Inside containers we want to respect limits imposed. + + When present /sys/fs/cgroup/cpu.max can contain something like: + $ podman/docker run -it --rm --cpus 1.5 ubuntu:latest cat /sys/fs/cgroup/cpu.max + 150000 100000 + # "max 100000" is returned when no limits are set. + + See: https://github.com/python/cpython/issues/80235 + See: https://github.com/python/cpython/issues/70879 + """ + os_cpu_count = multiprocessing.cpu_count() + # Cgroup CPU bandwidth limit available in Linux since 2.6 kernel + + cpu_max_fname = "/sys/fs/cgroup/cpu.max" + cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + if os.path.exists(cpu_max_fname): + # cgroup v2 + # https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html + with open(cpu_max_fname, encoding="utf-8") as fh: + cpu_quota_us, cpu_period_us = fh.read().strip().split() + elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname): + # cgroup v1 + # https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management + with open(cfs_quota_fname, encoding="utf-8") as fh: + cpu_quota_us = fh.read().strip() + with open(cfs_period_fname, encoding="utf-8") as fh: + cpu_period_us = fh.read().strip() + else: + # No Cgroup CPU bandwidth limit (e.g. non-Linux platform) + cpu_quota_us = "max" + cpu_period_us = "100000" # unused, for consistency with default values + + if cpu_quota_us == "max": + # No active Cgroup quota on a Cgroup-capable platform + return os_cpu_count + cpu_quota_us_int = int(cpu_quota_us) + cpu_period_us_int = int(cpu_period_us) + if cpu_quota_us_int > 0 and cpu_period_us_int > 0: + return math.ceil(cpu_quota_us_int / cpu_period_us_int) + # Setting a negative cpu_quota_us value is a valid way to disable + # cgroup CPU bandwidth limits + return os_cpu_count + + +def get_matches(rules: RulesCollection, options: Options) -> LintResult: + """Get matches for given rules and options. + + :param rules: Rules to use for linting. + :param options: Options to use for linting. + :returns: LintResult containing matches and checked files. + """ lintables = ansiblelint.utils.get_lintables(opts=options, args=options.lintables) for rule in rules: @@ -551,6 +674,7 @@ def _get_matches(rules: RulesCollection, options: Options) -> LintResult: verbosity=options.verbosity, checked_files=checked_files, project_dir=options.project_dir, + _skip_ansible_syntax_check=options._skip_ansible_syntax_check, # noqa: SLF001 ) matches.extend(runner.run()) |