diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-26 06:24:57 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-26 06:24:57 +0000 |
commit | 1faea9a6c75f33109e8f66b57b432fdad57b3f46 (patch) | |
tree | 4184ce38ac0cf9d5a46bbbae03c87be82927f12b /src/ansiblelint | |
parent | Adding upstream version 6.17.2. (diff) | |
download | ansible-lint-upstream.tar.xz ansible-lint-upstream.zip |
Adding upstream version 24.6.1.upstream/24.6.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/ansiblelint')
112 files changed, 4121 insertions, 1131 deletions
diff --git a/src/ansiblelint/__main__.py b/src/ansiblelint/__main__.py index af434d0..ca4a33b 100755 --- a/src/ansiblelint/__main__.py +++ b/src/ansiblelint/__main__.py @@ -30,12 +30,23 @@ import shutil import site import sys from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, TextIO +from typing import TYPE_CHECKING, Any, TextIO from ansible_compat.prerun import get_cache_dir from filelock import FileLock, Timeout from rich.markup import escape +from ansiblelint.constants import RC, SKIP_SCHEMA_UPDATE + +# safety check for broken ansible core, needs to happen first +try: + # pylint: disable=unused-import + from ansible.parsing.dataloader import DataLoader # noqa: F401 + +except Exception as _exc: # pylint: disable=broad-exception-caught # noqa: BLE001 + logging.fatal(_exc) + sys.exit(RC.INVALID_CONFIG) +# pylint: disable=ungrouped-imports from ansiblelint import cli from ansiblelint._mockings import _perform_mockings_cleanup from ansiblelint.app import get_app @@ -53,20 +64,21 @@ from ansiblelint.config import ( log_entries, options, ) -from ansiblelint.constants import RC from ansiblelint.loaders import load_ignore_txt +from ansiblelint.runner import get_matches from ansiblelint.skip_utils import normalize_tag from ansiblelint.version import __version__ if TYPE_CHECKING: # RulesCollection must be imported lazily or ansible gets imported too early. + from collections.abc import Callable + from ansiblelint.rules import RulesCollection from ansiblelint.runner import LintResult _logger = logging.getLogger(__name__) -cache_dir_lock: None | FileLock = None class LintLogHandler(logging.Handler): @@ -107,8 +119,9 @@ def initialize_logger(level: int = 0) -> None: _logger.debug("Logging initialized to level %s", logging_level) -def initialize_options(arguments: list[str] | None = None) -> None: +def initialize_options(arguments: list[str] | None = None) -> None | FileLock: """Load config options and store them inside options module.""" + cache_dir_lock = None new_options = cli.get_config(arguments or []) new_options.cwd = pathlib.Path.cwd() @@ -132,13 +145,13 @@ def initialize_options(arguments: list[str] | None = None) -> None: options.cache_dir.mkdir(parents=True, exist_ok=True) if not options.offline: # pragma: no cover - cache_dir_lock = FileLock( # pylint: disable=redefined-outer-name + cache_dir_lock = FileLock( f"{options.cache_dir}/.lock", ) try: cache_dir_lock.acquire(timeout=180) except Timeout: # pragma: no cover - _logger.error( + _logger.error( # noqa: TRY400 "Timeout waiting for another instance of ansible-lint to release the lock.", ) sys.exit(RC.LOCK_TIMEOUT) @@ -147,6 +160,8 @@ def initialize_options(arguments: list[str] | None = None) -> None: if "ANSIBLE_DEVEL_WARNING" not in os.environ: # pragma: no branch os.environ["ANSIBLE_DEVEL_WARNING"] = "false" + return cache_dir_lock + def _do_list(rules: RulesCollection) -> int: # On purpose lazy-imports to avoid pre-loading Ansible @@ -194,23 +209,85 @@ def _do_transform(result: LintResult, opts: Options) -> None: def support_banner() -> None: """Display support banner when running on unsupported platform.""" - if sys.version_info < (3, 9, 0): # pragma: no cover - prefix = "::warning::" if "GITHUB_ACTION" in os.environ else "WARNING: " - console_stderr.print( - f"{prefix}ansible-lint is no longer tested under Python {sys.version_info.major}.{sys.version_info.minor} and will soon require 3.9. Do not report bugs for this version.", - style="bold red", - ) -# pylint: disable=too-many-statements,too-many-locals +def fix(runtime_options: Options, result: LintResult, rules: RulesCollection) -> None: + """Fix the linting errors. + + :param options: Options object + :param result: LintResult object + """ + match_count = len(result.matches) + _logger.debug("Begin fixing: %s matches", match_count) + ruamel_safe_version = "0.17.26" + + # pylint: disable=import-outside-toplevel + from packaging.version import Version + from ruamel.yaml import __version__ as ruamel_yaml_version_str + + # pylint: enable=import-outside-toplevel + + if Version(ruamel_safe_version) > Version(ruamel_yaml_version_str): + _logger.warning( + "We detected use of `--fix` feature with a buggy ruamel-yaml %s library instead of >=%s, upgrade it before reporting any bugs like dropped comments.", + ruamel_yaml_version_str, + ruamel_safe_version, + ) + acceptable_tags = {"all", "none", *rules.known_tags()} + unknown_tags = set(options.write_list).difference(acceptable_tags) + + if unknown_tags: + _logger.error( + "Found invalid value(s) (%s) for --fix arguments, must be one of: %s", + ", ".join(unknown_tags), + ", ".join(acceptable_tags), + ) + sys.exit(RC.INVALID_CONFIG) + _do_transform(result, options) + + rerun = ["yaml"] + resolved = [] + for idx, match in reversed(list(enumerate(result.matches))): + _logger.debug("Fixing: (%s of %s) %s", match_count - idx, match_count, match) + if match.fixed: + _logger.debug("Fixed, removed: %s", match) + result.matches.pop(idx) + continue + if match.rule.id not in rerun: + _logger.debug("Not rerun eligible: %s", match) + continue + + uid = (match.rule.id, match.filename) + if uid in resolved: + _logger.debug("Previously resolved: %s", match) + result.matches.pop(idx) + continue + _logger.debug("Rerunning: %s", match) + runtime_options.tags = [match.rule.id] + runtime_options.lintables = [match.filename] + runtime_options._skip_ansible_syntax_check = True # noqa: SLF001 + new_results = get_matches(rules, runtime_options) + if not new_results.matches: + _logger.debug("Newly resolved: %s", match) + result.matches.pop(idx) + resolved.append(uid) + continue + if match in new_results.matches: + _logger.debug("Still found: %s", match) + continue + _logger.debug("Fixed, removed: %s", match) + result.matches.pop(idx) + + +# pylint: disable=too-many-locals def main(argv: list[str] | None = None) -> int: """Linter CLI entry point.""" # alter PATH if needed (venv support) - path_inject() + path_inject(argv[0] if argv and argv[0] else "") if argv is None: # pragma: no cover argv = sys.argv - initialize_options(argv[1:]) + cache_dir_lock = initialize_options(argv[1:]) console_options["force_terminal"] = options.colored reconfigure(console_options) @@ -236,7 +313,23 @@ def main(argv: list[str] | None = None) -> int: _logger.debug("Options: %s", options) _logger.debug("CWD: %s", Path.cwd()) - if not options.offline: + # checks if we have `ANSIBLE_LINT_SKIP_SCHEMA_UPDATE` set to bypass schema + # update. Also skip if in offline mode. + # env var set to skip schema refresh + skip_schema_update = ( + bool( + int( + os.environ.get( + SKIP_SCHEMA_UPDATE, + "0", + ), + ), + ) + or options.offline + or options.nodeps + ) + + if not skip_schema_update: # pylint: disable=import-outside-toplevel from ansiblelint.schemas.__main__ import refresh_schemas @@ -244,7 +337,6 @@ def main(argv: list[str] | None = None) -> int: # pylint: disable=import-outside-toplevel from ansiblelint.rules import RulesCollection - from ansiblelint.runner import _get_matches if options.list_profiles: from ansiblelint.generate_docs import profiles_as_rich @@ -265,20 +357,7 @@ def main(argv: list[str] | None = None) -> int: if isinstance(options.tags, str): options.tags = options.tags.split(",") # pragma: no cover - result = _get_matches(rules, options) - - if options.write_list: - ruamel_safe_version = "0.17.26" - from packaging.version import Version - from ruamel.yaml import __version__ as ruamel_yaml_version_str - - if Version(ruamel_safe_version) > Version(ruamel_yaml_version_str): - _logger.warning( - "We detected use of `--write` feature with a buggy ruamel-yaml %s library instead of >=%s, upgrade it before reporting any bugs like dropped comments.", - ruamel_yaml_version_str, - ruamel_safe_version, - ) - _do_transform(result, options) + result = get_matches(rules, options) mark_as_success = True @@ -292,6 +371,18 @@ def main(argv: list[str] | None = None) -> int: for match in result.matches: if match.tag in ignore_map[match.filename]: match.ignored = True + _logger.debug("Ignored: %s", match) + + if app.yamllint_config.incompatible: + logging.log( + level=logging.ERROR if options.write_list else logging.WARNING, + msg=app.yamllint_config.incompatible, + ) + + if options.write_list: + if app.yamllint_config.incompatible: + sys.exit(RC.INVALID_CONFIG) + fix(runtime_options=options, result=result, rules=rules) app.render_matches(result.matches) @@ -325,7 +416,7 @@ def _run_cli_entrypoint() -> None: raise SystemExit(exc) from exc -def path_inject() -> None: +def path_inject(own_location: str = "") -> None: """Add python interpreter path to top of PATH to fix outside venv calling.""" # This make it possible to call ansible-lint that was installed inside a # virtualenv without having to pre-activate it. Otherwise subprocess will @@ -350,6 +441,7 @@ def path_inject() -> None: inject_paths = [] userbase_bin_path = Path(site.getuserbase()) / "bin" + if ( str(userbase_bin_path) not in paths and (userbase_bin_path / "bin" / "ansible").exists() @@ -357,11 +449,23 @@ def path_inject() -> None: inject_paths.append(str(userbase_bin_path)) py_path = Path(sys.executable).parent - if str(py_path) not in paths and (py_path / "ansible").exists(): + pipx_path = os.environ.get("PIPX_HOME", "pipx") + if ( + str(py_path) not in paths + and (py_path / "ansible").exists() + and pipx_path not in str(py_path) + ): inject_paths.append(str(py_path)) + # last option, if nothing else is found, just look next to ourselves... + if own_location: + own_location = os.path.realpath(own_location) + parent = Path(own_location).parent + if (parent / "ansible").exists() and str(parent) not in paths: + inject_paths.append(str(parent)) + if not os.environ.get("PYENV_VIRTUAL_ENV", None): - if inject_paths: + if inject_paths and not all("pipx" in p for p in inject_paths): print( # noqa: T201 f"WARNING: PATH altered to include {', '.join(inject_paths)} :: This is usually a sign of broken local setup, which can cause unexpected behaviors.", file=sys.stderr, diff --git a/src/ansiblelint/_internal/rules.py b/src/ansiblelint/_internal/rules.py index acaf0f3..38cb835 100644 --- a/src/ansiblelint/_internal/rules.py +++ b/src/ansiblelint/_internal/rules.py @@ -1,4 +1,5 @@ """Internally used rule classes.""" + from __future__ import annotations import inspect @@ -9,6 +10,7 @@ from typing import TYPE_CHECKING, Any from ansiblelint.constants import RULE_DOC_URL if TYPE_CHECKING: + from ansiblelint.config import Options from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.rules import RulesCollection @@ -44,6 +46,8 @@ class BaseRule: link: str = "" has_dynamic_tags: bool = False needs_raw_task: bool = False + # Used to mark rules that we will never unload (internal ones) + unloadable: bool = False # We use _order to sort rules and to ensure that some run before others, # _order 0 for internal rules # _order 1 for rules that check that data can be loaded @@ -54,7 +58,7 @@ class BaseRule: _collection: RulesCollection | None = None @property - def help(self) -> str: # noqa: A003 + def help(self) -> str: """Return a help markdown string for the rule.""" if self._help is None: self._help = "" @@ -92,10 +96,11 @@ class BaseRule: _logger.warning( "Ignored exception from %s.%s while processing %s: %s", self.__class__.__name__, - method, + method.__name__, str(file), exc, ) + _logger.debug("Ignored exception details", exc_info=True) else: matches.extend(self.matchdir(file)) return matches @@ -157,6 +162,26 @@ class BaseRule: """ return getattr(cls, "_ids", {cls.id: cls.shortdesc}) + @property + def rule_config(self) -> dict[str, Any]: + """Retrieve rule specific configuration.""" + rule_config = {} + if self.options: + rule_config = self.options.rules.get(self.id, {}) + if not isinstance(rule_config, dict): # pragma: no branch + msg = f"Invalid rule config for {self.id}: {rule_config}" + raise RuntimeError(msg) # noqa: TRY004 + return rule_config + + @property + def options(self) -> Options | None: + """Used to access linter configuration.""" + if self._collection is None: + msg = f"A rule ({self.id}) that is not part of a collection cannot access its configuration." + _logger.warning(msg) + return None + return self._collection.options + # pylint: enable=unused-argument @@ -170,6 +195,7 @@ class RuntimeErrorRule(BaseRule): tags = ["core"] version_added = "v5.0.0" _order = 0 + unloadable = True class AnsibleParserErrorRule(BaseRule): @@ -181,6 +207,7 @@ class AnsibleParserErrorRule(BaseRule): tags = ["core"] version_added = "v5.0.0" _order = 0 + unloadable = True class LoadingFailureRule(BaseRule): @@ -196,6 +223,7 @@ class LoadingFailureRule(BaseRule): _ids = { "load-failure[not-found]": "File not found", } + unloadable = True class WarningRule(BaseRule): @@ -207,3 +235,4 @@ class WarningRule(BaseRule): tags = ["core", "experimental"] version_added = "v6.8.0" _order = 0 + unloadable = True diff --git a/src/ansiblelint/_mockings.py b/src/ansiblelint/_mockings.py index e0482b7..5c2a9a7 100644 --- a/src/ansiblelint/_mockings.py +++ b/src/ansiblelint/_mockings.py @@ -1,4 +1,5 @@ """Utilities for mocking ansible modules and roles.""" + from __future__ import annotations import contextlib @@ -46,7 +47,7 @@ def _make_module_stub(module_name: str, options: Options) -> None: path.mkdir(exist_ok=True, parents=True) _write_module_stub( filename=module_file, - name=module_file, + name=module_name, namespace=namespace, collection=collection, ) @@ -122,4 +123,4 @@ def _perform_mockings_cleanup(options: Options) -> None: else: path = options.cache_dir / "roles" / role_name with contextlib.suppress(OSError): - path.unlink() + path.rmdir() diff --git a/src/ansiblelint/app.py b/src/ansiblelint/app.py index 52581b3..3568f53 100644 --- a/src/ansiblelint/app.py +++ b/src/ansiblelint/app.py @@ -1,10 +1,12 @@ """Application.""" + from __future__ import annotations import copy import itertools import logging import os +import sys from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING, Any @@ -20,6 +22,7 @@ from ansiblelint.config import PROFILES, Options, get_version_warning from ansiblelint.config import options as default_options from ansiblelint.constants import RC, RULE_DOC_URL from ansiblelint.loaders import IGNORE_FILE +from ansiblelint.requirements import Reqs from ansiblelint.stats import SummarizedResults, TagStats if TYPE_CHECKING: @@ -30,6 +33,7 @@ if TYPE_CHECKING: _logger = logging.getLogger(__package__) +_CACHED_APP = None class App: @@ -46,7 +50,25 @@ class App: self.formatter = formatter_factory(options.cwd, options.display_relative_path) # Without require_module, our _set_collections_basedir may fail - self.runtime = Runtime(isolated=True, require_module=True) + self.runtime = Runtime( + isolated=True, + require_module=True, + verbosity=options.verbosity, + ) + self.reqs = Reqs("ansible-lint") + package = "ansible-core" + if not self.reqs.matches( + package, + str(self.runtime.version), + ): # pragma: no cover + msg = f"ansible-lint requires {package}{','.join(str(x) for x in self.reqs[package])} and current version is {self.runtime.version}" + logging.error(msg) + sys.exit(RC.INVALID_CONFIG) + + # pylint: disable=import-outside-toplevel + from ansiblelint.yaml_utils import load_yamllint_config + + self.yamllint_config = load_yamllint_config() def render_matches(self, matches: list[MatchError]) -> None: """Display given matches (if they are not fixed).""" @@ -54,7 +76,7 @@ class App: if isinstance( self.formatter, - (formatters.CodeclimateJSONFormatter, formatters.SarifFormatter), + formatters.CodeclimateJSONFormatter | formatters.SarifFormatter, ): # If formatter CodeclimateJSONFormatter or SarifFormatter is chosen, # then print only the matches in JSON @@ -205,7 +227,7 @@ class App: ignore_file.writelines(sorted(lines)) elif matched_rules and not self.options.quiet: console_stderr.print( - "Read [link=https://ansible-lint.readthedocs.io/configuring/#ignoring-rules-for-entire-files]documentation[/link] for instructions on how to ignore specific rule violations.", + "Read [link=https://ansible.readthedocs.io/projects/lint/configuring/#ignoring-rules-for-entire-files]documentation[/link] for instructions on how to ignore specific rule violations.", ) # Do not deprecate the old tags just yet. Why? Because it is not currently feasible @@ -223,7 +245,7 @@ class App: if self.options.write_list and "yaml" in self.options.skip_list: _logger.warning( - "You specified '--write', but no files can be modified " + "You specified '--fix', but no files can be modified " "because 'yaml' is in 'skip_list'.", ) @@ -332,7 +354,10 @@ class App: if self.options.profile: msg += f" Profile '{self.options.profile}' was required" if summary.passed_profile: - msg += f", but only '{summary.passed_profile}' profile passed." + if summary.passed_profile == self.options.profile: + msg += ", and it passed." + else: + msg += f", but '{summary.passed_profile}' profile passed." else: msg += "." elif summary.passed_profile: @@ -378,8 +403,19 @@ def _sanitize_list_options(tag_list: list[str]) -> list[str]: @lru_cache -def get_app(*, offline: bool | None = None) -> App: +def get_app(*, offline: bool | None = None, cached: bool = False) -> App: """Return the application instance, caching the return value.""" + # Avoids ever running the app initialization twice if cached argument + # is mentioned. + if cached: + if offline is not None: + msg = ( + "get_app should never be called with other arguments when cached=True." + ) + raise RuntimeError(msg) + if cached and _CACHED_APP is not None: + return _CACHED_APP + if offline is None: offline = default_options.offline diff --git a/src/ansiblelint/cli.py b/src/ansiblelint/cli.py index c9178a7..ce8d9ec 100644 --- a/src/ansiblelint/cli.py +++ b/src/ansiblelint/cli.py @@ -1,4 +1,5 @@ """CLI parser setup and helpers.""" + from __future__ import annotations import argparse @@ -7,7 +8,7 @@ import os import sys from argparse import Namespace from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any from ansiblelint.config import ( DEFAULT_KINDS, @@ -16,7 +17,7 @@ from ansiblelint.config import ( Options, log_entries, ) -from ansiblelint.constants import CUSTOM_RULESDIR_ENVVAR, DEFAULT_RULESDIR, RC +from ansiblelint.constants import CUSTOM_RULESDIR_ENVVAR, DEFAULT_RULESDIR, EPILOG, RC from ansiblelint.file_utils import ( Lintable, abspath, @@ -29,7 +30,7 @@ from ansiblelint.schemas.main import validate_file_schema from ansiblelint.yaml_utils import clean_json if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence _logger = logging.getLogger(__name__) @@ -91,7 +92,7 @@ def load_config(config_file: str | None) -> tuple[dict[Any, Any], str | None]: config = clean_json(config_lintable.data) if not isinstance(config, dict): msg = "Schema failed to properly validate the config file." - raise RuntimeError(msg) + raise TypeError(msg) config["config_file"] = config_path config_dir = os.path.dirname(config_path) expand_to_normalized_paths(config, config_dir) @@ -134,7 +135,7 @@ class AbspathArgAction(argparse.Action): values: str | Sequence[Any] | None, option_string: str | None = None, ) -> None: - if isinstance(values, (str, Path)): + if isinstance(values, str | Path): values = [values] if values: normalized_values = [ @@ -145,7 +146,7 @@ class AbspathArgAction(argparse.Action): class WriteArgAction(argparse.Action): - """Argparse action to handle the --write flag with optional args.""" + """Argparse action to handle the --fix flag with optional args.""" _default = "__default__" @@ -174,8 +175,8 @@ class WriteArgAction(argparse.Action): super().__init__( option_strings=option_strings, dest=dest, - nargs="?", # either 0 (--write) or 1 (--write=a,b,c) argument - const=self._default, # --write (no option) implicitly stores this + nargs="?", # either 0 (--fix) or 1 (--fix=a,b,c) argument + const=self._default, # --fix (no option) implicitly stores this default=default, type=type, choices=choices, @@ -194,8 +195,8 @@ class WriteArgAction(argparse.Action): lintables = getattr(namespace, "lintables", None) if not lintables and isinstance(values, str): # args are processed in order. - # If --write is after lintables, then that is not ambiguous. - # But if --write comes first, then it might actually be a lintable. + # If --fix is after lintables, then that is not ambiguous. + # But if --fix comes first, then it might actually be a lintable. maybe_lintable = Path(values) if maybe_lintable.exists(): namespace.lintables = [values] @@ -211,26 +212,40 @@ class WriteArgAction(argparse.Action): setattr(namespace, self.dest, values) @classmethod - def merge_write_list_config( + def merge_fix_list_config( cls, from_file: list[str], from_cli: list[str], ) -> list[str]: - """Combine the write_list from file config with --write CLI arg. + """Determine the write_list value based on cli vs config. + + When --fix is not passed from command line the from_cli is an empty list, + so we use the file. - Handles the implicit "all" when "__default__" is present and file config is empty. + When from_cli is not an empty list, we ignore the from_file value. """ - if not from_file or "none" in from_cli: - # --write is the same as --write=all - return ["all" if value == cls._default else value for value in from_cli] - # --write means use the config from the config file - from_cli = [value for value in from_cli if value != cls._default] - return from_file + from_cli + if not from_file: + arguments = ["all"] if from_cli == [cls._default] else from_cli + else: + arguments = from_file + for magic_value in ("none", "all"): + if magic_value in arguments and len(arguments) > 1: + msg = f"When passing '{magic_value}' to '--fix', you cannot pass other values." + raise RuntimeError( + msg, + ) + if len(arguments) == 1 and arguments[0] == "none": + arguments = [] + return arguments def get_cli_parser() -> argparse.ArgumentParser: """Initialize an argument parser.""" - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + epilog=EPILOG, + # Avoid rewrapping description and epilog + formatter_class=argparse.RawTextHelpFormatter, + ) listing_group = parser.add_mutually_exclusive_group() listing_group.add_argument( @@ -338,22 +353,16 @@ def get_cli_parser() -> argparse.ArgumentParser: help="Return non-zero exit code on warnings as well as errors", ) parser.add_argument( - "--write", + "--fix", dest="write_list", # this is a tri-state argument that takes an optional comma separated list: action=WriteArgAction, - help="Allow ansible-lint to reformat YAML files and run rule transforms " - "(Reformatting YAML files standardizes spacing, quotes, etc. " - "A rule transform can fix or simplify fixing issues identified by that rule). " + help="Allow ansible-lint to perform auto-fixes, including YAML reformatting. " "You can limit the effective rule transforms (the 'write_list') by passing a " "keywords 'all' or 'none' or a comma separated list of rule ids or rule tags. " - "YAML reformatting happens whenever '--write' or '--write=' is used. " - "'--write' and '--write=all' are equivalent: they allow all transforms to run. " - "The effective list of transforms comes from 'write_list' in the config file, " - "followed whatever '--write' args are provided on the commandline. " - "'--write=none' resets the list of transforms to allow reformatting YAML " - "without running any of the transforms (ie '--write=none,rule-id' will " - "ignore write_list in the config file and only run the rule-id transform).", + "YAML reformatting happens whenever '--fix' or '--fix=' is used. " + "'--fix' and '--fix=all' are equivalent: they allow all transforms to run. " + "Presence of --fix in command overrides config file value.", ) parser.add_argument( "--show-relpath", @@ -490,6 +499,7 @@ def merge_config(file_config: dict[Any, Any], cli_config: Options) -> Options: "enable_list": [], "only_builtins_allow_collections": [], "only_builtins_allow_modules": [], + "supported_ansible_also": [], # do not include "write_list" here. See special logic below. } @@ -506,6 +516,10 @@ def merge_config(file_config: dict[Any, Any], cli_config: Options) -> Options: for entry, default in lists_map.items(): if not getattr(cli_config, entry, None): setattr(cli_config, entry, default) + if cli_config.write_list is None: + cli_config.write_list = [] + elif cli_config.write_list == [WriteArgAction._default]: # noqa: SLF001 + cli_config.write_list = ["all"] return cli_config for entry in bools: @@ -513,8 +527,8 @@ def merge_config(file_config: dict[Any, Any], cli_config: Options) -> Options: v = getattr(cli_config, entry) or file_value setattr(cli_config, entry, v) - for entry, default in scalar_map.items(): - file_value = file_config.pop(entry, default) + for entry, default_scalar in scalar_map.items(): + file_value = file_config.pop(entry, default_scalar) v = getattr(cli_config, entry, None) or file_value setattr(cli_config, entry, v) @@ -533,7 +547,7 @@ def merge_config(file_config: dict[Any, Any], cli_config: Options) -> Options: setattr( cli_config, entry, - WriteArgAction.merge_write_list_config( + WriteArgAction.merge_fix_list_config( from_file=file_config.pop(entry, []), from_cli=getattr(cli_config, entry, []) or [], ), @@ -557,6 +571,13 @@ def merge_config(file_config: dict[Any, Any], cli_config: Options) -> Options: def get_config(arguments: list[str]) -> Options: """Extract the config based on given args.""" parser = get_cli_parser() + # translate deprecated options + for i, value in enumerate(arguments): + if arguments[i].startswith("--write"): + arguments[i] = value.replace("--write", "--fix") + _logger.warning( + "Replaced deprecated '--write' option with '--fix', change you call to avoid future regressions when we remove old option.", + ) options = Options(**vars(parser.parse_args(arguments))) # docs is not document, being used for internal documentation building diff --git a/src/ansiblelint/color.py b/src/ansiblelint/color.py index 8f31e1c..d72d98d 100644 --- a/src/ansiblelint/color.py +++ b/src/ansiblelint/color.py @@ -1,4 +1,5 @@ """Console coloring and terminal support.""" + from __future__ import annotations from typing import Any diff --git a/src/ansiblelint/config.py b/src/ansiblelint/config.py index 6164b10..ee9dea0 100644 --- a/src/ansiblelint/config.py +++ b/src/ansiblelint/config.py @@ -1,4 +1,5 @@ """Store configuration options as a singleton.""" + from __future__ import annotations import json @@ -67,7 +68,7 @@ DEFAULT_KINDS = [ {"requirements": "**/requirements.{yaml,yml}"}, # v2 and v1 {"playbook": "**/molecule/*/*.{yaml,yml}"}, # molecule playbooks {"yaml": "**/{.ansible-lint,.yamllint}"}, - {"changelog": "**/changelogs/changelog.yaml"}, + {"changelog": "**/changelogs/changelog.{yaml,yml}"}, {"yaml": "**/*.{yaml,yml}"}, {"yaml": "**/.*.{yaml,yml}"}, {"sanity-ignore-file": "**/tests/sanity/ignore-*.txt"}, @@ -98,22 +99,41 @@ BASE_KINDS = [ {"text/python": "**/*.py"}, ] +# File kinds that are recognized by ansible, used internally to force use of +# YAML 1.1 instead of 1.2 due to ansible-core dependency on pyyaml. +ANSIBLE_OWNED_KINDS = { + "handlers", + "galaxy", + "meta", + "meta-runtime", + "playbook", + "requirements", + "role-arg-spec", + "rulebook", + "tasks", + "vars", +} + PROFILES = yaml_from_file(Path(__file__).parent / "data" / "profiles.yml") LOOP_VAR_PREFIX = "^(__|{role}_)" @dataclass -class Options: # pylint: disable=too-many-instance-attributes,too-few-public-methods +class Options: # pylint: disable=too-many-instance-attributes """Store ansible-lint effective configuration options.""" + # Private attributes + _skip_ansible_syntax_check: bool = False + + # Public attributes cache_dir: Path | None = None colored: bool = True configured: bool = False - cwd: Path = Path(".") + cwd: Path = Path() display_relative_path: bool = True exclude_paths: list[str] = field(default_factory=list) - format: str = "brief" # noqa: A003 + format: str = "brief" lintables: list[str] = field(default_factory=list) list_rules: bool = False list_tags: bool = False @@ -152,6 +172,27 @@ class Options: # pylint: disable=too-many-instance-attributes,too-few-public-me version: bool = False # display version command list_profiles: bool = False # display profiles command ignore_file: Path | None = None + max_tasks: int = 100 + max_block_depth: int = 20 + # Refer to https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + _default_supported = ["2.15.", "2.16.", "2.17."] + supported_ansible_also: list[str] = field(default_factory=list) + + @property + def nodeps(self) -> bool: + """Returns value of nodeps feature.""" + # We do not want this to be cached as it would affect our testings. + return bool(int(os.environ.get("ANSIBLE_LINT_NODEPS", "0"))) + + def __post_init__(self) -> None: + """Extra initialization logic.""" + if self.nodeps: + self.offline = True + + @property + def supported_ansible(self) -> list[str]: + """Returns list of ansible versions that are considered supported.""" + return sorted([*self._default_supported, *self.supported_ansible_also]) options = Options() @@ -166,15 +207,6 @@ collection_list: list[str] = [] log_entries: list[tuple[int, str]] = [] -def get_rule_config(rule_id: str) -> dict[str, Any]: - """Get configurations for the rule ``rule_id``.""" - rule_config = options.rules.get(rule_id, {}) - if not isinstance(rule_config, dict): # pragma: no branch - msg = f"Invalid rule config for {rule_id}: {rule_config}" - raise RuntimeError(msg) - return rule_config - - @lru_cache def ansible_collections_path() -> str: """Return collection path variable for current version of Ansible.""" @@ -241,7 +273,6 @@ def guess_install_method() -> str: else: logging.debug("Skipping %s as it is not installed.", package_name) use_pip = False - # pylint: disable=broad-except except (AttributeError, ModuleNotFoundError) as exc: # On Fedora 36, we got a AttributeError exception from pip that we want to avoid # On NixOS, we got a ModuleNotFoundError exception from pip that we want to avoid @@ -269,6 +300,11 @@ def get_version_warning() -> str: # 0.1dev1 is special fallback version if __version__ == "0.1.dev1": # pragma: no cover return "" + pip = guess_install_method() + # If we do not know how to upgrade, we do not want to show any warnings + # about version. + if not pip: + return "" msg = "" data = {} @@ -309,9 +345,6 @@ def get_version_warning() -> str: msg = "[dim]You are using a pre-release version of ansible-lint.[/]" elif current_version < new_version: msg = f"""[warning]A new release of ansible-lint is available: [red]{current_version}[/] → [green][link={html_url}]{new_version}[/][/][/]""" - - pip = guess_install_method() - if pip: - msg += f" Upgrade by running: [info]{pip}[/]" + msg += f" Upgrade by running: [info]{pip}[/]" return msg diff --git a/src/ansiblelint/constants.py b/src/ansiblelint/constants.py index 6b8bd12..56cf71b 100644 --- a/src/ansiblelint/constants.py +++ b/src/ansiblelint/constants.py @@ -1,11 +1,26 @@ """Constants used by AnsibleLint.""" + from enum import Enum from pathlib import Path from typing import Literal DEFAULT_RULESDIR = Path(__file__).parent / "rules" CUSTOM_RULESDIR_ENVVAR = "ANSIBLE_LINT_CUSTOM_RULESDIR" -RULE_DOC_URL = "https://ansible-lint.readthedocs.io/rules/" +RULE_DOC_URL = "https://ansible.readthedocs.io/projects/lint/rules/" +SKIP_SCHEMA_UPDATE = "ANSIBLE_LINT_SKIP_SCHEMA_UPDATE" + +ENV_VARS_HELP = { + CUSTOM_RULESDIR_ENVVAR: "Used for adding another folder into the lookup path for new rules.", + "ANSIBLE_LINT_IGNORE_FILE": "Define it to override the name of the default ignore file `.ansible-lint-ignore`", + "ANSIBLE_LINT_WRITE_TMP": "Tells linter to dump fixes into different temp files instead of overriding original. Used internally for testing.", + SKIP_SCHEMA_UPDATE: "Tells ansible-lint to skip schema refresh.", + "ANSIBLE_LINT_NODEPS": "Avoids installing content dependencies and avoids performing checks that would fail when modules are not installed. Far less violations will be reported.", +} + +EPILOG = ( + "The following environment variables are also recognized but there is no guarantee that they will work in future versions:\n\n" + + "\n".join(f"{key}: {value}\n" for key, value in ENV_VARS_HELP.items()) +) # Not using an IntEnum because only starting with py3.11 it will evaluate it @@ -126,6 +141,36 @@ PLAYBOOK_TASK_KEYWORDS = [ "pre_tasks", "post_tasks", ] +PLAYBOOK_ROLE_KEYWORDS = [ + "any_errors_fatal", + "become", + "become_exe", + "become_flags", + "become_method", + "become_user", + "check_mode", + "collections", + "connection", + "debugger", + "delegate_facts", + "delegate_to", + "diff", + "environment", + "ignore_errors", + "ignore_unreachable", + "module_defaults", + "name", + "role", + "no_log", + "port", + "remote_user", + "run_once", + "tags", + "throttle", + "timeout", + "vars", + "when", +] NESTED_TASK_KEYS = [ "block", "always", diff --git a/src/ansiblelint/data/.yamllint b/src/ansiblelint/data/.yamllint new file mode 100644 index 0000000..6ff09f0 --- /dev/null +++ b/src/ansiblelint/data/.yamllint @@ -0,0 +1,25 @@ +extends: default +rules: + comments: + # https://github.com/prettier/prettier/issues/6780 + min-spaces-from-content: 1 + # https://github.com/adrienverge/yamllint/issues/384 + comments-indentation: false + document-start: disable + # 160 chars was the default used by old E204 rule, but + # you can easily change it or disable in your .yamllint file. + line-length: + max: 160 + # We are adding an extra space inside braces as that's how prettier does it + # and we are trying not to fight other linters. + braces: + min-spaces-inside: 0 # yamllint defaults to 0 + max-spaces-inside: 1 # yamllint defaults to 0 + # key-duplicates: + # forbid-duplicated-merge-keys: true # not enabled by default + octal-values: + forbid-implicit-octal: true # yamllint defaults to false + forbid-explicit-octal: true # yamllint defaults to false + # quoted-strings: + # quote-type: double + # required: only-when-needed diff --git a/src/ansiblelint/errors.py b/src/ansiblelint/errors.py index c8458b8..5ee2d6f 100644 --- a/src/ansiblelint/errors.py +++ b/src/ansiblelint/errors.py @@ -1,4 +1,5 @@ """Exceptions and error representations.""" + from __future__ import annotations import functools @@ -6,7 +7,6 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any from ansiblelint._internal.rules import BaseRule, RuntimeErrorRule -from ansiblelint.config import options from ansiblelint.file_utils import Lintable if TYPE_CHECKING: @@ -27,15 +27,9 @@ class WarnSource: message: str | None = None -class StrictModeError(RuntimeError): - """Raise when we encounter a warning in strict mode.""" - - def __init__( - self, - message: str = "Warning treated as error due to --strict option.", - ): - """Initialize a StrictModeError instance.""" - super().__init__(message) +@dataclass(frozen=True) +class RuleMatchTransformMeta: + """Additional metadata about a match error to be used during transformation.""" # pylint: disable=too-many-instance-attributes @@ -54,7 +48,6 @@ class MatchError(ValueError): # order matters for these: message: str = field(init=True, repr=False, default="") lintable: Lintable = field(init=True, repr=False, default=Lintable(name="")) - filename: str = field(init=True, repr=False, default="") tag: str = field(init=True, repr=False, default="") lineno: int = 1 @@ -65,13 +58,11 @@ class MatchError(ValueError): rule: BaseRule = field(hash=False, default=RuntimeErrorRule()) ignored: bool = False fixed: bool = False # True when a transform has resolved this MatchError + transform_meta: RuleMatchTransformMeta | None = None def __post_init__(self) -> None: """Can be use by rules that can report multiple errors type, so we can still filter by them.""" - if not self.lintable and self.filename: - self.lintable = Lintable(self.filename) - elif self.lintable and not self.filename: - self.filename = self.lintable.name + self.filename = self.lintable.name # We want to catch accidental MatchError() which contains no useful # information. When no arguments are passed, the '_message' field is @@ -104,11 +95,22 @@ class MatchError(ValueError): msg = "MatchError called incorrectly as column numbers start with 1" raise RuntimeError(msg) + self.lineno += self.lintable.line_offset + + # We make the lintable aware that we found a match inside it, as this + # can be used to skip running other rules that do require current one + # to pass. + self.lintable.matches.append(self) + @functools.cached_property def level(self) -> str: """Return the level of the rule: error, warning or notice.""" - if not self.ignored and {self.tag, self.rule.id, *self.rule.tags}.isdisjoint( - options.warn_list, + if ( + not self.ignored + and self.rule.options + and {self.tag, self.rule.id, *self.rule.tags}.isdisjoint( + self.rule.options.warn_list, + ) ): return "error" return "warning" @@ -128,6 +130,10 @@ class MatchError(ValueError): self.details, ) + def __str__(self) -> str: + """Return a MatchError instance string representation.""" + return self.__repr__() + @property def position(self) -> str: """Return error positioning, with column number if available.""" diff --git a/src/ansiblelint/file_utils.py b/src/ansiblelint/file_utils.py index 15c92d2..04ce3cd 100644 --- a/src/ansiblelint/file_utils.py +++ b/src/ansiblelint/file_utils.py @@ -1,4 +1,5 @@ """Utility functions related to file operations.""" + from __future__ import annotations import copy @@ -16,12 +17,15 @@ import wcmatch.pathlib import wcmatch.wcmatch from yaml.error import YAMLError -from ansiblelint.config import BASE_KINDS, Options, options +from ansiblelint.app import get_app +from ansiblelint.config import ANSIBLE_OWNED_KINDS, BASE_KINDS, Options, options from ansiblelint.constants import CONFIG_FILENAMES, FileType, States if TYPE_CHECKING: from collections.abc import Iterator, Sequence + from ansiblelint.errors import MatchError + _logger = logging.getLogger(__package__) @@ -69,9 +73,9 @@ def is_relative_to(path: Path, *other: Any) -> bool: """Return True if the path is relative to another path or False.""" try: path.resolve().absolute().relative_to(*other) - return True except ValueError: return False + return True def normpath_path(path: str | Path) -> Path: @@ -197,6 +201,10 @@ class Lintable: self.exc: Exception | None = None # Stores data loading exceptions self.parent = parent self.explicit = False # Indicates if the file was explicitly provided or was indirectly included. + self.line_offset = ( + 0 # Amount to offset line numbers by to get accurate position + ) + self.matches: list[MatchError] = [] if isinstance(name, str): name = Path(name) @@ -219,7 +227,12 @@ class Lintable: parts = self.path.parent.parts if "roles" in parts: role = self.path - while role.parent.name != "roles" and role.name: + roles_path = get_app(cached=True).runtime.config.default_roles_path + while ( + str(role.parent.absolute()) not in roles_path + and role.parent.name != "roles" + and role.name + ): role = role.parent if role.exists(): self.role = role.name @@ -252,7 +265,12 @@ class Lintable: self.parent = _guess_parent(self) if self.kind == "yaml": - _ = self.data # pylint: disable=pointless-statement + _ = self.data + + def __del__(self) -> None: + """Clean up temporary files when the instance is cleaned up.""" + if hasattr(self, "file"): + self.file.close() def _guess_kind(self) -> None: if self.kind == "yaml": @@ -350,10 +368,16 @@ class Lintable: lintable.write(force=True) """ - if not force and not self.updated: + dump_filename = self.path.expanduser().resolve() + if os.environ.get("ANSIBLE_LINT_WRITE_TMP", "0") == "1": + dump_filename = dump_filename.with_suffix( + f".tmp{dump_filename.suffix}", + ) + elif not force and not self.updated: # No changes to write. return - self.path.expanduser().resolve().write_text( + + dump_filename.write_text( self._content or "", encoding="utf-8", ) @@ -372,6 +396,16 @@ class Lintable: """Return user friendly representation of a lintable.""" return f"{self.name} ({self.kind})" + def is_owned_by_ansible(self) -> bool: + """Return true for YAML files that are managed by Ansible.""" + return self.kind in ANSIBLE_OWNED_KINDS + + def failed(self) -> bool: + """Return true if we already found syntax-check errors on this file.""" + return any( + match.rule.id in ("syntax-check", "load-failure") for match in self.matches + ) + @property def data(self) -> Any: """Return loaded data representation for current file, if possible.""" @@ -396,7 +430,11 @@ class Lintable: # pylint: disable=import-outside-toplevel from ansiblelint.skip_utils import append_skipped_rules - self.state = append_skipped_rules(self.state, self) + # pylint: disable=possibly-used-before-assignment + self.state = append_skipped_rules( + self.state, + self, + ) else: logging.debug( "data set to None for %s due to being '%s' (%s) kind.", @@ -513,7 +551,7 @@ def expand_dirs_in_lintables(lintables: set[Lintable]) -> None: for item in copy.copy(lintables): if item.path.is_dir(): for filename in all_files: - if filename.startswith(str(item.path)): + if filename.startswith((str(item.path), str(item.path.absolute()))): lintables.add(Lintable(filename)) diff --git a/src/ansiblelint/formatters/__init__.py b/src/ansiblelint/formatters/__init__.py index 9ddca00..187d803 100644 --- a/src/ansiblelint/formatters/__init__.py +++ b/src/ansiblelint/formatters/__init__.py @@ -1,4 +1,5 @@ """Output formatters.""" + from __future__ import annotations import hashlib @@ -14,6 +15,7 @@ from ansiblelint.version import __version__ if TYPE_CHECKING: from ansiblelint.errors import MatchError + from ansiblelint.rules import BaseRule # type: ignore[attr-defined] T = TypeVar("T", bound="BaseFormatter") # type: ignore[type-arg] @@ -27,6 +29,7 @@ class BaseFormatter(Generic[T]): ---- base_dir (str|Path): reference directory against which display relative path. display_relative_path (bool): whether to show path as relative or absolute + """ def __init__(self, base_dir: str | Path, display_relative_path: bool) -> None: @@ -143,7 +146,7 @@ class CodeclimateJSONFormatter(BaseFormatter[Any]): """Format a list of match errors as a JSON string.""" if not isinstance(matches, list): msg = f"The {self.__class__} was expecting a list of MatchError." - raise RuntimeError(msg) + raise TypeError(msg) result = [] for match in matches: @@ -210,7 +213,7 @@ class SarifFormatter(BaseFormatter[Any]): """Format a list of match errors as a JSON string.""" if not isinstance(matches, list): msg = f"The {self.__class__} was expecting a list of MatchError." - raise RuntimeError(msg) + raise TypeError(msg) root_path = Path(str(self.base_dir)).as_uri() root_path = root_path + "/" if not root_path.endswith("/") else root_path @@ -264,7 +267,7 @@ class SarifFormatter(BaseFormatter[Any]): "text": str(match.message), }, "defaultConfiguration": { - "level": self._to_sarif_level(match), + "level": self.get_sarif_rule_severity_level(match.rule), }, "help": { "text": str(match.rule.description), @@ -275,12 +278,21 @@ class SarifFormatter(BaseFormatter[Any]): return rule def _to_sarif_result(self, match: MatchError) -> dict[str, Any]: + # https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790898 + if match.level not in ("warning", "error", "note", "none"): + msg = "Unexpected failure to map '%s' level to SARIF." + raise RuntimeError( + msg, + match.level, + ) + result: dict[str, Any] = { "ruleId": match.tag, + "level": self.get_sarif_result_severity_level(match), "message": { - "text": str(match.details) - if str(match.details) - else str(match.message), + "text": ( + str(match.details) if str(match.details) else str(match.message) + ), }, "locations": [ { @@ -303,6 +315,37 @@ class SarifFormatter(BaseFormatter[Any]): return result @staticmethod - def _to_sarif_level(match: MatchError) -> str: - # sarif accepts only 4 levels: error, warning, note, none - return match.level + def get_sarif_rule_severity_level(rule: BaseRule) -> str: + """General SARIF severity level for a rule. + + Note: Can differ from an actual result/match severity. + Possible values: "none", "note", "warning", "error" + + see: https://github.com/oasis-tcs/sarif-spec/blob/123e95847b13fbdd4cbe2120fa5e33355d4a042b/Schemata/sarif-schema-2.1.0.json#L1934-L1939 + """ + if rule.severity in ["VERY_HIGH", "HIGH"]: + return "error" + + if rule.severity in ["MEDIUM", "LOW", "VERY_LOW"]: + return "warning" + + if rule.severity == "INFO": + return "note" + + return "none" + + @staticmethod + def get_sarif_result_severity_level(match: MatchError) -> str: + """SARIF severity level for an actual result/match. + + Possible values: "none", "note", "warning", "error" + + see: https://github.com/oasis-tcs/sarif-spec/blob/123e95847b13fbdd4cbe2120fa5e33355d4a042b/Schemata/sarif-schema-2.1.0.json#L2066-L2071 + """ + if not match.level: + return "none" + + if match.level in ["warning", "error"]: + return match.level + + return "note" diff --git a/src/ansiblelint/generate_docs.py b/src/ansiblelint/generate_docs.py index 1498a67..6e319fb 100644 --- a/src/ansiblelint/generate_docs.py +++ b/src/ansiblelint/generate_docs.py @@ -1,4 +1,5 @@ """Utils to generate rules documentation.""" + import logging from collections.abc import Iterable @@ -9,7 +10,7 @@ from rich.table import Table from ansiblelint.config import PROFILES from ansiblelint.constants import RULE_DOC_URL -from ansiblelint.rules import RulesCollection +from ansiblelint.rules import RulesCollection, TransformMixin DOC_HEADER = """ # Default Rules @@ -27,6 +28,8 @@ def rules_as_str(rules: RulesCollection) -> RenderableType: """Return rules as string.""" table = Table(show_header=False, header_style="title", box=box.SIMPLE) for rule in rules.alphabetical(): + if issubclass(rule.__class__, TransformMixin): + rule.tags.insert(0, "autofix") tag = f"[dim] ({', '.join(rule.tags)})[/dim]" if rule.tags else "" table.add_row( f"[link={RULE_DOC_URL}{rule.id}/]{rule.id}[/link]", @@ -56,6 +59,12 @@ def rules_as_md(rules: RulesCollection) -> str: result += f"\n\n## {title}\n\n**{rule.shortdesc}**\n\n{description}" + # Safety net for preventing us from adding autofix to rules and + # forgetting to mention it inside their documentation. + if "autofix" in rule.tags and "autofix" not in rule.description: + msg = f"Rule {rule.id} is invalid because it has 'autofix' tag but this ability is not documented in its description." + raise RuntimeError(msg) + return result diff --git a/src/ansiblelint/loaders.py b/src/ansiblelint/loaders.py index 49e38f1..c369c89 100644 --- a/src/ansiblelint/loaders.py +++ b/src/ansiblelint/loaders.py @@ -1,11 +1,12 @@ """Utilities for loading various files.""" + from __future__ import annotations import logging import os -from collections import defaultdict, namedtuple +from collections import defaultdict from functools import partial -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple import yaml from yaml import YAMLError @@ -19,7 +20,14 @@ except (ImportError, AttributeError): if TYPE_CHECKING: from pathlib import Path -IgnoreFile = namedtuple("IgnoreFile", "default alternative") + +class IgnoreFile(NamedTuple): + """IgnoreFile n.""" + + default: str + alternative: str + + IGNORE_FILE = IgnoreFile(".ansible-lint-ignore", ".config/ansible-lint-ignore.txt") yaml_load = partial(yaml.load, Loader=FullLoader) diff --git a/src/ansiblelint/logger.py b/src/ansiblelint/logger.py index f0477cd..cb3bb19 100644 --- a/src/ansiblelint/logger.py +++ b/src/ansiblelint/logger.py @@ -1,4 +1,5 @@ """Utils related to logging.""" + import logging import time from collections.abc import Iterator @@ -17,15 +18,3 @@ def timed_info(msg: Any, *args: Any) -> Iterator[None]: finally: elapsed = time.time() - start _logger.info(msg + " (%.2fs)", *(*args, elapsed)) # noqa: G003 - - -def warn_or_fail(message: str) -> None: - """Warn or fail depending on the strictness level.""" - # pylint: disable=import-outside-toplevel - from ansiblelint.config import options - from ansiblelint.errors import StrictModeError - - if options.strict: - raise StrictModeError(message) - - _logger.warning(message) diff --git a/src/ansiblelint/requirements.py b/src/ansiblelint/requirements.py new file mode 100644 index 0000000..96381b9 --- /dev/null +++ b/src/ansiblelint/requirements.py @@ -0,0 +1,28 @@ +"""Utilities for checking python packages requirements.""" + +import importlib_metadata +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import Version + + +class Reqs(dict[str, SpecifierSet]): + """Utility class for working with package dependencies.""" + + reqs: dict[str, SpecifierSet] + + def __init__(self, name: str = "ansible-lint") -> None: + """Load linter metadata requirements.""" + for req_str in importlib_metadata.metadata(name).json["requires_dist"]: + req = Requirement(req_str) + if req.name: + self[req.name] = req.specifier + + def matches(self, req_name: str, req_version: str | Version) -> bool: + """Verify if given version is matching current metadata dependencies.""" + if req_name not in self: + return False + return all( + specifier.contains(str(req_version), prereleases=True) + for specifier in self[req_name] + ) diff --git a/src/ansiblelint/rules/__init__.py b/src/ansiblelint/rules/__init__.py index acb7df1..a1743a0 100644 --- a/src/ansiblelint/rules/__init__.py +++ b/src/ansiblelint/rules/__init__.py @@ -1,4 +1,5 @@ """All internal ansible-lint rules.""" + from __future__ import annotations import copy @@ -23,7 +24,7 @@ from ansiblelint._internal.rules import ( WarningRule, ) from ansiblelint.app import App, get_app -from ansiblelint.config import PROFILES, Options, get_rule_config +from ansiblelint.config import PROFILES, Options from ansiblelint.config import options as default_options from ansiblelint.constants import LINE_NUMBER_KEY, RULE_DOC_URL, SKIPPED_RULES_KEY from ansiblelint.errors import MatchError @@ -32,6 +33,8 @@ from ansiblelint.file_utils import Lintable, expand_paths_vars if TYPE_CHECKING: from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.errors import RuleMatchTransformMeta + _logger = logging.getLogger(__name__) match_types = { @@ -53,11 +56,6 @@ class AnsibleLintRule(BaseRule): """Return rule documentation url.""" return RULE_DOC_URL + self.id + "/" - @property - def rule_config(self) -> dict[str, Any]: - """Retrieve rule specific configuration.""" - return get_rule_config(self.id) - def get_config(self, key: str) -> Any: """Return a configured value for given key string.""" return self.rule_config.get(key, None) @@ -78,6 +76,7 @@ class AnsibleLintRule(BaseRule): details: str = "", filename: Lintable | None = None, tag: str = "", + transform_meta: RuleMatchTransformMeta | None = None, ) -> MatchError: """Instantiate a new MatchError.""" match = MatchError( @@ -87,13 +86,14 @@ class AnsibleLintRule(BaseRule): lintable=filename or Lintable(""), rule=copy.copy(self), tag=tag, + transform_meta=transform_meta, ) # search through callers to find one of the match* methods frame = inspect.currentframe() match_type: str | None = None while not match_type and frame is not None: func_name = frame.f_code.co_name - match_type = match_types.get(func_name, None) + match_type = match_types.get(func_name) if match_type: # add the match_type to the match match.match_type = match_type @@ -109,8 +109,8 @@ class AnsibleLintRule(BaseRule): match.task = task if not match.details: match.details = "Task/Handler: " + ansiblelint.utils.task_to_str(task) - if match.lineno < task[LINE_NUMBER_KEY]: - match.lineno = task[LINE_NUMBER_KEY] + + match.lineno = max(match.lineno, task[LINE_NUMBER_KEY]) def matchlines(self, file: Lintable) -> list[MatchError]: matches: list[MatchError] = [] @@ -224,7 +224,16 @@ class AnsibleLintRule(BaseRule): if isinstance(yaml, str): if yaml.startswith("$ANSIBLE_VAULT"): return [] - return [MatchError(lintable=file, rule=LoadingFailureRule())] + if self._collection is None: + msg = f"Rule {self.id} was not added to a collection." + raise RuntimeError(msg) + return [ + # pylint: disable=E1136 + MatchError( + lintable=file, + rule=self._collection["load-failure"], + ), + ] if not yaml: return matches @@ -250,7 +259,7 @@ class AnsibleLintRule(BaseRule): class TransformMixin: """A mixin for AnsibleLintRule to enable transforming files. - If ansible-lint is started with the ``--write`` option, then the ``Transformer`` + If ansible-lint is started with the ``--fix`` option, then the ``Transformer`` will call the ``transform()`` method for every MatchError identified if the rule that identified it subclasses this ``TransformMixin``. Only the rule that identified a MatchError can do transforms to fix that match. @@ -324,7 +333,6 @@ class TransformMixin: return target -# pylint: disable=too-many-nested-blocks def load_plugins( dirs: list[str], ) -> Iterator[AnsibleLintRule]: @@ -370,7 +378,7 @@ def load_plugins( class RulesCollection: """Container for a collection of rules.""" - def __init__( + def __init__( # pylint: disable=too-many-arguments self, rulesdirs: list[str] | list[Path] | None = None, options: Options | None = None, @@ -388,7 +396,7 @@ class RulesCollection: else: self.options = options self.profile = [] - self.app = app or get_app(offline=True) + self.app = app or get_app(cached=True) if profile_name: self.profile = PROFILES[profile_name] @@ -405,6 +413,8 @@ class RulesCollection: WarningRule(), ], ) + for rule in self.rules: + rule._collection = self # noqa: SLF001 for rule in load_plugins(rulesdirs_str): self.register(rule, conditional=conditional) self.rules = sorted(self.rules) @@ -443,6 +453,17 @@ class RulesCollection: """Return the length of the RulesCollection data.""" return len(self.rules) + def __getitem__(self, item: Any) -> BaseRule: + """Return a rule from inside the collection based on its id.""" + if not isinstance(item, str): + msg = f"Expected str but got {type(item)} when trying to access rule by it's id" + raise TypeError(msg) + for rule in self.rules: + if rule.id == item: + return rule + msg = f"Rule {item} is not present inside this collection." + raise ValueError(msg) + def extend(self, more: list[AnsibleLintRule]) -> None: """Combine rules.""" self.rules.extend(more) @@ -469,7 +490,7 @@ class RulesCollection: MatchError( message=str(exc), lintable=file, - rule=LoadingFailureRule(), + rule=self["load-failure"], tag=f"{LoadingFailureRule.id}[{exc.__class__.__name__.lower()}]", ), ] @@ -482,10 +503,18 @@ class RulesCollection: or rule.has_dynamic_tags or not set(rule.tags).union([rule.id]).isdisjoint(tags) ): - rule_definition = set(rule.tags) - rule_definition.add(rule.id) - if set(rule_definition).isdisjoint(skip_list): - matches.extend(rule.getmatches(file)) + if tags and set(rule.tags).union(list(rule.ids().keys())).isdisjoint( + tags, + ): + _logger.debug("Skipping rule %s", rule.id) + else: + _logger.debug("Running rule %s", rule.id) + rule_definition = set(rule.tags) + rule_definition.add(rule.id) + if set(rule_definition).isdisjoint(skip_list): + matches.extend(rule.getmatches(file)) + else: + _logger.debug("Skipping rule %s", rule.id) # some rules can produce matches with tags that are inside our # skip_list, so we need to cleanse the matches @@ -499,6 +528,15 @@ class RulesCollection: [rule.verbose() for rule in sorted(self.rules, key=lambda x: x.id)], ) + def known_tags(self) -> list[str]: + """Return a list of known tags, without returning no sub-tags.""" + tags = set() + for rule in self.rules: + tags.add(rule.id) + for tag in rule.tags: + tags.add(tag) + return sorted(tags) + def list_tags(self) -> str: """Return a string with all the tags in the RulesCollection.""" tag_desc = { @@ -525,11 +563,10 @@ class RulesCollection: msg = f"Rule {rule} does not have any of the required tags: {', '.join(tag_desc.keys())}" raise RuntimeError(msg) for tag in rule.tags: - for id_ in rule.ids(): - tags[tag].append(id_) + tags[tag] = list(rule.ids()) result = "# List of tags and rules they cover\n" for tag in sorted(tags): - desc = tag_desc.get(tag, None) + desc = tag_desc.get(tag) if desc: result += f"{tag}: # {desc}\n" else: @@ -550,6 +587,8 @@ def filter_rules_with_profile(rule_col: list[BaseRule], profile: str) -> None: included.add(rule) extends = PROFILES[extends].get("extends", None) for rule in rule_col.copy(): + if rule.unloadable: + continue if rule.id not in included: _logger.debug( "Unloading %s rule due to not being part of %s profile.", diff --git a/src/ansiblelint/rules/args.py b/src/ansiblelint/rules/args.py index 2acf32e..fb9f991 100644 --- a/src/ansiblelint/rules/args.py +++ b/src/ansiblelint/rules/args.py @@ -1,4 +1,5 @@ """Rule definition to validate task options.""" + from __future__ import annotations import contextlib @@ -8,7 +9,6 @@ import json import logging import re import sys -from functools import lru_cache from typing import TYPE_CHECKING, Any # pylint: disable=preferred-module @@ -18,11 +18,11 @@ from unittest.mock import patch # pylint: disable=reimported import ansible.module_utils.basic as mock_ansible_module from ansible.module_utils import basic -from ansible.plugins.loader import PluginLoadContext, module_loader from ansiblelint.constants import LINE_NUMBER_KEY from ansiblelint.rules import AnsibleLintRule, RulesCollection from ansiblelint.text import has_jinja +from ansiblelint.utils import load_plugin from ansiblelint.yaml_utils import clean_json if TYPE_CHECKING: @@ -66,12 +66,6 @@ workarounds_inject_map = { } -@lru_cache -def load_module(module_name: str) -> PluginLoadContext: - """Load plugin from module name and cache it.""" - return module_loader.find_plugin_with_context(module_name) - - class ValidationPassedError(Exception): """Exception to be raised when validation passes.""" @@ -103,7 +97,7 @@ class ArgsRule(AnsibleLintRule): task: Task, file: Lintable | None = None, ) -> list[MatchError]: - # pylint: disable=too-many-locals,too-many-return-statements + # pylint: disable=too-many-return-statements results: list[MatchError] = [] module_name = task["action"]["__ansible_module_original__"] failed_msg = None @@ -111,7 +105,7 @@ class ArgsRule(AnsibleLintRule): if module_name in self.module_aliases: return [] - loaded_module = load_module(module_name) + loaded_module = load_plugin(module_name) # https://github.com/ansible/ansible-lint/issues/3200 # since "ps1" modules cannot be executed on POSIX platforms, we will @@ -150,14 +144,10 @@ class ArgsRule(AnsibleLintRule): CustomAnsibleModule, ): spec = importlib.util.spec_from_file_location( - name=loaded_module.resolved_fqcn, + name=loaded_module.plugin_resolved_name, location=loaded_module.plugin_resolved_path, ) - if spec: - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - else: + if not spec: assert file is not None _logger.warning( "Unable to load module %s at %s:%s for options validation", @@ -166,6 +156,9 @@ class ArgsRule(AnsibleLintRule): task[LINE_NUMBER_KEY], ) return [] + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) try: if not hasattr(module, "main"): @@ -196,9 +189,9 @@ class ArgsRule(AnsibleLintRule): ) sanitized_results = self._sanitize_results(results, module_name) - return sanitized_results except ValidationPassedError: return [] + return sanitized_results # pylint: disable=unused-argument def _sanitize_results( diff --git a/src/ansiblelint/rules/avoid_implicit.py b/src/ansiblelint/rules/avoid_implicit.py index 8d1fe26..d752ec7 100644 --- a/src/ansiblelint/rules/avoid_implicit.py +++ b/src/ansiblelint/rules/avoid_implicit.py @@ -1,4 +1,5 @@ """Implementation of avoid-implicit rule.""" + # https://github.com/ansible/ansible-lint/issues/2501 from __future__ import annotations @@ -40,8 +41,8 @@ class AvoidImplicitRule(AnsibleLintRule): # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner def test_template_instead_of_copy_positive() -> None: """Positive test for avoid-implicit.""" diff --git a/src/ansiblelint/rules/command_instead_of_module.py b/src/ansiblelint/rules/command_instead_of_module.py index 068e430..538141b 100644 --- a/src/ansiblelint/rules/command_instead_of_module.py +++ b/src/ansiblelint/rules/command_instead_of_module.py @@ -1,4 +1,5 @@ """Implementation of command-instead-of-module rule.""" + # Copyright (c) 2013-2014 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -25,7 +26,7 @@ from pathlib import Path from typing import TYPE_CHECKING from ansiblelint.rules import AnsibleLintRule -from ansiblelint.utils import convert_to_boolean, get_first_cmd_arg, get_second_cmd_arg +from ansiblelint.utils import get_first_cmd_arg, get_second_cmd_arg if TYPE_CHECKING: from ansiblelint.file_utils import Lintable @@ -68,9 +69,17 @@ class CommandsInsteadOfModulesRule(AnsibleLintRule): } _executable_options = { - "git": ["branch", "log", "lfs"], - "systemctl": ["--version", "kill", "set-default", "show-environment", "status"], - "yum": ["clean"], + "git": ["branch", "log", "lfs", "rev-parse"], + "systemctl": [ + "--version", + "get-default", + "kill", + "set-default", + "set-property", + "show-environment", + "status", + ], + "yum": ["clean", "history", "info"], "rpm": ["--nodeps"], } @@ -97,9 +106,7 @@ class CommandsInsteadOfModulesRule(AnsibleLintRule): ): return False - if executable in self._modules and convert_to_boolean( - task["action"].get("warn", True), - ): + if executable in self._modules: message = "{0} used in place of {1} module" return message.format(executable, self._modules[executable]) return False @@ -108,8 +115,9 @@ class CommandsInsteadOfModulesRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/command_instead_of_shell.md b/src/ansiblelint/rules/command_instead_of_shell.md index 0abf69d..1e64d2c 100644 --- a/src/ansiblelint/rules/command_instead_of_shell.md +++ b/src/ansiblelint/rules/command_instead_of_shell.md @@ -28,3 +28,7 @@ environment variable expansion or chaining multiple commands using pipes. ansible.builtin.command: echo hello changed_when: false ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/command_instead_of_shell.py b/src/ansiblelint/rules/command_instead_of_shell.py index 346a071..789adca 100644 --- a/src/ansiblelint/rules/command_instead_of_shell.py +++ b/src/ansiblelint/rules/command_instead_of_shell.py @@ -1,4 +1,5 @@ """Implementation of command-instead-of-shell rule.""" + # Copyright (c) 2016 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -23,15 +24,18 @@ from __future__ import annotations import sys from typing import TYPE_CHECKING -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin from ansiblelint.utils import get_cmd_args if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task -class UseCommandInsteadOfShellRule(AnsibleLintRule): +class UseCommandInsteadOfShellRule(AnsibleLintRule, TransformMixin): """Use shell only when shell functionality is required.""" id = "command-instead-of-shell" @@ -62,13 +66,27 @@ class UseCommandInsteadOfShellRule(AnsibleLintRule): return not any(ch in jinja_stripped_cmd for ch in "&|<>;$\n*[]{}?`") return False + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if match.tag == "command-instead-of-shell": + target_task = self.seek(match.yaml_path, data) + for _ in range(len(target_task)): + k, v = target_task.popitem(False) + target_task["ansible.builtin.command" if "shell" in k else k] = v + match.fixed = True + # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/complexity.md b/src/ansiblelint/rules/complexity.md new file mode 100644 index 0000000..aa25a1e --- /dev/null +++ b/src/ansiblelint/rules/complexity.md @@ -0,0 +1,19 @@ +# complexity + +This rule aims to warn about Ansible content that seems to be overly complex, +suggesting refactoring for better readability and maintainability. + +## complexity[tasks] + +`complexity[tasks]` will be triggered if the total number of tasks inside a file +is above 100. If encountered, you should consider using +[`ansible.builtin.include_tasks`](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_tasks_module.html) +to split your tasks into smaller files. + +## complexity[nesting] + +`complexity[nesting]` will appear when a block contains too many tasks, by +default that number is 20 but it can be changed inside the configuration file by +defining `max_block_depth` value. + + Replace nested block with an include_tasks to make code easier to maintain. Maximum block depth allowed is ... diff --git a/src/ansiblelint/rules/complexity.py b/src/ansiblelint/rules/complexity.py new file mode 100644 index 0000000..04d92d0 --- /dev/null +++ b/src/ansiblelint/rules/complexity.py @@ -0,0 +1,115 @@ +"""Implementation of limiting number of tasks.""" + +from __future__ import annotations + +import re +import sys +from typing import TYPE_CHECKING, Any + +from ansiblelint.constants import LINE_NUMBER_KEY +from ansiblelint.rules import AnsibleLintRule, RulesCollection + +if TYPE_CHECKING: + from ansiblelint.config import Options + from ansiblelint.errors import MatchError + from ansiblelint.file_utils import Lintable + from ansiblelint.utils import Task + + +class ComplexityRule(AnsibleLintRule): + """Rule for limiting number of tasks inside a file.""" + + id = "complexity" + description = "There should be limited tasks executed inside any file" + severity = "MEDIUM" + tags = ["experimental", "idiom"] + version_added = "v6.18.0 (last update)" + _re_templated_inside = re.compile(r".*\{\{.*\}\}.*\w.*$") + + def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: + """Call matchplay for up to no_of_max_tasks inside file and return aggregate results.""" + results: list[MatchError] = [] + + if file.kind != "playbook": + return [] + tasks = data.get("tasks", []) + if not isinstance(self._collection, RulesCollection): + msg = "Rules cannot be run outside a rule collection." + raise TypeError(msg) + if len(tasks) > self._collection.options.max_tasks: + results.append( + self.create_matcherror( + message=f"Maximum tasks allowed in a play is {self._collection.options.max_tasks}.", + lineno=data[LINE_NUMBER_KEY], + tag=f"{self.id}[play]", + filename=file, + ), + ) + return results + + def matchtask(self, task: Task, file: Lintable | None = None) -> list[MatchError]: + """Check if the task is a block and count the number of items inside it.""" + results: list[MatchError] = [] + + if not isinstance(self._collection, RulesCollection): + msg = "Rules cannot be run outside a rule collection." + raise TypeError(msg) + + if task.action == "block/always/rescue": + block_depth = self.calculate_block_depth(task) + if block_depth > self._collection.options.max_block_depth: + results.append( + self.create_matcherror( + message=f"Replace nested block with an include_tasks to make code easier to maintain. Maximum block depth allowed is {self._collection.options.max_block_depth}.", + lineno=task[LINE_NUMBER_KEY], + tag=f"{self.id}[nesting]", + filename=file, + ), + ) + return results + + def calculate_block_depth(self, task: Task) -> int: + """Recursively calculate the block depth of a task.""" + if not isinstance(task.position, str): + raise NotImplementedError + return task.position.count(".block") + + +if "pytest" in sys.modules: + import pytest + + # pylint: disable=ungrouped-imports + from ansiblelint.runner import Runner + + @pytest.mark.parametrize( + ("file", "expected_results"), + ( + pytest.param( + "examples/playbooks/rule-complexity-pass.yml", + [], + id="pass", + ), + pytest.param( + "examples/playbooks/rule-complexity-fail.yml", + ["complexity[play]", "complexity[nesting]"], + id="fail", + ), + ), + ) + def test_complexity( + file: str, + expected_results: list[str], + monkeypatch: pytest.MonkeyPatch, + config_options: Options, + ) -> None: + """Test rule.""" + monkeypatch.setattr(config_options, "max_tasks", 5) + monkeypatch.setattr(config_options, "max_block_depth", 3) + collection = RulesCollection(options=config_options) + collection.register(ComplexityRule()) + results = Runner(file, rules=collection).run() + + assert len(results) == len(expected_results) + for i, result in enumerate(results): + assert result.rule.id == ComplexityRule.id, result + assert result.tag == expected_results[i] diff --git a/src/ansiblelint/rules/conftest.py b/src/ansiblelint/rules/conftest.py index f4df7a5..5a22ffd 100644 --- a/src/ansiblelint/rules/conftest.py +++ b/src/ansiblelint/rules/conftest.py @@ -1,3 +1,4 @@ """Makes pytest fixtures available.""" + # pylint: disable=wildcard-import,unused-wildcard-import from ansiblelint.testing.fixtures import * # noqa: F403 diff --git a/src/ansiblelint/rules/deprecated_bare_vars.py b/src/ansiblelint/rules/deprecated_bare_vars.py index 1756e92..7b1ab08 100644 --- a/src/ansiblelint/rules/deprecated_bare_vars.py +++ b/src/ansiblelint/rules/deprecated_bare_vars.py @@ -27,7 +27,7 @@ import sys from typing import TYPE_CHECKING, Any from ansiblelint.rules import AnsibleLintRule -from ansiblelint.text import has_glob, has_jinja +from ansiblelint.text import has_glob, has_jinja, is_fqcn_or_name if TYPE_CHECKING: from ansiblelint.file_utils import Lintable @@ -66,7 +66,7 @@ class UsingBareVariablesIsDeprecatedRule(AnsibleLintRule): # we just need to check that one variable, and not iterate over it like # it's a list. Otherwise, loop through and check all items. items = task[loop_type] - if not isinstance(items, (list, tuple)): + if not isinstance(items, list | tuple): items = [items] for var in items: return self._matchvar(var, task, loop_type) @@ -84,7 +84,11 @@ class UsingBareVariablesIsDeprecatedRule(AnsibleLintRule): task: dict[str, Any], loop_type: str, ) -> bool | str: - if isinstance(varstring, str) and not has_jinja(varstring): + if ( + isinstance(varstring, str) + and not has_jinja(varstring) + and is_fqcn_or_name(varstring) + ): valid = loop_type == "with_fileglob" and bool( has_jinja(varstring) or has_glob(varstring), ) @@ -121,4 +125,4 @@ if "pytest" in sys.modules: failure = "examples/playbooks/rule-deprecated-bare-vars-fail.yml" bad_runner = Runner(failure, rules=collection) errs = bad_runner.run() - assert len(errs) == 12 + assert len(errs) == 11 diff --git a/src/ansiblelint/rules/deprecated_local_action.md b/src/ansiblelint/rules/deprecated_local_action.md index c52eb9d..68f4345 100644 --- a/src/ansiblelint/rules/deprecated_local_action.md +++ b/src/ansiblelint/rules/deprecated_local_action.md @@ -19,3 +19,7 @@ This rule recommends using `delegate_to: localhost` instead of the ansible.builtin.debug: delegate_to: localhost # <-- recommended way to run on localhost ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/deprecated_local_action.py b/src/ansiblelint/rules/deprecated_local_action.py index fc3e4ff..4e09795 100644 --- a/src/ansiblelint/rules/deprecated_local_action.py +++ b/src/ansiblelint/rules/deprecated_local_action.py @@ -1,19 +1,33 @@ """Implementation for deprecated-local-action rule.""" + # Copyright (c) 2016, Tsukinowa Inc. <info@tsukinowa.jp> # Copyright (c) 2018, Ansible Project from __future__ import annotations +import copy +import logging +import os import sys +from pathlib import Path from typing import TYPE_CHECKING -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin +from ansiblelint.runner import get_matches +from ansiblelint.transformer import Transformer if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + from ansiblelint.config import Options + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task -class TaskNoLocalAction(AnsibleLintRule): +_logger = logging.getLogger(__name__) + + +class TaskNoLocalAction(AnsibleLintRule, TransformMixin): """Do not use 'local_action', use 'delegate_to: localhost'.""" id = "deprecated-local-action" @@ -35,11 +49,46 @@ class TaskNoLocalAction(AnsibleLintRule): return False + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if match.tag == self.id: + # we do not want perform a partial modification accidentally + original_target_task = self.seek(match.yaml_path, data) + target_task = copy.deepcopy(original_target_task) + for _ in range(len(target_task)): + k, v = target_task.popitem(False) + if k == "local_action": + if isinstance(v, dict): + module_name = v["module"] + target_task[module_name] = None + target_task["delegate_to"] = "localhost" + elif isinstance(v, str): + module_name, module_value = v.split(" ", 1) + target_task[module_name] = module_value + target_task["delegate_to"] = "localhost" + else: + _logger.debug( + "Ignored unexpected data inside %s transform.", + self.id, + ) + return + else: + target_task[k] = v + match.fixed = True + original_target_task.clear() + original_target_task.update(target_task) + # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from unittest import mock + + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner def test_local_action(default_rules_collection: RulesCollection) -> None: """Positive test deprecated_local_action.""" @@ -50,3 +99,34 @@ if "pytest" in sys.modules: assert len(results) == 1 assert results[0].tag == "deprecated-local-action" + + @mock.patch.dict(os.environ, {"ANSIBLE_LINT_WRITE_TMP": "1"}, clear=True) + def test_local_action_transform( + config_options: Options, + default_rules_collection: RulesCollection, + ) -> None: + """Test transform functionality for no-log-password rule.""" + playbook = Path("examples/playbooks/tasks/local_action.yml") + config_options.write_list = ["all"] + + config_options.lintables = [str(playbook)] + runner_result = get_matches( + rules=default_rules_collection, + options=config_options, + ) + transformer = Transformer(result=runner_result, options=config_options) + transformer.run() + matches = runner_result.matches + assert len(matches) == 3 + + orig_content = playbook.read_text(encoding="utf-8") + expected_content = playbook.with_suffix( + f".transformed{playbook.suffix}", + ).read_text(encoding="utf-8") + transformed_content = playbook.with_suffix(f".tmp{playbook.suffix}").read_text( + encoding="utf-8", + ) + + assert orig_content != transformed_content + assert expected_content == transformed_content + playbook.with_suffix(f".tmp{playbook.suffix}").unlink() diff --git a/src/ansiblelint/rules/deprecated_module.py b/src/ansiblelint/rules/deprecated_module.py index 03c9361..72e328f 100644 --- a/src/ansiblelint/rules/deprecated_module.py +++ b/src/ansiblelint/rules/deprecated_module.py @@ -1,4 +1,5 @@ """Implementation of deprecated-module rule.""" + # Copyright (c) 2018, Ansible Project from __future__ import annotations diff --git a/src/ansiblelint/rules/empty_string_compare.py b/src/ansiblelint/rules/empty_string_compare.py index 5c7cafc..6870ed2 100644 --- a/src/ansiblelint/rules/empty_string_compare.py +++ b/src/ansiblelint/rules/empty_string_compare.py @@ -1,4 +1,5 @@ """Implementation of empty-string-compare rule.""" + # Copyright (c) 2016, Will Thames and contributors # Copyright (c) 2018, Ansible Project @@ -54,8 +55,8 @@ class ComparisonToEmptyStringRule(AnsibleLintRule): # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner def test_rule_empty_string_compare_fail() -> None: """Test rule matches.""" diff --git a/src/ansiblelint/rules/fqcn.md b/src/ansiblelint/rules/fqcn.md index 0165477..a64a324 100644 --- a/src/ansiblelint/rules/fqcn.md +++ b/src/ansiblelint/rules/fqcn.md @@ -87,3 +87,7 @@ structure in a backward-compatible way by adding redirects like in # Use the FQCN for the builtin shell module. ansible.builtin.shell: ssh ssh_user@{{ ansible_ssh_host }} ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/fqcn.py b/src/ansiblelint/rules/fqcn.py index 768fb9e..b571db3 100644 --- a/src/ansiblelint/rules/fqcn.py +++ b/src/ansiblelint/rules/fqcn.py @@ -1,17 +1,19 @@ """Rule definition for usage of fully qualified collection names for builtins.""" + from __future__ import annotations import logging import sys from typing import TYPE_CHECKING, Any -from ansible.plugins.loader import module_loader +from ruamel.yaml.comments import CommentedSeq from ansiblelint.constants import LINE_NUMBER_KEY from ansiblelint.rules import AnsibleLintRule, TransformMixin +from ansiblelint.utils import load_plugin if TYPE_CHECKING: - from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ruamel.yaml.comments import CommentedMap from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable @@ -114,11 +116,16 @@ class FQCNBuiltinsRule(AnsibleLintRule, TransformMixin): task: Task, file: Lintable | None = None, ) -> list[MatchError]: - result = [] + result: list[MatchError] = [] + if file and file.failed(): + return result module = task["action"]["__ansible_module_original__"] + if not isinstance(module, str): + msg = "Invalid data for module." + raise TypeError(msg) if module not in self.module_aliases: - loaded_module = module_loader.find_plugin_with_context(module) + loaded_module = load_plugin(module) target = loaded_module.resolved_fqcn self.module_aliases[module] = target if target is None: @@ -137,40 +144,45 @@ class FQCNBuiltinsRule(AnsibleLintRule, TransformMixin): 1, ) if module != legacy_module: + if module == "ansible.builtin.include": + message = f"Avoid deprecated module ({module})" + details = "Use `ansible.builtin.include_task` or `ansible.builtin.import_tasks` instead." + else: + message = f"Use FQCN for builtin module actions ({module})." + details = f"Use `{module_alias}` or `{legacy_module}` instead." result.append( self.create_matcherror( - message=f"Use FQCN for builtin module actions ({module}).", - details=f"Use `{module_alias}` or `{legacy_module}` instead.", + message=message, + details=details, filename=file, lineno=task["__line__"], tag="fqcn[action-core]", ), ) - else: - if module.count(".") < 2: - result.append( - self.create_matcherror( - message=f"Use FQCN for module actions, such `{self.module_aliases[module]}`.", - details=f"Action `{module}` is not FQCN.", - filename=file, - lineno=task["__line__"], - tag="fqcn[action]", - ), - ) - # TODO(ssbarnea): Remove the c.g. and c.n. exceptions from here once # noqa: FIX002 - # community team is flattening these. - # https://github.com/ansible-community/community-topics/issues/147 - elif not module.startswith("community.general.") or module.startswith( - "community.network.", - ): - result.append( - self.create_matcherror( - message=f"You should use canonical module name `{self.module_aliases[module]}` instead of `{module}`.", - filename=file, - lineno=task["__line__"], - tag="fqcn[canonical]", - ), - ) + elif module.count(".") < 2: + result.append( + self.create_matcherror( + message=f"Use FQCN for module actions, such `{self.module_aliases[module]}`.", + details=f"Action `{module}` is not FQCN.", + filename=file, + lineno=task["__line__"], + tag="fqcn[action]", + ), + ) + # TODO(ssbarnea): Remove the c.g. and c.n. exceptions from here once # noqa: FIX002 + # community team is flattening these. + # https://github.com/ansible-community/community-topics/issues/147 + elif not module.startswith("community.general.") or module.startswith( + "community.network.", + ): + result.append( + self.create_matcherror( + message=f"You should use canonical module name `{self.module_aliases[module]}` instead of `{module}`.", + filename=file, + lineno=task["__line__"], + tag="fqcn[canonical]", + ), + ) return result def matchyaml(self, file: Lintable) -> list[MatchError]: @@ -220,6 +232,8 @@ class FQCNBuiltinsRule(AnsibleLintRule, TransformMixin): target_task = self.seek(match.yaml_path, data) # Unfortunately, a lot of data about Ansible content gets lost here, you only get a simple dict. # For now, just parse the error messages for the data about action names etc. and fix this later. + current_action = "" + new_action = "" if match.tag == "fqcn[action-core]": # split at the first bracket, cut off the last bracket and dot current_action = match.message.split("(")[1][:-2] @@ -233,6 +247,8 @@ class FQCNBuiltinsRule(AnsibleLintRule, TransformMixin): current_action = match.message.split("`")[3] new_action = match.message.split("`")[1] for _ in range(len(target_task)): + if isinstance(target_task, CommentedSeq): + continue k, v = target_task.popitem(False) target_task[new_action if k == current_action else k] = v match.fixed = True @@ -241,7 +257,7 @@ class FQCNBuiltinsRule(AnsibleLintRule, TransformMixin): # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: from ansiblelint.rules import RulesCollection - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.runner import Runner def test_fqcn_builtin_fail() -> None: """Test rule matches.""" @@ -269,7 +285,7 @@ if "pytest" in sys.modules: """Test rule matches.""" collection = RulesCollection() collection.register(FQCNBuiltinsRule()) - failure = "examples/collection/plugins/modules/deep/beta.py" + failure = "examples/.collection/plugins/modules/deep/beta.py" results = Runner(failure, rules=collection).run() assert len(results) == 1 assert results[0].tag == "fqcn[deep]" @@ -279,6 +295,6 @@ if "pytest" in sys.modules: """Test rule does not match.""" collection = RulesCollection() collection.register(FQCNBuiltinsRule()) - success = "examples/collection/plugins/modules/alpha.py" + success = "examples/.collection/plugins/modules/alpha.py" results = Runner(success, rules=collection).run() assert len(results) == 0 diff --git a/src/ansiblelint/rules/galaxy.md b/src/ansiblelint/rules/galaxy.md index 61fc5c5..d719e30 100644 --- a/src/ansiblelint/rules/galaxy.md +++ b/src/ansiblelint/rules/galaxy.md @@ -26,6 +26,8 @@ This rule can produce messages such: - `galaxy[tags]` - `galaxy.yaml` must have one of the required tags: `application`, `cloud`, `database`, `infrastructure`, `linux`, `monitoring`, `networking`, `security`, `storage`, `tools`, `windows`. +- `galaxy[invalid-dependency-version]` = Invalid collection metadata. Dependency + version spec range is invalid If you want to ignore some of the messages above, you can add any of them to the `ignore_list`. @@ -60,12 +62,14 @@ description: "..." # Changelog Details -This rule expects a `CHANGELOG.md` or `.rst` file in the collection root or a -`changelogs/changelog.yaml` file. +This rule expects a `CHANGELOG.md`, `CHANGELOG.rst`, +`changelogs/changelog.yaml`, or `changelogs/changelog.yml` file in the +collection root. -If a `changelogs/changelog.yaml` file exists, the schema will be checked. +If a `changelogs/changelog.yaml` or `changelogs/changelog.yml` file exists, the +schema will be checked. -## Minimum required changelog.yaml file +## Minimum required changelog.yaml/changelog.yml file ```yaml # changelog.yaml diff --git a/src/ansiblelint/rules/galaxy.py b/src/ansiblelint/rules/galaxy.py index 2f627f5..e9b21d3 100644 --- a/src/ansiblelint/rules/galaxy.py +++ b/src/ansiblelint/rules/galaxy.py @@ -1,11 +1,12 @@ """Implementation of GalaxyRule.""" + from __future__ import annotations import sys from functools import total_ordering from typing import TYPE_CHECKING, Any -from ansiblelint.constants import LINE_NUMBER_KEY +from ansiblelint.constants import FILENAME_KEY, LINE_NUMBER_KEY from ansiblelint.rules import AnsibleLintRule if TYPE_CHECKING: @@ -27,6 +28,7 @@ class GalaxyRule(AnsibleLintRule): "galaxy[version-missing]": "galaxy.yaml should have version tag.", "galaxy[version-incorrect]": "collection version should be greater than or equal to 1.0.0", "galaxy[no-runtime]": "meta/runtime.yml file not found.", + "galaxy[invalid-dependency-version]": "Invalid collection metadata. Dependency version spec range is invalid", } def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: @@ -39,6 +41,7 @@ class GalaxyRule(AnsibleLintRule): "application", "cloud", "database", + "eda", "infrastructure", "linux", "monitoring", @@ -55,6 +58,7 @@ class GalaxyRule(AnsibleLintRule): changelog_found = 0 changelog_paths = [ base_path / "changelogs" / "changelog.yaml", + base_path / "changelogs" / "changelog.yml", base_path / "CHANGELOG.rst", base_path / "CHANGELOG.md", ] @@ -62,8 +66,21 @@ class GalaxyRule(AnsibleLintRule): for path in changelog_paths: if path.is_file(): changelog_found = 1 - - galaxy_tag_list = data.get("tags", None) + galaxy_tag_list = data.get("tags") + collection_deps = data.get("dependencies") + if collection_deps: + for dep, ver in collection_deps.items(): + if ( + dep not in [LINE_NUMBER_KEY, FILENAME_KEY] + and len(str(ver).strip()) == 0 + ): + results.append( + self.create_matcherror( + message=f"Invalid collection metadata. Dependency version spec range is invalid for '{dep}'.", + tag="galaxy[invalid-dependency-version]", + filename=file, + ), + ) # Changelog Check - building off Galaxy rule as there is no current way to check # for a nonexistent file @@ -108,7 +125,6 @@ class GalaxyRule(AnsibleLintRule): results.append( self.create_matcherror( message="collection version should be greater than or equal to 1.0.0", - # pylint: disable=protected-access lineno=version._line_number, # noqa: SLF001 tag="galaxy[version-incorrect]", filename=file, @@ -154,7 +170,7 @@ class Version: def _coerce(other: object) -> Version: if isinstance(other, str): other = Version(other) - if isinstance(other, (int, float)): + if isinstance(other, int | float): other = Version(str(other)) if isinstance(other, Version): return other @@ -172,7 +188,7 @@ if "pytest" in sys.modules: """Positive test for collection version in galaxy.""" collection = RulesCollection() collection.register(GalaxyRule()) - success = "examples/collection/galaxy.yml" + success = "examples/.collection/galaxy.yml" good_runner = Runner(success, rules=collection) assert [] == good_runner.run() @@ -189,7 +205,7 @@ if "pytest" in sys.modules: """Test for no collection version in galaxy.""" collection = RulesCollection() collection.register(GalaxyRule()) - failure = "examples/no_collection_version/galaxy.yml" + failure = "examples/.no_collection_version/galaxy.yml" bad_runner = Runner(failure, rules=collection) errs = bad_runner.run() assert len(errs) == 1 @@ -222,17 +238,25 @@ if "pytest" in sys.modules: id="pass", ), pytest.param( - "examples/collection/galaxy.yml", + "examples/.collection/galaxy.yml", ["schema[galaxy]"], id="schema", ), pytest.param( - "examples/no_changelog/galaxy.yml", + "examples/.invalid_dependencies/galaxy.yml", + [ + "galaxy[invalid-dependency-version]", + "galaxy[invalid-dependency-version]", + ], + id="invalid-dependency-version", + ), + pytest.param( + "examples/.no_changelog/galaxy.yml", ["galaxy[no-changelog]"], id="no-changelog", ), pytest.param( - "examples/no_collection_version/galaxy.yml", + "examples/.no_collection_version/galaxy.yml", ["schema[galaxy]", "galaxy[version-missing]"], id="no-collection-version", ), diff --git a/src/ansiblelint/rules/ignore_errors.py b/src/ansiblelint/rules/ignore_errors.py index 4144f2d..29f0408 100644 --- a/src/ansiblelint/rules/ignore_errors.py +++ b/src/ansiblelint/rules/ignore_errors.py @@ -1,4 +1,5 @@ """IgnoreErrorsRule used with ansible-lint.""" + from __future__ import annotations import sys @@ -44,7 +45,7 @@ if "pytest" in sys.modules: import pytest if TYPE_CHECKING: - from ansiblelint.testing import RunFromText # pylint: disable=ungrouped-imports + from ansiblelint.testing import RunFromText IGNORE_ERRORS_TRUE = """ - hosts: all diff --git a/src/ansiblelint/rules/inline_env_var.py b/src/ansiblelint/rules/inline_env_var.py index f578fb7..1f0747e 100644 --- a/src/ansiblelint/rules/inline_env_var.py +++ b/src/ansiblelint/rules/inline_env_var.py @@ -1,4 +1,5 @@ """Implementation of inside-env-var rule.""" + # Copyright (c) 2016 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -48,7 +49,6 @@ class EnvVarsInCommandRule(AnsibleLintRule): "executable", "removes", "stdin", - "warn", "stdin_add_newline", "strip_empty_ends", "cmd", diff --git a/src/ansiblelint/rules/jinja.md b/src/ansiblelint/rules/jinja.md index 8e1732e..e4720d7 100644 --- a/src/ansiblelint/rules/jinja.md +++ b/src/ansiblelint/rules/jinja.md @@ -12,7 +12,7 @@ version can report: As jinja2 syntax is closely following Python one we aim to follow [black](https://black.readthedocs.io/en/stable/) formatting rules. If you are -curious how black would reformat a small sniped feel free to visit +curious how black would reformat a small snippet feel free to visit [online black formatter](https://black.vercel.app/) site. Keep in mind to not include the entire jinja2 template, so instead of `{{ 1+2==3 }}`, do paste only `1+2==3`. @@ -53,3 +53,7 @@ In its current form, this rule presents the following limitations: does not support tilde as a binary operator. Example: `{{ a ~ b }}`. - Jinja2 blocks that use dot notation with numbers are ignored because python and black do not allow it. Example: `{{ foo.0.bar }}` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/jinja.py b/src/ansiblelint/rules/jinja.py index 08254bc..ff124a8 100644 --- a/src/ansiblelint/rules/jinja.py +++ b/src/ansiblelint/rules/jinja.py @@ -1,28 +1,35 @@ """Rule for checking content of jinja template strings.""" + from __future__ import annotations import logging +import os import re import sys -from collections import namedtuple +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple import black import jinja2 -from ansible.errors import AnsibleError, AnsibleParserError +from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleParserError from ansible.parsing.yaml.objects import AnsibleUnicode from jinja2.exceptions import TemplateSyntaxError from ansiblelint.constants import LINE_NUMBER_KEY +from ansiblelint.errors import RuleMatchTransformMeta from ansiblelint.file_utils import Lintable -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin +from ansiblelint.runner import get_matches from ansiblelint.skip_utils import get_rule_skips_from_line from ansiblelint.text import has_jinja from ansiblelint.utils import parse_yaml_from_file, template from ansiblelint.yaml_utils import deannotate, nested_items_path if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + from ansiblelint.config import Options from ansiblelint.errors import MatchError from ansiblelint.utils import Task @@ -30,7 +37,14 @@ if TYPE_CHECKING: _logger = logging.getLogger(__package__) KEYWORDS_WITH_IMPLICIT_TEMPLATE = ("changed_when", "failed_when", "until", "when") -Token = namedtuple("Token", "lineno token_type value") + +class Token(NamedTuple): + """Token.""" + + lineno: int + token_type: str + value: str + ignored_re = re.compile( "|".join( # noqa: FLY002 @@ -53,7 +67,27 @@ ignored_re = re.compile( ) -class JinjaRule(AnsibleLintRule): +@dataclass(frozen=True) +class JinjaRuleTMetaSpacing(RuleMatchTransformMeta): + """JinjaRule transform metadata. + + :param key: Key or index within the task + :param value: Value of the key + :param path: Path to the key + :param fixed: Value with spacing fixed + """ + + key: str | int + value: str | int + path: tuple[str | int, ...] + fixed: str + + def __str__(self) -> str: + """Return string representation.""" + return f"{self.key}={self.value} at {self.path} fixed to {self.fixed}" + + +class JinjaRule(AnsibleLintRule, TransformMixin): """Rule that looks inside jinja2 templates.""" id = "jinja" @@ -94,11 +128,13 @@ class JinjaRule(AnsibleLintRule): if isinstance(v, str): try: template( - basedir=file.path.parent if file else Path("."), + basedir=file.path.parent if file else Path(), value=v, variables=deannotate(task.get("vars", {})), fail_on_error=True, # we later decide which ones to ignore or not ) + except AnsibleFilterError: + bypass = True # ValueError RepresenterError except AnsibleError as exc: bypass = False @@ -111,7 +147,7 @@ class JinjaRule(AnsibleLintRule): ) if ignored_re.search(orig_exc_message) or isinstance( orig_exc, - AnsibleParserError, + AnsibleParserError | TypeError, ): # An unhandled exception occurred while running the lookup plugin 'template'. Error was a <class 'ansible.errors.AnsibleError'>, original message: the template file ... could not be found for the lookup. the template file ... could not be found for the lookup @@ -119,7 +155,7 @@ class JinjaRule(AnsibleLintRule): # AnsibleError(TemplateSyntaxError): template error while templating string: Could not load "ipwrap": 'Invalid plugin FQCN (ansible.netcommon.ipwrap): unable to locate collection ansible.netcommon'. String: Foo {{ buildset_registry.host | ipwrap }}. Could not load "ipwrap": 'Invalid plugin FQCN (ansible.netcommon.ipwrap): unable to locate collection ansible.netcommon' bypass = True elif ( - isinstance(orig_exc, (AnsibleError, TemplateSyntaxError)) + isinstance(orig_exc, AnsibleError | TemplateSyntaxError) and match ): error = match.group("error") @@ -166,6 +202,12 @@ class JinjaRule(AnsibleLintRule): details=details, filename=file, tag=f"{self.id}[{tag}]", + transform_meta=JinjaRuleTMetaSpacing( + key=key, + value=v, + path=tuple(path), + fixed=reformatted, + ), ), ) except Exception as exc: @@ -181,7 +223,6 @@ class JinjaRule(AnsibleLintRule): if str(file.kind) == "vars": data = parse_yaml_from_file(str(file.path)) - # pylint: disable=unused-variable for key, v, _path in nested_items_path(data): if isinstance(v, AnsibleUnicode): reformatted, details, tag = self.check_whitespace( @@ -249,7 +290,7 @@ class JinjaRule(AnsibleLintRule): last_value = value return result - # pylint: disable=too-many-statements,too-many-locals + # pylint: disable=too-many-locals def check_whitespace( self, text: str, @@ -327,7 +368,7 @@ class JinjaRule(AnsibleLintRule): # process expression # pylint: disable=unsupported-membership-test if isinstance(expr_str, str) and "\n" in expr_str: - raise NotImplementedError + raise NotImplementedError # noqa: TRY301 leading_spaces = " " * (len(expr_str) - len(expr_str.lstrip())) expr_str = leading_spaces + blacken(expr_str.lstrip()) if tokens[ @@ -348,7 +389,6 @@ class JinjaRule(AnsibleLintRule): except jinja2.exceptions.TemplateSyntaxError as exc: return "", str(exc.message), "invalid" - # https://github.com/PyCQA/pylint/issues/7433 - py311 only # pylint: disable=c-extension-no-member except (NotImplementedError, black.parsing.InvalidInput) as exc: # black is not able to recognize all valid jinja2 templates, so we @@ -370,6 +410,68 @@ class JinjaRule(AnsibleLintRule): ) return reformatted, details, "spacing" + def transform( + self: JinjaRule, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + """Transform jinja2 errors. + + :param match: MatchError instance + :param lintable: Lintable instance + :param data: data to transform + """ + if match.tag == "jinja[spacing]": + self._transform_spacing(match, data) + + def _transform_spacing( + self: JinjaRule, + match: MatchError, + data: CommentedMap | CommentedSeq | str, + ) -> None: + """Transform jinja2 spacing errors. + + The match error was found on a normalized task so we cannot compare the path + instead we only compare the key and value, if the task has 2 identical keys with the + exact same jinja spacing issue, we may transform them out of order + + :param match: MatchError instance + :param data: data to transform + """ + if not isinstance(match.transform_meta, JinjaRuleTMetaSpacing): + return + if isinstance(data, str): + return + + obj = self.seek(match.yaml_path, data) + if obj is None: + return + + ignored_keys = ("block", "ansible.builtin.block", "ansible.legacy.block") + for key, value, path in nested_items_path( + data_collection=obj, + ignored_keys=ignored_keys, + ): + if key == match.transform_meta.key and value == match.transform_meta.value: + if not path: + continue + for pth in path[:-1]: + try: + obj = obj[pth] + except (KeyError, TypeError) as exc: + err = f"Unable to transform {match.transform_meta}: {exc}" + _logger.error(err) # noqa: TRY400 + return + try: + obj[path[-1]][key] = match.transform_meta.fixed + match.fixed = True + + except (KeyError, TypeError) as exc: + err = f"Unable to transform {match.transform_meta}: {exc}" + _logger.error(err) # noqa: TRY400 + return + def blacken(text: str) -> str: """Format Jinja2 template using black.""" @@ -380,10 +482,14 @@ def blacken(text: str) -> str: if "pytest" in sys.modules: + from unittest import mock + import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner + from ansiblelint.transformer import Transformer @pytest.fixture(name="error_expected_lines") def fixture_error_expected_lines() -> list[int]: @@ -725,6 +831,38 @@ if "pytest" in sys.modules: errs = Runner(success, rules=collection).run() assert len(errs) == 0 + @mock.patch.dict(os.environ, {"ANSIBLE_LINT_WRITE_TMP": "1"}, clear=True) + def test_jinja_transform( + config_options: Options, + default_rules_collection: RulesCollection, + ) -> None: + """Test transform functionality for jinja rule.""" + playbook = Path("examples/playbooks/rule-jinja-before.yml") + config_options.write_list = ["all"] + + config_options.lintables = [str(playbook)] + runner_result = get_matches( + rules=default_rules_collection, + options=config_options, + ) + transformer = Transformer(result=runner_result, options=config_options) + transformer.run() + + matches = runner_result.matches + assert len(matches) == 2 + + orig_content = playbook.read_text(encoding="utf-8") + expected_content = playbook.with_suffix( + f".transformed{playbook.suffix}", + ).read_text(encoding="utf-8") + transformed_content = playbook.with_suffix(f".tmp{playbook.suffix}").read_text( + encoding="utf-8", + ) + + assert orig_content != transformed_content + assert expected_content == transformed_content + playbook.with_suffix(f".tmp{playbook.suffix}").unlink() + def _get_error_line(task: dict[str, Any], path: list[str | int]) -> int: """Return error line number.""" @@ -736,5 +874,5 @@ def _get_error_line(task: dict[str, Any], path: list[str | int]) -> int: line = ctx[LINE_NUMBER_KEY] if not isinstance(line, int): msg = "Line number is not an integer" - raise RuntimeError(msg) + raise TypeError(msg) return line diff --git a/src/ansiblelint/rules/key_order.md b/src/ansiblelint/rules/key_order.md index 378d8a5..bcef36a 100644 --- a/src/ansiblelint/rules/key_order.md +++ b/src/ansiblelint/rules/key_order.md @@ -61,3 +61,7 @@ we concluded that the block keys must be the last ones. Another common practice was to put `tags` as the last property. Still, for the same reasons, we decided that they should not be put after block keys either. + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/key_order.py b/src/ansiblelint/rules/key_order.py index 897da64..0c0a2f1 100644 --- a/src/ansiblelint/rules/key_order.py +++ b/src/ansiblelint/rules/key_order.py @@ -1,14 +1,19 @@ """All tasks should be have name come first.""" + from __future__ import annotations import functools import sys -from typing import TYPE_CHECKING +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.constants import ANNOTATION_KEYS, LINE_NUMBER_KEY +from ansiblelint.errors import MatchError, RuleMatchTransformMeta +from ansiblelint.rules import AnsibleLintRule, TransformMixin if TYPE_CHECKING: - from ansiblelint.errors import MatchError + from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task @@ -46,7 +51,21 @@ def task_property_sorter(property1: str, property2: str) -> int: return (v_1 > v_2) - (v_1 < v_2) -class KeyOrderRule(AnsibleLintRule): +@dataclass(frozen=True) +class KeyOrderTMeta(RuleMatchTransformMeta): + """Key Order transform metadata. + + :param fixed: tuple with updated key order + """ + + fixed: tuple[str | int, ...] + + def __str__(self) -> str: + """Return string representation.""" + return f"Fixed to {self.fixed}" + + +class KeyOrderRule(AnsibleLintRule, TransformMixin): """Ensure specific order of keys in mappings.""" id = "key-order" @@ -59,6 +78,25 @@ class KeyOrderRule(AnsibleLintRule): "key-order[task]": "You can improve the task key order", } + def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: + """Return matches found for a specific play (entry in playbook).""" + result: list[MatchError] = [] + if file.kind != "playbook": + return result + keys = [str(key) for key, val in data.items() if key not in ANNOTATION_KEYS] + sorted_keys = sorted(keys, key=functools.cmp_to_key(task_property_sorter)) + if keys != sorted_keys: + result.append( + self.create_matcherror( + f"You can improve the play key order to: {', '.join(sorted_keys)}", + filename=file, + tag=f"{self.id}[play]", + lineno=data[LINE_NUMBER_KEY], + transform_meta=KeyOrderTMeta(fixed=tuple(sorted_keys)), + ), + ) + return result + def matchtask( self, task: Task, @@ -66,7 +104,7 @@ class KeyOrderRule(AnsibleLintRule): ) -> list[MatchError]: result = [] raw_task = task["__raw_task__"] - keys = [key for key in raw_task if not key.startswith("_")] + keys = [str(key) for key in raw_task if not key.startswith("_")] sorted_keys = sorted(keys, key=functools.cmp_to_key(task_property_sorter)) if keys != sorted_keys: result.append( @@ -74,17 +112,43 @@ class KeyOrderRule(AnsibleLintRule): f"You can improve the task key order to: {', '.join(sorted_keys)}", filename=file, tag="key-order[task]", + transform_meta=KeyOrderTMeta(fixed=tuple(sorted_keys)), ), ) return result + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if not isinstance(match.transform_meta, KeyOrderTMeta): + return + + if match.tag == f"{self.id}[play]": + play = self.seek(match.yaml_path, data) + for key in match.transform_meta.fixed: + # other transformation might change the key + if key in play: + play[key] = play.pop(key) + match.fixed = True + if match.tag == f"{self.id}[task]": + task = self.seek(match.yaml_path, data) + for key in match.transform_meta.fixed: + # other transformation might change the key + if key in task: + task[key] = task.pop(key) + match.fixed = True + # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/latest.py b/src/ansiblelint/rules/latest.py index 0838feb..ef57b94 100644 --- a/src/ansiblelint/rules/latest.py +++ b/src/ansiblelint/rules/latest.py @@ -1,4 +1,5 @@ """Implementation of latest rule.""" + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/src/ansiblelint/rules/literal_compare.py b/src/ansiblelint/rules/literal_compare.py index 1129d1d..151398a 100644 --- a/src/ansiblelint/rules/literal_compare.py +++ b/src/ansiblelint/rules/literal_compare.py @@ -1,4 +1,5 @@ """Implementation of the literal-compare rule.""" + # Copyright (c) 2016, Will Thames and contributors # Copyright (c) 2018-2021, Ansible Project @@ -55,8 +56,9 @@ class ComparisonToLiteralBoolRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/loop_var_prefix.md b/src/ansiblelint/rules/loop_var_prefix.md index 33adbd7..5d1b9b0 100644 --- a/src/ansiblelint/rules/loop_var_prefix.md +++ b/src/ansiblelint/rules/loop_var_prefix.md @@ -1,15 +1,15 @@ # loop-var-prefix -This rule avoids conflicts with nested looping tasks by configuring a variable -prefix with `loop_var`. Ansible sets `item` as the loop variable. You can use -`loop_var` to specify a prefix for loop variables and ensure they are unique to -each task. +This rule avoids conflicts with nested looping tasks by enforcing an individual +variable name in loops. Ansible defaults to `item` as the loop variable. You can +use `loop_var` to rename it. Optionally require a prefix on the variable name. +The prefix can be configured via the `<loop_var_prefix>` setting. This rule can produce the following messages: - `loop-var-prefix[missing]` - Replace any unsafe implicit `item` loop variable - by adding `loop_var: <loop_var_prefix>...`. -- `loop-var-prefix[wrong]` - Ensure loop variables start with + by adding `loop_var: <variable_name>...`. +- `loop-var-prefix[wrong]` - Ensure the loop variable starts with `<loop_var_prefix>`. This rule originates from the [Naming parameters section of Ansible Best @@ -41,20 +41,20 @@ enable_list: - name: Example playbook hosts: localhost tasks: - - name: Does not set a prefix for loop variables. + - name: Does not set a variable name for loop variables. ansible.builtin.debug: - var: item + var: item # <- When in a nested loop, "item" is ambiguous loop: - foo - - bar # <- These items do not have a unique prefix. - - name: Sets a prefix that is not unique. + - bar + - name: Sets a variable name that doesn't start with <loop_var_prefix>. ansible.builtin.debug: var: zz_item loop: - foo - bar loop_control: - loop_var: zz_item # <- This prefix is not unique. + loop_var: zz_item # <- zz is not the role name so the prefix is wrong ``` ## Correct Code @@ -64,14 +64,14 @@ enable_list: - name: Example playbook hosts: localhost tasks: - - name: Sets a unique prefix for loop variables. + - name: Sets a unique variable_name with role as prefix for loop variables. ansible.builtin.debug: - var: zz_item + var: myrole_item loop: - foo - bar loop_control: - loop_var: my_prefix # <- Specifies a unique prefix for loop variables. + loop_var: myrole_item # <- Unique variable name with role as prefix ``` [cop314]: diff --git a/src/ansiblelint/rules/loop_var_prefix.py b/src/ansiblelint/rules/loop_var_prefix.py index 8f1bb56..9f7a2ca 100644 --- a/src/ansiblelint/rules/loop_var_prefix.py +++ b/src/ansiblelint/rules/loop_var_prefix.py @@ -1,4 +1,5 @@ """Optional Ansible-lint rule to enforce use of prefix on role loop vars.""" + from __future__ import annotations import re @@ -81,8 +82,9 @@ Looping inside roles has the risk of clashing with loops from user-playbooks.\ if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/meta_incorrect.py b/src/ansiblelint/rules/meta_incorrect.py index 4252254..ed8d8d9 100644 --- a/src/ansiblelint/rules/meta_incorrect.py +++ b/src/ansiblelint/rules/meta_incorrect.py @@ -1,4 +1,5 @@ """Implementation of meta-incorrect rule.""" + # Copyright (c) 2018, Ansible Project from __future__ import annotations @@ -56,8 +57,8 @@ class MetaChangeFromDefaultRule(AnsibleLintRule): if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner def test_default_galaxy_info( default_rules_collection: RulesCollection, diff --git a/src/ansiblelint/rules/meta_no_tags.py b/src/ansiblelint/rules/meta_no_tags.py index c27a30e..3e9b636 100644 --- a/src/ansiblelint/rules/meta_no_tags.py +++ b/src/ansiblelint/rules/meta_no_tags.py @@ -1,4 +1,5 @@ """Implementation of meta-no-tags rule.""" + from __future__ import annotations import re diff --git a/src/ansiblelint/rules/meta_runtime.md b/src/ansiblelint/rules/meta_runtime.md index 6ed6f17..3e05c59 100644 --- a/src/ansiblelint/rules/meta_runtime.md +++ b/src/ansiblelint/rules/meta_runtime.md @@ -1,26 +1,21 @@ # meta-runtime -This rule checks the meta/runtime.yml `requires_ansible` key against the list of currently supported versions of ansible-core. - -This rule can produce messages such: - -- `requires_ansible` key must be set to a supported version. - -Currently supported versions of ansible-core are: - -- `2.9.10` -- `2.11.x` -- `2.12.x` -- `2.13.x` -- `2.14.x` -- `2.15.x` -- `2.16.x` (in development) +This rule checks the meta/runtime.yml `requires_ansible` key against the list of +currently supported versions of ansible-core. This rule can produce messages such as: -- `meta-runtime[unsupported-version]` - `requires_ansible` key must contain a supported version, shown in the list above. -- `meta-runtime[invalid-version]` - `requires_ansible` key must be a valid version identifier. +- `meta-runtime[unsupported-version]` - `requires_ansible` key must refer to a + currently supported version such as: >=2.14.0, >=2.15.0, >=2.16.0 +- `meta-runtime[invalid-version]` - `requires_ansible` is not a valid + requirement specification +Please note that the linter will allow only a full version of Ansible such +`2.16.0` and not allow their short form, like `2.16`. This is a safety measure +for asking authors to mention an explicit version that they tested with. Over +the years we spotted multiple problems caused by the use of the short versions, +users ended up trying an outdated version that was never tested against by the +collection maintainer. ## Problematic code @@ -30,11 +25,10 @@ This rule can produce messages such as: requires_ansible: ">=2.9" ``` - ```yaml # runtime.yml --- -requires_ansible: "2.9" +requires_ansible: "2.15" ``` ## Correct code @@ -42,5 +36,17 @@ requires_ansible: "2.9" ```yaml # runtime.yml --- -requires_ansible: ">=2.9.10" +requires_ansible: ">=2.15.0" +``` + +## Configuration + +In addition to the internal list of supported Ansible versions, users can +configure additional values. This allows those that want to maintain content +that requires a version of ansible-core that is already out of support. + +```yaml +# Also recognize these versions of Ansible as supported: +supported_ansible_also: + - "2.14" ``` diff --git a/src/ansiblelint/rules/meta_runtime.py b/src/ansiblelint/rules/meta_runtime.py index fed7121..3df2826 100644 --- a/src/ansiblelint/rules/meta_runtime.py +++ b/src/ansiblelint/rules/meta_runtime.py @@ -1,4 +1,5 @@ """Implementation of meta-runtime rule.""" + from __future__ import annotations import sys @@ -22,17 +23,15 @@ class CheckRequiresAnsibleVersion(AnsibleLintRule): id = "meta-runtime" description = ( "The ``requires_ansible`` key in runtime.yml must specify " - "a supported platform version of ansible-core and be a valid version." + "a supported platform version of ansible-core and be a valid version value " + "in x.y.z format." ) severity = "VERY_HIGH" tags = ["metadata"] version_added = "v6.11.0 (last update)" - # Refer to https://access.redhat.com/support/policy/updates/ansible-automation-platform - # Also add devel to this list - supported_ansible = ["2.9.10", "2.11.", "2.12.", "2.13.", "2.14.", "2.15.", "2.16."] _ids = { - "meta-runtime[unsupported-version]": "requires_ansible key must be set to a supported version.", + "meta-runtime[unsupported-version]": "'requires_ansible' key must refer to a currently supported version", "meta-runtime[invalid-version]": "'requires_ansible' is not a valid requirement specification", } @@ -47,22 +46,26 @@ class CheckRequiresAnsibleVersion(AnsibleLintRule): if file.kind != "meta-runtime": return [] - version_required = file.data.get("requires_ansible", None) + requires_ansible = file.data.get("requires_ansible", None) - if version_required: - if not any( - version in version_required for version in self.supported_ansible + if requires_ansible: + if self.options and not any( + version in requires_ansible + for version in self.options.supported_ansible ): + supported_ansible = [f">={x}0" for x in self.options.supported_ansible] + msg = f"'requires_ansible' key must refer to a currently supported version such as: {', '.join(supported_ansible)}" + results.append( self.create_matcherror( - message="requires_ansible key must be set to a supported version.", + message=msg, tag="meta-runtime[unsupported-version]", filename=file, ), ) try: - SpecifierSet(version_required) + SpecifierSet(requires_ansible) except ValueError: results.append( self.create_matcherror( @@ -79,17 +82,18 @@ class CheckRequiresAnsibleVersion(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures", "tags"), ( pytest.param( - "examples/meta_runtime_version_checks/pass/meta/runtime.yml", + "examples/meta_runtime_version_checks/pass_0/meta/runtime.yml", 0, "meta-runtime[unsupported-version]", - id="pass", + id="pass0", ), pytest.param( "examples/meta_runtime_version_checks/fail_0/meta/runtime.yml", @@ -111,16 +115,37 @@ if "pytest" in sys.modules: ), ), ) - def test_meta_supported_version( + def test_default_meta_supported_version( default_rules_collection: RulesCollection, test_file: str, failures: int, tags: str, ) -> None: - """Test rule matches.""" + """Test for default supported ansible versions.""" default_rules_collection.register(CheckRequiresAnsibleVersion()) results = Runner(test_file, rules=default_rules_collection).run() for result in results: assert result.rule.id == CheckRequiresAnsibleVersion().id assert result.tag == tags assert len(results) == failures + + @pytest.mark.parametrize( + ("test_file", "failures"), + ( + pytest.param( + "examples/meta_runtime_version_checks/pass_1/meta/runtime.yml", + 0, + id="pass1", + ), + ), + ) + def test_added_meta_supported_version( + default_rules_collection: RulesCollection, + test_file: str, + failures: int, + ) -> None: + """Test for added supported ansible versions in the config.""" + default_rules_collection.register(CheckRequiresAnsibleVersion()) + default_rules_collection.options.supported_ansible_also = ["2.9"] + results = Runner(test_file, rules=default_rules_collection).run() + assert len(results) == failures diff --git a/src/ansiblelint/rules/meta_video_links.py b/src/ansiblelint/rules/meta_video_links.py index 5d4941a..fa19cc6 100644 --- a/src/ansiblelint/rules/meta_video_links.py +++ b/src/ansiblelint/rules/meta_video_links.py @@ -1,4 +1,5 @@ """Implementation of meta-video-links rule.""" + # Copyright (c) 2018, Ansible Project from __future__ import annotations @@ -86,8 +87,9 @@ class MetaVideoLinksRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/name.md b/src/ansiblelint/rules/name.md index 9df4213..0c5080a 100644 --- a/src/ansiblelint/rules/name.md +++ b/src/ansiblelint/rules/name.md @@ -21,12 +21,21 @@ If you want to ignore some of the messages above, you can add any of them to the ## name[prefix] -This rule applies only to included task files that are not named `main.yml`. It -suggests adding the stem of the file as a prefix to the task name. +This rule applies only to included task files that are not named `main.yml` or +are embedded within subdirectories. It suggests adding the stems of the file +path as a prefix to the task name. For example, if you have a task named `Restart server` inside a file named `tasks/deploy.yml`, this rule suggests renaming it to `deploy | Restart server`, -so it would be easier to identify where it comes from. +so it would be easier to identify where it comes from. If the file was named +`tasks/main.yml`, then the rule would have no effect. + +For task files that are embedded within subdirectories, these subdirectories +will also be appended as part of the prefix. For example, if you have a task +named `Terminate server` inside a file named `tasks/foo/destroy.yml`, this rule +suggests renaming it to `foo | destroy | Terminate server`. If the file was +named `tasks/foo/main.yml` then the rule would recommend renaming the task to +`foo | main | Terminate server`. For the moment, this sub-rule is just an **opt-in**, so you need to add it to your `enable_list` to activate it. @@ -59,3 +68,7 @@ your `enable_list` to activate it. - name: Create placeholder file ansible.builtin.command: touch /tmp/.placeholder ``` + +!!! note + + `name[casing]` can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/name.py b/src/ansiblelint/rules/name.py index 41ce5cb..b814a41 100644 --- a/src/ansiblelint/rules/name.py +++ b/src/ansiblelint/rules/name.py @@ -1,19 +1,23 @@ """Implementation of NameRule.""" + from __future__ import annotations import re import sys -from copy import deepcopy from typing import TYPE_CHECKING, Any +import wcmatch.pathlib +import wcmatch.wcmatch + from ansiblelint.constants import LINE_NUMBER_KEY +from ansiblelint.file_utils import Lintable from ansiblelint.rules import AnsibleLintRule, TransformMixin if TYPE_CHECKING: from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.config import Options from ansiblelint.errors import MatchError - from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task @@ -39,9 +43,11 @@ class NameRule(AnsibleLintRule, TransformMixin): def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: """Return matches found for a specific play (entry in playbook).""" - results = [] + results: list[MatchError] = [] if file.kind != "playbook": return [] + if file.failed(): + return results if "name" not in data: return [ self.create_matcherror( @@ -65,7 +71,9 @@ class NameRule(AnsibleLintRule, TransformMixin): task: Task, file: Lintable | None = None, ) -> list[MatchError]: - results = [] + results: list[MatchError] = [] + if file and file.failed(): + return results name = task.get("name") if not name: results.append( @@ -84,6 +92,7 @@ class NameRule(AnsibleLintRule, TransformMixin): lineno=task[LINE_NUMBER_KEY], ), ) + return results def _prefix_check( @@ -120,10 +129,15 @@ class NameRule(AnsibleLintRule, TransformMixin): # stage one check prefix effective_name = name if self._collection and lintable: - prefix = self._collection.options.task_name_prefix.format( - stem=lintable.path.stem, - ) - if lintable.kind == "tasks" and lintable.path.stem != "main": + full_stem = self._find_full_stem(lintable) + stems = [ + self._collection.options.task_name_prefix.format(stem=stem) + for stem in wcmatch.pathlib.PurePath( + full_stem, + ).parts + ] + prefix = "".join(stems) + if lintable.kind == "tasks" and full_stem != "main": if not name.startswith(prefix): # For the moment in order to raise errors this rule needs to be # enabled manually. Still, we do allow use of prefixes even without @@ -165,6 +179,38 @@ class NameRule(AnsibleLintRule, TransformMixin): ) return results + def _find_full_stem(self, lintable: Lintable) -> str: + lintable_dir = wcmatch.pathlib.PurePath(lintable.dir) + stem = lintable.path.stem + kind = str(lintable.kind) + + stems = [lintable_dir.name] + lintable_dir = lintable_dir.parent + pathex = lintable_dir / stem + glob = "" + + if self.options: + for entry in self.options.kinds: + for key, value in entry.items(): + if kind == key: + glob = value + + while pathex.globmatch( + glob, + flags=( + wcmatch.pathlib.GLOBSTAR + | wcmatch.pathlib.BRACE + | wcmatch.pathlib.DOTGLOB + ), + ): + stems.insert(0, lintable_dir.name) + lintable_dir = lintable_dir.parent + pathex = lintable_dir / stem + + if stems[0].startswith(kind): + del stems[0] + return str(wcmatch.pathlib.PurePath(*stems, stem)) + def transform( self, match: MatchError, @@ -172,17 +218,44 @@ class NameRule(AnsibleLintRule, TransformMixin): data: CommentedMap | CommentedSeq | str, ) -> None: if match.tag == "name[casing]": + + def update_task_name(task_name: str) -> str: + """Capitalize the first work of the task name.""" + # Not using capitalize(), since that rewrites the rest of the name to lower case + if "|" in task_name: # if using prefix + [file_name, update_task_name] = task_name.split("|") + return f"{file_name.strip()} | {update_task_name.strip()[:1].upper()}{update_task_name.strip()[1:]}" + + return f"{task_name[:1].upper()}{task_name[1:]}" + target_task = self.seek(match.yaml_path, data) - # Not using capitalize(), since that rewrites the rest of the name to lower case - target_task[ - "name" - ] = f"{target_task['name'][:1].upper()}{target_task['name'][1:]}" - match.fixed = True + orig_task_name = target_task.get("name", None) + # pylint: disable=too-many-nested-blocks + if orig_task_name: + updated_task_name = update_task_name(orig_task_name) + for item in data: + if isinstance(item, dict) and "tasks" in item: + for task in item["tasks"]: + # We want to rewrite task names in the notify keyword, but + # if there isn't a notify section, there's nothing to do. + if "notify" not in task: + continue + + if ( + isinstance(task["notify"], str) + and orig_task_name == task["notify"] + ): + task["notify"] = updated_task_name + elif isinstance(task["notify"], list): + for idx in range(len(task["notify"])): + if orig_task_name == task["notify"][idx]: + task["notify"][idx] = updated_task_name + + target_task["name"] = updated_task_name + match.fixed = True if "pytest" in sys.modules: - from ansiblelint.config import options - from ansiblelint.file_utils import Lintable # noqa: F811 from ansiblelint.rules import RulesCollection from ansiblelint.runner import Runner @@ -203,11 +276,23 @@ if "pytest" in sys.modules: errs = bad_runner.run() assert len(errs) == 5 - def test_name_prefix_negative() -> None: + def test_name_prefix_positive(config_options: Options) -> None: + """Positive test for name[prefix].""" + config_options.enable_list = ["name[prefix]"] + collection = RulesCollection(options=config_options) + collection.register(NameRule()) + success = Lintable( + "examples/playbooks/tasks/main.yml", + kind="tasks", + ) + good_runner = Runner(success, rules=collection) + results = good_runner.run() + assert len(results) == 0 + + def test_name_prefix_negative(config_options: Options) -> None: """Negative test for name[missing].""" - custom_options = deepcopy(options) - custom_options.enable_list = ["name[prefix]"] - collection = RulesCollection(options=custom_options) + config_options.enable_list = ["name[prefix]"] + collection = RulesCollection(options=config_options) collection.register(NameRule()) failure = Lintable( "examples/playbooks/tasks/rule-name-prefix-fail.yml", @@ -221,6 +306,36 @@ if "pytest" in sys.modules: assert results[1].tag == "name[prefix]" assert results[2].tag == "name[prefix]" + def test_name_prefix_negative_2(config_options: Options) -> None: + """Negative test for name[prefix].""" + config_options.enable_list = ["name[prefix]"] + collection = RulesCollection(options=config_options) + collection.register(NameRule()) + failure = Lintable( + "examples/playbooks/tasks/partial_prefix/foo.yml", + kind="tasks", + ) + bad_runner = Runner(failure, rules=collection) + results = bad_runner.run() + assert len(results) == 2 + assert results[0].tag == "name[prefix]" + assert results[1].tag == "name[prefix]" + + def test_name_prefix_negative_3(config_options: Options) -> None: + """Negative test for name[prefix].""" + config_options.enable_list = ["name[prefix]"] + collection = RulesCollection(options=config_options) + collection.register(NameRule()) + failure = Lintable( + "examples/playbooks/tasks/partial_prefix/main.yml", + kind="tasks", + ) + bad_runner = Runner(failure, rules=collection) + results = bad_runner.run() + assert len(results) == 2 + assert results[0].tag == "name[prefix]" + assert results[1].tag == "name[prefix]" + def test_rule_name_lowercase() -> None: """Negative test for a task that starts with lowercase.""" collection = RulesCollection() @@ -255,6 +370,5 @@ if "pytest" in sys.modules: def test_when_no_lintable() -> None: """Test when lintable is None.""" name_rule = NameRule() - # pylint: disable=protected-access result = name_rule._prefix_check("Foo", None, 1) # noqa: SLF001 assert len(result) == 0 diff --git a/src/ansiblelint/rules/no_changed_when.py b/src/ansiblelint/rules/no_changed_when.py index 28ba427..e71934d 100644 --- a/src/ansiblelint/rules/no_changed_when.py +++ b/src/ansiblelint/rules/no_changed_when.py @@ -1,4 +1,5 @@ """Implementation of the no-changed-when rule.""" + # Copyright (c) 2016 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -75,8 +76,9 @@ class CommandHasChangesCheckRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/no_free_form.md b/src/ansiblelint/rules/no_free_form.md index 0ffc0ac..ae05d0f 100644 --- a/src/ansiblelint/rules/no_free_form.md +++ b/src/ansiblelint/rules/no_free_form.md @@ -56,3 +56,7 @@ This rule can produce messages as: executable: /bin/bash # <-- explicit is better changed_when: false ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/no_free_form.py b/src/ansiblelint/rules/no_free_form.py index e89333b..13489ef 100644 --- a/src/ansiblelint/rules/no_free_form.py +++ b/src/ansiblelint/rules/no_free_form.py @@ -1,20 +1,25 @@ """Implementation of NoFreeFormRule.""" + from __future__ import annotations +import functools import re import sys -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from ansiblelint.constants import INCLUSION_ACTION_NAMES, LINE_NUMBER_KEY -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin +from ansiblelint.rules.key_order import task_property_sorter if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task -class NoFreeFormRule(AnsibleLintRule): +class NoFreeFormRule(AnsibleLintRule, TransformMixin): """Rule for detecting discouraged free-form syntax for action modules.""" id = "no-free-form" @@ -75,7 +80,7 @@ class NoFreeFormRule(AnsibleLintRule): "win_command", "win_shell", ): - if self.cmd_shell_re.match(action_value): + if self.cmd_shell_re.search(action_value): fail = True else: fail = True @@ -89,12 +94,97 @@ class NoFreeFormRule(AnsibleLintRule): ) return results + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if "no-free-form" in match.tag: + task = self.seek(match.yaml_path, data) + + def filter_values( + val: str, + filter_key: str, + filter_dict: dict[str, Any], + ) -> str: + """Pull out key=value pairs from a string and set them in filter_dict. + + Returns unmatched strings. + """ + if filter_key not in val: + return val + + extra = "" + [k, v] = val.split(filter_key, 1) + if " " in k: + extra, k = k.rsplit(" ", 1) + + if v[0] in "\"'": + # Keep quoted strings together + quote = v[0] + _, v, remainder = v.split(quote, 2) + v = f"{quote}{v}{quote}" + else: + try: + v, remainder = v.split(" ", 1) + except ValueError: + remainder = "" + + filter_dict[k] = v + + extra = " ".join( + (extra, filter_values(remainder, filter_key, filter_dict)), + ) + return extra.strip() + + if match.tag == "no-free-form": + module_opts: dict[str, Any] = {} + for _ in range(len(task)): + k, v = task.popitem(False) + # identify module as key and process its value + if len(k.split(".")) == 3 and isinstance(v, str): + cmd = filter_values(v, "=", module_opts) + if cmd: + module_opts["cmd"] = cmd + + sorted_module_opts = {} + for key in sorted( + module_opts.keys(), + key=functools.cmp_to_key(task_property_sorter), + ): + sorted_module_opts[key] = module_opts[key] + + task[k] = sorted_module_opts + else: + task[k] = v + + match.fixed = True + elif match.tag == "no-free-form[raw]": + exec_key_val: dict[str, Any] = {} + for _ in range(len(task)): + k, v = task.popitem(False) + if isinstance(v, str) and "executable" in v: + # Filter the executable and other parts from the string + task[k] = " ".join( + [ + item + for item in v.split(" ") + if filter_values(item, "=", exec_key_val) + ], + ) + task["args"] = exec_key_val + else: + task[k] = v + match.fixed = True + if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/no_handler.py b/src/ansiblelint/rules/no_handler.py index 380fd61..ae8f820 100644 --- a/src/ansiblelint/rules/no_handler.py +++ b/src/ansiblelint/rules/no_handler.py @@ -69,25 +69,27 @@ class UseHandlerRatherThanWhenChangedRule(AnsibleLintRule): task: Task, file: Lintable | None = None, ) -> bool | str: - if task["__ansible_action_type__"] != "task": + if task["__ansible_action_type__"] != "task" or task.is_handler(): return False when = task.get("when") + result = False if isinstance(when, list): - if len(when) > 1: - return False - return _changed_in_when(when[0]) - if isinstance(when, str): - return _changed_in_when(when) - return False + if len(when) <= 1: + result = _changed_in_when(when[0]) + elif isinstance(when, str): + result = _changed_in_when(when) + return result if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner + from ansiblelint.testing import run_ansible_lint @pytest.mark.parametrize( ("test_file", "failures"), @@ -106,3 +108,10 @@ if "pytest" in sys.modules: assert len(results) == failures for result in results: assert result.tag == "no-handler" + + def test_role_with_handler() -> None: + """Test role with handler.""" + role_path = "examples/roles/role_with_handler" + + results = run_ansible_lint("-v", role_path) + assert "no-handler" not in results.stdout diff --git a/src/ansiblelint/rules/no_jinja_when.md b/src/ansiblelint/rules/no_jinja_when.md index 702e807..5a2c736 100644 --- a/src/ansiblelint/rules/no_jinja_when.md +++ b/src/ansiblelint/rules/no_jinja_when.md @@ -30,3 +30,7 @@ anti-pattern and does not produce expected results. ansible.builtin.command: /sbin/shutdown -t now when: ansible_facts['os_family'] == "Debian" # <- Uses facts in a conditional statement. ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/no_jinja_when.py b/src/ansiblelint/rules/no_jinja_when.py index 807081d..a5fc030 100644 --- a/src/ansiblelint/rules/no_jinja_when.py +++ b/src/ansiblelint/rules/no_jinja_when.py @@ -1,19 +1,23 @@ """Implementation of no-jinja-when rule.""" + from __future__ import annotations +import re import sys from typing import TYPE_CHECKING, Any from ansiblelint.constants import LINE_NUMBER_KEY -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task -class NoFormattingInWhenRule(AnsibleLintRule): +class NoFormattingInWhenRule(AnsibleLintRule, TransformMixin): """No Jinja2 in when.""" id = "no-jinja-when" @@ -44,19 +48,19 @@ class NoFormattingInWhenRule(AnsibleLintRule): if isinstance(data, dict): if "roles" not in data or data["roles"] is None: return errors - for role in data["roles"]: + errors = [ + self.create_matcherror( + details=str({"when": role}), + filename=file, + lineno=role[LINE_NUMBER_KEY], + ) + for role in data["roles"] if ( isinstance(role, dict) and "when" in role and not self._is_valid(role["when"]) - ): - errors.append( - self.create_matcherror( - details=str({"when": role}), - filename=file, - lineno=role[LINE_NUMBER_KEY], - ), - ) + ) + ] return errors def matchtask( @@ -66,6 +70,37 @@ class NoFormattingInWhenRule(AnsibleLintRule): ) -> bool | str: return "when" in task.raw_task and not self._is_valid(task.raw_task["when"]) + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if match.tag == self.id: + task = self.seek(match.yaml_path, data) + key_to_check = ("when", "changed_when", "failed_when") + for _ in range(len(task)): + k, v = task.popitem(False) + if k == "roles" and isinstance(v, list): + transform_for_roles(v, key_to_check=key_to_check) + elif k in key_to_check: + v = re.sub(r"{{ (.*?) }}", r"\1", v) + task[k] = v + match.fixed = True + + +def transform_for_roles(v: list[Any], key_to_check: tuple[str, ...]) -> None: + """Additional transform logic in case of roles.""" + for idx, new_dict in enumerate(v): + for new_key, new_value in new_dict.items(): + if new_key in key_to_check: + if isinstance(new_value, list): + for index, nested_value in enumerate(new_value): + new_value[index] = re.sub(r"{{ (.*?) }}", r"\1", nested_value) + v[idx][new_key] = new_value + if isinstance(new_value, str): + v[idx][new_key] = re.sub(r"{{ (.*?) }}", r"\1", new_value) + if "pytest" in sys.modules: # Tests for no-jinja-when rule. diff --git a/src/ansiblelint/rules/no_log_password.md b/src/ansiblelint/rules/no_log_password.md index 579dd11..3629ef6 100644 --- a/src/ansiblelint/rules/no_log_password.md +++ b/src/ansiblelint/rules/no_log_password.md @@ -43,3 +43,7 @@ Explicitly adding `no_log: true` prevents accidentally exposing secrets. - wow no_log: true # <- Sets the no_log attribute to a non-false value. ``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/no_log_password.py b/src/ansiblelint/rules/no_log_password.py index 7cc7439..c3f6d34 100644 --- a/src/ansiblelint/rules/no_log_password.py +++ b/src/ansiblelint/rules/no_log_password.py @@ -15,17 +15,25 @@ """NoLogPasswordsRule used with ansible-lint.""" from __future__ import annotations +import os import sys +from pathlib import Path from typing import TYPE_CHECKING -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, RulesCollection, TransformMixin +from ansiblelint.runner import get_matches +from ansiblelint.transformer import Transformer from ansiblelint.utils import Task, convert_to_boolean if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + from ansiblelint.config import Options + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable -class NoLogPasswordsRule(AnsibleLintRule): +class NoLogPasswordsRule(AnsibleLintRule, TransformMixin): """Password should not be logged.""" id = "no-log-password" @@ -72,12 +80,26 @@ class NoLogPasswordsRule(AnsibleLintRule): has_password and not convert_to_boolean(no_log) and len(has_loop) > 0, ) + def transform( + self, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + if match.tag == self.id: + task = self.seek(match.yaml_path, data) + task["no_log"] = True + + match.fixed = True + if "pytest" in sys.modules: + from unittest import mock + import pytest if TYPE_CHECKING: - from ansiblelint.testing import RunFromText # pylint: disable=ungrouped-imports + from ansiblelint.testing import RunFromText NO_LOG_UNUSED = """ - name: Test @@ -304,3 +326,33 @@ if "pytest" in sys.modules: """The task does not actually lock the user.""" results = rule_runner.run_playbook(PASSWORD_LOCK_FALSE) assert len(results) == 0 + + @mock.patch.dict(os.environ, {"ANSIBLE_LINT_WRITE_TMP": "1"}, clear=True) + def test_no_log_password_transform( + config_options: Options, + ) -> None: + """Test transform functionality for no-log-password rule.""" + playbook = Path("examples/playbooks/transform-no-log-password.yml") + config_options.write_list = ["all"] + rules = RulesCollection(options=config_options) + rules.register(NoLogPasswordsRule()) + + config_options.lintables = [str(playbook)] + runner_result = get_matches(rules=rules, options=config_options) + transformer = Transformer(result=runner_result, options=config_options) + transformer.run() + + matches = runner_result.matches + assert len(matches) == 2 + + orig_content = playbook.read_text(encoding="utf-8") + expected_content = playbook.with_suffix( + f".transformed{playbook.suffix}", + ).read_text(encoding="utf-8") + transformed_content = playbook.with_suffix(f".tmp{playbook.suffix}").read_text( + encoding="utf-8", + ) + + assert orig_content != transformed_content + assert expected_content == transformed_content + playbook.with_suffix(f".tmp{playbook.suffix}").unlink() diff --git a/src/ansiblelint/rules/no_prompting.py b/src/ansiblelint/rules/no_prompting.py index 6622771..c5d11d8 100644 --- a/src/ansiblelint/rules/no_prompting.py +++ b/src/ansiblelint/rules/no_prompting.py @@ -1,4 +1,5 @@ """Implementation of no-prompting rule.""" + from __future__ import annotations import sys @@ -8,6 +9,7 @@ from ansiblelint.constants import LINE_NUMBER_KEY from ansiblelint.rules import AnsibleLintRule if TYPE_CHECKING: + from ansiblelint.config import Options from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.utils import Task @@ -32,7 +34,7 @@ class NoPromptingRule(AnsibleLintRule): if file.kind != "playbook": # pragma: no cover return [] - vars_prompt = data.get("vars_prompt", None) + vars_prompt = data.get("vars_prompt") if not vars_prompt: return [] return [ @@ -60,15 +62,14 @@ class NoPromptingRule(AnsibleLintRule): if "pytest" in sys.modules: - from ansiblelint.config import options - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner - def test_no_prompting_fail() -> None: + def test_no_prompting_fail(config_options: Options) -> None: """Negative test for no-prompting.""" # For testing we want to manually enable opt-in rules - options.enable_list = ["no-prompting"] - rules = RulesCollection(options=options) + config_options.enable_list = ["no-prompting"] + rules = RulesCollection(options=config_options) rules.register(NoPromptingRule()) results = Runner("examples/playbooks/rule-no-prompting.yml", rules=rules).run() assert len(results) == 2 diff --git a/src/ansiblelint/rules/no_relative_paths.py b/src/ansiblelint/rules/no_relative_paths.py index 470b1b8..de22641 100644 --- a/src/ansiblelint/rules/no_relative_paths.py +++ b/src/ansiblelint/rules/no_relative_paths.py @@ -1,4 +1,5 @@ """Implementation of no-relative-paths rule.""" + # Copyright (c) 2016, Tsukinowa Inc. <info@tsukinowa.jp> # Copyright (c) 2018, Ansible Project @@ -53,8 +54,9 @@ class RoleRelativePath(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/no_same_owner.py b/src/ansiblelint/rules/no_same_owner.py index 021900e..23290e0 100644 --- a/src/ansiblelint/rules/no_same_owner.py +++ b/src/ansiblelint/rules/no_same_owner.py @@ -1,4 +1,5 @@ """Optional rule for avoiding keeping owner/group when transferring files.""" + from __future__ import annotations import re @@ -84,8 +85,9 @@ should not be preserved when transferring files between them. if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures"), diff --git a/src/ansiblelint/rules/no_tabs.py b/src/ansiblelint/rules/no_tabs.py index c53f1bb..2614a1a 100644 --- a/src/ansiblelint/rules/no_tabs.py +++ b/src/ansiblelint/rules/no_tabs.py @@ -1,4 +1,5 @@ """Implementation of no-tabs rule.""" + # Copyright (c) 2016, Will Thames and contributors # Copyright (c) 2018, Ansible Project from __future__ import annotations @@ -7,6 +8,7 @@ import sys from typing import TYPE_CHECKING from ansiblelint.rules import AnsibleLintRule +from ansiblelint.text import has_jinja from ansiblelint.yaml_utils import nested_items_path if TYPE_CHECKING: @@ -27,6 +29,10 @@ class NoTabsRule(AnsibleLintRule): ("lineinfile", "insertbefore"), ("lineinfile", "regexp"), ("lineinfile", "line"), + ("win_lineinfile", "insertafter"), + ("win_lineinfile", "insertbefore"), + ("win_lineinfile", "regexp"), + ("win_lineinfile", "line"), ("ansible.builtin.lineinfile", "insertafter"), ("ansible.builtin.lineinfile", "insertbefore"), ("ansible.builtin.lineinfile", "regexp"), @@ -35,6 +41,10 @@ class NoTabsRule(AnsibleLintRule): ("ansible.legacy.lineinfile", "insertbefore"), ("ansible.legacy.lineinfile", "regexp"), ("ansible.legacy.lineinfile", "line"), + ("community.windows.win_lineinfile", "insertafter"), + ("community.windows.win_lineinfile", "insertbefore"), + ("community.windows.win_lineinfile", "regexp"), + ("community.windows.win_lineinfile", "line"), ] def matchtask( @@ -44,17 +54,22 @@ class NoTabsRule(AnsibleLintRule): ) -> bool | str: action = task["action"]["__ansible_module__"] for k, v, _ in nested_items_path(task): - if isinstance(k, str) and "\t" in k: + if isinstance(k, str) and "\t" in k and not has_jinja(k): return True - if isinstance(v, str) and "\t" in v and (action, k) not in self.allow_list: + if ( + isinstance(v, str) + and "\t" in v + and (action, k) not in self.allow_list + and not has_jinja(v) + ): return True return False # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner def test_no_tabs_rule(default_rules_collection: RulesCollection) -> None: """Test rule matches.""" @@ -62,6 +77,12 @@ if "pytest" in sys.modules: "examples/playbooks/rule-no-tabs.yml", rules=default_rules_collection, ).run() - assert results[0].lineno == 10 - assert results[0].message == NoTabsRule().shortdesc - assert len(results) == 2 + expected_results = [ + (10, NoTabsRule().shortdesc), + (13, NoTabsRule().shortdesc), + ] + for i, expected in enumerate(expected_results): + assert len(results) >= i + 1 + assert results[i].lineno == expected[0] + assert results[i].message == expected[1] + assert len(results) == len(expected), results diff --git a/src/ansiblelint/rules/only_builtins.py b/src/ansiblelint/rules/only_builtins.py index 78ad93a..3757af8 100644 --- a/src/ansiblelint/rules/only_builtins.py +++ b/src/ansiblelint/rules/only_builtins.py @@ -1,11 +1,11 @@ """Rule definition for usage of builtin actions only.""" + from __future__ import annotations import os import sys from typing import TYPE_CHECKING -from ansiblelint.config import options from ansiblelint.rules import AnsibleLintRule from ansiblelint.rules.fqcn import builtins from ansiblelint.skip_utils import is_nested_task @@ -33,9 +33,11 @@ class OnlyBuiltinsRule(AnsibleLintRule): allowed_collections = [ "ansible.builtin", "ansible.legacy", - *options.only_builtins_allow_collections, ] - allowed_modules = builtins + options.only_builtins_allow_modules + allowed_modules = builtins + if self.options: + allowed_collections += self.options.only_builtins_allow_collections + allowed_modules += self.options.only_builtins_allow_modules is_allowed = ( any(module.startswith(f"{prefix}.") for prefix in allowed_collections) diff --git a/src/ansiblelint/rules/package_latest.md b/src/ansiblelint/rules/package_latest.md index c7e0d82..c965548 100644 --- a/src/ansiblelint/rules/package_latest.md +++ b/src/ansiblelint/rules/package_latest.md @@ -7,7 +7,7 @@ In production environments, you should set `state` to `present` and specify a ta Setting `state` to `latest` not only installs software, it performs an update and installs additional packages. This can result in performance degradation or loss of service. -If you do want to update packages to the latest version, you should also set the `update_only` parameter to `true` to avoid installing additional packages. +If you do want to update packages to the latest version, you should also set the `update_only` or `only_upgrade` parameter to `true` based on package manager to avoid installing additional packages. ## Problematic Code @@ -32,11 +32,17 @@ If you do want to update packages to the latest version, you should also set the name: some-package state: latest # <- Installs the latest package. - - name: Install Ansible with update_only to false + - name: Install sudo with update_only to false ansible.builtin.yum: name: sudo state: latest update_only: false # <- Updates and installs packages. + + - name: Install sudo with only_upgrade to false + ansible.builtin.apt: + name: sudo + state: latest + only_upgrade: false # <- Upgrades and installs packages ``` ## Correct Code @@ -63,9 +69,15 @@ If you do want to update packages to the latest version, you should also set the name: some-package state: present # <- Ensures the package is installed. - - name: Update Ansible with update_only to true + - name: Update sudo with update_only to true ansible.builtin.yum: name: sudo state: latest update_only: true # <- Updates but does not install additional packages. + + - name: Install sudo with only_upgrade to true + ansible.builtin.apt: + name: sudo + state: latest + only_upgrade: true # <- Upgrades but does not install additional packages. ``` diff --git a/src/ansiblelint/rules/package_latest.py b/src/ansiblelint/rules/package_latest.py index a00a540..9c8ce3c 100644 --- a/src/ansiblelint/rules/package_latest.py +++ b/src/ansiblelint/rules/package_latest.py @@ -1,4 +1,5 @@ """Implementations of the package-latest rule.""" + # Copyright (c) 2016 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -79,5 +80,6 @@ class PackageIsNotLatestRule(AnsibleLintRule): task["action"]["__ansible_module__"] in self._package_managers and not task["action"].get("version") and not task["action"].get("update_only") + and not task["action"].get("only_upgrade") and task["action"].get("state") == "latest" ) diff --git a/src/ansiblelint/rules/partial_become.md b/src/ansiblelint/rules/partial_become.md index 01f9dae..672ef96 100644 --- a/src/ansiblelint/rules/partial_become.md +++ b/src/ansiblelint/rules/partial_become.md @@ -5,6 +5,13 @@ This rule checks that privilege escalation is activated when changing users. To perform an action as a different user with the `become_user` directive, you must set `become: true`. +This rule can produce the following messages: + +- `partial-become[play]`: become_user requires become to work as expected, at + play level. +- `partial-become[task]`: become_user requires become to work as expected, at + task level. + !!! warning While Ansible inherits have of `become` and `become_user` from upper levels, @@ -19,12 +26,13 @@ must set `become: true`. --- - name: Example playbook hosts: localhost + become: true # <- Activates privilege escalation. tasks: - name: Start the httpd service as the apache user ansible.builtin.service: name: httpd state: started - become_user: apache # <- Does not change the user because "become: true" is not set. + become_user: apache # <- Does not change the user because "become: true" is not set. ``` ## Correct Code @@ -37,6 +45,82 @@ must set `become: true`. ansible.builtin.service: name: httpd state: started - become: true # <- Activates privilege escalation. - become_user: apache # <- Changes the user with the desired privileges. + become: true # <- Activates privilege escalation. + become_user: apache # <- Changes the user with the desired privileges. + +# Stand alone playbook alternative, applies to all tasks + +- name: Example playbook + hosts: localhost + become: true # <- Activates privilege escalation. + become_user: apache # <- Changes the user with the desired privileges. + tasks: + - name: Start the httpd service as the apache user + ansible.builtin.service: + name: httpd + state: started +``` + +## Problematic Code + +```yaml +--- +- name: Example playbook 1 + hosts: localhost + become: true # <- Activates privilege escalation. + tasks: + - name: Include a task file + ansible.builtin.include_tasks: tasks.yml ``` + +```yaml +--- +- name: Example playbook 2 + hosts: localhost + tasks: + - name: Include a task file + ansible.builtin.include_tasks: tasks.yml +``` + +```yaml +# tasks.yml +- name: Start the httpd service as the apache user + ansible.builtin.service: + name: httpd + state: started + become_user: apache # <- Does not change the user because "become: true" is not set. +``` + +## Correct Code + +```yaml +--- +- name: Example playbook 1 + hosts: localhost + tasks: + - name: Include a task file + ansible.builtin.include_tasks: tasks.yml +``` + +```yaml +--- +- name: Example playbook 2 + hosts: localhost + tasks: + - name: Include a task file + ansible.builtin.include_tasks: tasks.yml +``` + +```yaml +# tasks.yml +- name: Start the httpd service as the apache user + ansible.builtin.service: + name: httpd + state: started + become: true # <- Activates privilege escalation. + become_user: apache # <- Does not change the user because "become: true" is not set. +``` + +!!! note + + This rule can be automatically fixed using [`--fix`](../autofix.md) option. diff --git a/src/ansiblelint/rules/partial_become.py b/src/ansiblelint/rules/partial_become.py index d14c06f..879b186 100644 --- a/src/ansiblelint/rules/partial_become.py +++ b/src/ansiblelint/rules/partial_become.py @@ -1,4 +1,5 @@ """Implementation of partial-become rule.""" + # Copyright (c) 2016 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -21,115 +22,231 @@ from __future__ import annotations import sys -from functools import reduce from typing import TYPE_CHECKING, Any +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from ansiblelint.constants import LINE_NUMBER_KEY -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin if TYPE_CHECKING: + from collections.abc import Iterator + from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable + from ansiblelint.utils import Task -def _get_subtasks(data: dict[str, Any]) -> list[Any]: - result: list[Any] = [] - block_names = [ - "tasks", - "pre_tasks", - "post_tasks", - "handlers", - "block", - "always", - "rescue", - ] - for name in block_names: - if data and name in data: - result += data[name] or [] - return result - - -def _nested_search(term: str, data: dict[str, Any]) -> Any: - if data and term in data: - return True - return reduce( - (lambda x, y: x or _nested_search(term, y)), - _get_subtasks(data), - False, - ) - - -def _become_user_without_become(becomeuserabove: bool, data: dict[str, Any]) -> Any: - if "become" in data: - # If become is in lineage of tree then correct - return False - if "become_user" in data and _nested_search("become", data): - # If 'become_user' on tree and become somewhere below - # we must check for a case of a second 'become_user' without a - # 'become' in its lineage - subtasks = _get_subtasks(data) - return reduce( - (lambda x, y: x or _become_user_without_become(False, y)), - subtasks, - False, - ) - if _nested_search("become_user", data): - # Keep searching down if 'become_user' exists in the tree below current task - subtasks = _get_subtasks(data) - return len(subtasks) == 0 or reduce( - ( - lambda x, y: x - or _become_user_without_become( - becomeuserabove or "become_user" in data, - y, - ) - ), - subtasks, - False, - ) - # If at bottom of tree, flag up if 'become_user' existed in the lineage of the tree and - # 'become' was not. This is an error if any lineage has a 'become_user' but no become - return becomeuserabove - - -class BecomeUserWithoutBecomeRule(AnsibleLintRule): - """become_user requires become to work as expected.""" +class BecomeUserWithoutBecomeRule(AnsibleLintRule, TransformMixin): + """``become_user`` should have a corresponding ``become`` at the play or task level.""" id = "partial-become" - description = "``become_user`` without ``become`` will not actually change user" + description = "``become_user`` should have a corresponding ``become`` at the play or task level." severity = "VERY_HIGH" tags = ["unpredictability"] version_added = "historic" - def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: - if file.kind == "playbook": - result = _become_user_without_become(False, data) - if result: - return [ - self.create_matcherror( - message=self.shortdesc, - filename=file, - lineno=data[LINE_NUMBER_KEY], - ), - ] - return [] + def matchplay( + self: BecomeUserWithoutBecomeRule, + file: Lintable, + data: dict[str, Any], + ) -> list[MatchError]: + """Match become_user without become in play. + + :param file: The file to lint. + :param data: The data to lint (play) + :returns: A list of errors. + """ + if file.kind != "playbook": + return [] + errors = [] + partial = "become_user" in data and "become" not in data + if partial: + error = self.create_matcherror( + message=self.shortdesc, + filename=file, + tag=f"{self.id}[play]", + lineno=data[LINE_NUMBER_KEY], + ) + errors.append(error) + return errors + + def matchtask( + self: BecomeUserWithoutBecomeRule, + task: Task, + file: Lintable | None = None, + ) -> list[MatchError]: + """Match become_user without become in task. + + :param task: The task to lint. + :param file: The file to lint. + :returns: A list of errors. + """ + data = task.normalized_task + errors = [] + partial = "become_user" in data and "become" not in data + if partial: + error = self.create_matcherror( + message=self.shortdesc, + filename=file, + tag=f"{self.id}[task]", + lineno=task[LINE_NUMBER_KEY], + ) + errors.append(error) + return errors + + def _dive(self: BecomeUserWithoutBecomeRule, data: CommentedSeq) -> Iterator[Any]: + """Dive into the data and yield each item. + + :param data: The data to dive into. + :yield: Each item in the data. + """ + for item in data: + for nested in ("block", "rescue", "always"): + if nested in item: + yield from self._dive(item[nested]) + yield item + + def transform( + self: BecomeUserWithoutBecomeRule, + match: MatchError, + lintable: Lintable, + data: CommentedMap | CommentedSeq | str, + ) -> None: + """Transform the data. + + :param match: The match to transform. + :param lintable: The file to transform. + :param data: The data to transform. + """ + if not isinstance(data, CommentedSeq): + return + + obj = self.seek(match.yaml_path, data) + if "become" in obj and "become_user" in obj: + match.fixed = True + return + if "become" not in obj and "become_user" not in obj: + match.fixed = True + return + + self._transform_plays(plays=data) + + if "become" in obj and "become_user" in obj: + match.fixed = True + return + if "become" not in obj and "become_user" not in obj: + match.fixed = True + return + + def is_ineligible_for_transform( + self: BecomeUserWithoutBecomeRule, + data: CommentedMap, + ) -> bool: + """Check if the data is eligible for transformation. + + :param data: The data to check. + :returns: True if ineligible, False otherwise. + """ + if any("include" in key for key in data): + return True + if "notify" in data: + return True + return False + + def _transform_plays(self, plays: CommentedSeq) -> None: + """Transform the plays. + + :param plays: The plays to transform. + """ + for play in plays: + self._transform_play(play=play) + + def _transform_play(self, play: CommentedMap) -> None: + """Transform the play. + + :param play: The play to transform. + """ + # Ensure we have no includes in this play + task_groups = ("tasks", "pre_tasks", "post_tasks", "handlers") + for task_group in task_groups: + tasks = self._dive(play.get(task_group, [])) + for task in tasks: + if self.is_ineligible_for_transform(task): + return + remove_play_become_user = False + for task_group in task_groups: + tasks = self._dive(play.get(task_group, [])) + for task in tasks: + b_in_t = "become" in task + bu_in_t = "become_user" in task + b_in_p = "become" in play + bu_in_p = "become_user" in play + if b_in_t and not bu_in_t and bu_in_p: + # Preserve the end comment if become is the last key + comment = None + if list(task.keys())[-1] == "become" and "become" in task.ca.items: + comment = task.ca.items.pop("become") + become_index = list(task.keys()).index("become") + task.insert(become_index + 1, "become_user", play["become_user"]) + if comment: + self._attach_comment_end(task, comment) + remove_play_become_user = True + if bu_in_t and not b_in_t and b_in_p: + become_user_index = list(task.keys()).index("become_user") + task.insert(become_user_index, "become", play["become"]) + if bu_in_t and not b_in_t and not b_in_p: + # Preserve the end comment if become_user is the last key + comment = None + if ( + list(task.keys())[-1] == "become_user" + and "become_user" in task.ca.items + ): + comment = task.ca.items.pop("become_user") + task.pop("become_user") + if comment: + self._attach_comment_end(task, comment) + if remove_play_become_user: + del play["become_user"] + + def _attach_comment_end( + self, + obj: CommentedMap | CommentedSeq, + comment: Any, + ) -> None: + """Attach a comment to the end of the object. + + :param obj: The object to attach the comment to. + :param comment: The comment to attach. + """ + if isinstance(obj, CommentedMap): + last = list(obj.keys())[-1] + if not isinstance(obj[last], CommentedSeq | CommentedMap): + obj.ca.items[last] = comment + return + self._attach_comment_end(obj[last], comment) + elif isinstance(obj, CommentedSeq): + if not isinstance(obj[-1], CommentedSeq | CommentedMap): + obj.ca.items[len(obj)] = comment + return + self._attach_comment_end(obj[-1], comment) # testing code to be loaded only with pytest or when executed the rule file if "pytest" in sys.modules: - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner - def test_partial_become_positive() -> None: - """Positive test for partial-become.""" + def test_partial_become_pass() -> None: + """No errors found for partial-become.""" collection = RulesCollection() collection.register(BecomeUserWithoutBecomeRule()) success = "examples/playbooks/rule-partial-become-without-become-pass.yml" good_runner = Runner(success, rules=collection) assert [] == good_runner.run() - def test_partial_become_negative() -> None: - """Negative test for partial-become.""" + def test_partial_become_fail() -> None: + """Errors found for partial-become.""" collection = RulesCollection() collection.register(BecomeUserWithoutBecomeRule()) failure = "examples/playbooks/rule-partial-become-without-become-fail.yml" diff --git a/src/ansiblelint/rules/playbook_extension.py b/src/ansiblelint/rules/playbook_extension.py index b4ca41c..a08c984 100644 --- a/src/ansiblelint/rules/playbook_extension.py +++ b/src/ansiblelint/rules/playbook_extension.py @@ -1,4 +1,5 @@ """Implementation of playbook-extension rule.""" + # Copyright (c) 2016, Tsukinowa Inc. <info@tsukinowa.jp> # Copyright (c) 2018, Ansible Project from __future__ import annotations @@ -39,7 +40,8 @@ class PlaybookExtensionRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/risky_file_permissions.md b/src/ansiblelint/rules/risky_file_permissions.md index 2a62a6d..ad46871 100644 --- a/src/ansiblelint/rules/risky_file_permissions.md +++ b/src/ansiblelint/rules/risky_file_permissions.md @@ -50,7 +50,7 @@ Modules that are checked: - name: Safe example of using ini_file (2nd solution) community.general.ini_file: path: foo - mode: 0600 # explicitly sets the desired permissions, to make the results predictable + mode: "0600" # explicitly sets the desired permissions, to make the results predictable - name: Safe example of using copy (3rd solution) ansible.builtin.copy: diff --git a/src/ansiblelint/rules/risky_file_permissions.py b/src/ansiblelint/rules/risky_file_permissions.py index f4494eb..7fe3870 100644 --- a/src/ansiblelint/rules/risky_file_permissions.py +++ b/src/ansiblelint/rules/risky_file_permissions.py @@ -137,8 +137,9 @@ class MissingFilePermissionsRule(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.testing import RunFromText # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.testing import RunFromText @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/risky_octal.py b/src/ansiblelint/rules/risky_octal.py index e3651ea..e3dad38 100644 --- a/src/ansiblelint/rules/risky_octal.py +++ b/src/ansiblelint/rules/risky_octal.py @@ -1,4 +1,5 @@ """Implementation of risky-octal rule.""" + # Copyright (c) 2013-2014 Will Thames <will@thames.id.au> # # Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/src/ansiblelint/rules/risky_shell_pipe.md b/src/ansiblelint/rules/risky_shell_pipe.md index 302d0d9..dfede8e 100644 --- a/src/ansiblelint/rules/risky_shell_pipe.md +++ b/src/ansiblelint/rules/risky_shell_pipe.md @@ -7,7 +7,7 @@ The return status of a pipeline is the exit status of the command. The `pipefail` option ensures that tasks fail as expected if the first command fails. -As this requirement does apply to PowerShell, for shell commands that have +As this requirement does not apply to PowerShell, for shell commands that have `pwsh` inside `executable` attribute, this rule will not trigger. ## Problematic Code @@ -30,10 +30,14 @@ As this requirement does apply to PowerShell, for shell commands that have become: false tasks: - name: Pipeline with pipefail - ansible.builtin.shell: set -o pipefail && false | cat + ansible.builtin.shell: + cmd: set -o pipefail && false | cat + executable: /bin/bash - name: Pipeline with pipefail, multi-line - ansible.builtin.shell: | - set -o pipefail # <-- adding this will prevent surprises - false | cat + ansible.builtin.shell: + cmd: | + set -o pipefail # <-- adding this will prevent surprises + false | cat + executable: /bin/bash ``` diff --git a/src/ansiblelint/rules/risky_shell_pipe.py b/src/ansiblelint/rules/risky_shell_pipe.py index 58a6f5f..b0c6063 100644 --- a/src/ansiblelint/rules/risky_shell_pipe.py +++ b/src/ansiblelint/rules/risky_shell_pipe.py @@ -1,4 +1,5 @@ """Implementation of risky-shell-pipe rule.""" + from __future__ import annotations import re @@ -62,8 +63,9 @@ class ShellWithoutPipefail(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("file", "expected"), diff --git a/src/ansiblelint/rules/role_name.py b/src/ansiblelint/rules/role_name.py index 499c086..ebe0b1a 100644 --- a/src/ansiblelint/rules/role_name.py +++ b/src/ansiblelint/rules/role_name.py @@ -1,4 +1,5 @@ """Implementation of role-name rule.""" + # Copyright (c) 2020 Gael Chamoulaud <gchamoul@redhat.com> # Copyright (c) 2020 Sorin Sbarnea <ssbarnea@redhat.com> # @@ -94,6 +95,26 @@ class RoleNames(AnsibleLintRule): if file.kind not in ("meta", "role", "playbook"): return result + if file.kind == "meta": + for role in file.data.get("dependencies", []): + if isinstance(role, dict): + role_name = role["role"] + elif isinstance(role, str): + role_name = role + else: + msg = "Role dependency has unexpected type." + raise TypeError(msg) + if "/" in role_name: + result.append( + self.create_matcherror( + f"Avoid using paths when importing roles. ({role_name})", + filename=file, + lineno=role_name.ansible_pos[1], + tag=f"{self.id}[path]", + ), + ) + return result + if file.kind == "playbook": for play in file.data: if "roles" in play: @@ -143,7 +164,7 @@ class RoleNames(AnsibleLintRule): if meta_data: try: return str(meta_data["galaxy_info"]["role_name"]) - except KeyError: + except (KeyError, TypeError): pass return default @@ -151,8 +172,9 @@ class RoleNames(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failure"), @@ -168,3 +190,44 @@ if "pytest" in sys.modules: for result in results: assert result.tag == "role-name[path]" assert len(results) == failure + + @pytest.mark.parametrize( + ("test_file", "failure"), + (pytest.param("examples/roles/role_with_deps_paths", 3, id="fail"),), + ) + def test_role_deps_path_names( + default_rules_collection: RulesCollection, + test_file: str, + failure: int, + ) -> None: + """Test rule matches.""" + results = Runner( + test_file, + rules=default_rules_collection, + ).run() + expected_errors = ( + ("role-name[path]", 3), + ("role-name[path]", 9), + ("role-name[path]", 10), + ) + assert len(expected_errors) == failure + for idx, result in enumerate(results): + assert result.tag == expected_errors[idx][0] + assert result.lineno == expected_errors[idx][1] + assert len(results) == failure + + @pytest.mark.parametrize( + ("test_file", "failure"), + (pytest.param("examples/roles/test-no-deps-role", 0, id="no_deps"),), + ) + def test_role_no_deps( + default_rules_collection: RulesCollection, + test_file: str, + failure: int, + ) -> None: + """Test role if no dependencies are present in meta/main.yml.""" + results = Runner( + test_file, + rules=default_rules_collection, + ).run() + assert len(results) == failure diff --git a/src/ansiblelint/rules/run_once.py b/src/ansiblelint/rules/run_once.py index 78968b6..d656711 100644 --- a/src/ansiblelint/rules/run_once.py +++ b/src/ansiblelint/rules/run_once.py @@ -1,4 +1,5 @@ """Optional Ansible-lint rule to warn use of run_once with strategy free.""" + from __future__ import annotations import sys @@ -34,7 +35,7 @@ class RunOnce(AnsibleLintRule): if not file or file.kind != "playbook" or not data: return [] - strategy = data.get("strategy", None) + strategy = data.get("strategy") run_once = data.get("run_once", False) if (not strategy and not run_once) or strategy != "free": return [] @@ -43,7 +44,6 @@ class RunOnce(AnsibleLintRule): message="Play uses strategy: free", filename=file, tag=f"{self.id}[play]", - # pylint: disable=protected-access lineno=strategy._line_number, # noqa: SLF001 ), ] @@ -74,8 +74,9 @@ class RunOnce(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failure"), diff --git a/src/ansiblelint/rules/sanity.md b/src/ansiblelint/rules/sanity.md index 5b4f3a4..f17cdaf 100644 --- a/src/ansiblelint/rules/sanity.md +++ b/src/ansiblelint/rules/sanity.md @@ -1,10 +1,10 @@ # sanity This rule checks the `tests/sanity/ignore-x.x.txt` file for disallowed ignores. -This rule is extremely opinionated and enforced by Partner Engineering. The +This rule is extremely opinionated and enforced by Partner Engineering as a requirement for Red Hat Certification. The currently allowed ruleset is subject to change, but is starting at a minimal number of allowed ignores for maximum test enforcement. Any commented-out ignore -entries are not evaluated. +entries are not evaluated, and ignore files for unsupported versions of ansible-core are not evaluated. This rule can produce messages like: @@ -29,10 +29,9 @@ Currently allowed ignores for all Ansible versions are: - `compile-2.7!skip` - `compile-3.5` - `compile-3.5!skip` - -Additionally allowed ignores for Ansible 2.9 are: -- `validate-modules:deprecation-mismatch` -- `validate-modules:invalid-documentation` +- `shellcheck` +- `shebang` +- `pylint:used-before-assignment` ## Problematic code diff --git a/src/ansiblelint/rules/sanity.py b/src/ansiblelint/rules/sanity.py index 09fe7cc..921e712 100644 --- a/src/ansiblelint/rules/sanity.py +++ b/src/ansiblelint/rules/sanity.py @@ -1,6 +1,8 @@ """Implementation of sanity rule.""" + from __future__ import annotations +import re import sys from typing import TYPE_CHECKING @@ -27,12 +29,7 @@ class CheckSanityIgnoreFiles(AnsibleLintRule): # Partner Engineering defines this list. Please contact PE for changes. - allowed_ignores_v2_9 = [ - "validate-modules:deprecation-mismatch", # Note: 2.9 expects a deprecated key in the METADATA. It was removed in later versions. - "validate-modules:invalid-documentation", # Note: The removed_at_date key in the deprecated section is invalid for 2.9. - ] - - allowed_ignores_all = [ + allowed_ignores = [ "validate-modules:missing-gplv3-license", "action-plugin-docs", # Added for Networking Collections "import-2.6", @@ -47,7 +44,18 @@ class CheckSanityIgnoreFiles(AnsibleLintRule): "compile-2.7!skip", "compile-3.5", "compile-3.5!skip", + "shebang", # Unreliable test + "shellcheck", # Unreliable test + "pylint:used-before-assignment", # Unreliable test + ] + + no_check_ignore_files = [ + "ignore-2.9", + "ignore-2.10", + "ignore-2.11", + "ignore-2.12", ] + _ids = { "sanity[cannot-ignore]": "Ignore file contains ... at line ..., which is not a permitted ignore.", "sanity[bad-ignore]": "Ignore file entry at ... is formatted incorrectly. Please review.", @@ -62,44 +70,55 @@ class CheckSanityIgnoreFiles(AnsibleLintRule): results: list[MatchError] = [] test = "" + check_dirs = { + "plugins", + "roles", + } + if file.kind != "sanity-ignore-file": return [] with file.path.open(encoding="utf-8") as ignore_file: entries = ignore_file.read().splitlines() - ignores = self.allowed_ignores_all - - # If there is a ignore-2.9.txt file, add the v2_9 list of allowed ignores - if "ignore-2.9.txt" in str(file.abspath): - ignores = self.allowed_ignores_all + self.allowed_ignores_v2_9 + if any(name in str(file.abspath) for name in self.no_check_ignore_files): + return [] for line_num, entry in enumerate(entries, 1): - if entry and entry[0] != "#": - try: - if "#" in entry: - entry, _ = entry.split("#") - (_, test) = entry.split() - if test not in ignores: + base_ignore_dir = "" + + if entry: + # match up to the first "/" + regex = re.match("[^/]*", entry) + + if regex: + base_ignore_dir = regex.group(0) + + if base_ignore_dir in check_dirs: + try: + if "#" in entry: + entry, _ = entry.split("#") + (_, test) = entry.split() + if test not in self.allowed_ignores: + results.append( + self.create_matcherror( + message=f"Ignore file contains {test} at line {line_num}, which is not a permitted ignore.", + tag="sanity[cannot-ignore]", + lineno=line_num, + filename=file, + ), + ) + + except ValueError: results.append( self.create_matcherror( - message=f"Ignore file contains {test} at line {line_num}, which is not a permitted ignore.", - tag="sanity[cannot-ignore]", + message=f"Ignore file entry at {line_num} is formatted incorrectly. Please review.", + tag="sanity[bad-ignore]", lineno=line_num, filename=file, ), ) - except ValueError: - results.append( - self.create_matcherror( - message=f"Ignore file entry at {line_num} is formatted incorrectly. Please review.", - tag="sanity[bad-ignore]", - lineno=line_num, - filename=file, - ), - ) - return results @@ -107,8 +126,9 @@ class CheckSanityIgnoreFiles(AnsibleLintRule): if "pytest" in sys.modules: import pytest - from ansiblelint.rules import RulesCollection # pylint: disable=ungrouped-imports - from ansiblelint.runner import Runner # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports + from ansiblelint.rules import RulesCollection + from ansiblelint.runner import Runner @pytest.mark.parametrize( ("test_file", "failures", "tags"), diff --git a/src/ansiblelint/rules/schema.py b/src/ansiblelint/rules/schema.py index 32ff2eb..6997acd 100644 --- a/src/ansiblelint/rules/schema.py +++ b/src/ansiblelint/rules/schema.py @@ -1,7 +1,9 @@ """Rule definition for JSON Schema Validations.""" + from __future__ import annotations import logging +import re import sys from typing import TYPE_CHECKING, Any @@ -13,6 +15,7 @@ from ansiblelint.schemas.main import validate_file_schema from ansiblelint.text import has_jinja if TYPE_CHECKING: + from ansiblelint.config import Options from ansiblelint.utils import Task @@ -94,7 +97,11 @@ class ValidateSchemaRule(AnsibleLintRule): def matchplay(self, file: Lintable, data: dict[str, Any]) -> list[MatchError]: """Return matches found for a specific playbook.""" results: list[MatchError] = [] - if not data or file.kind not in ("tasks", "handlers", "playbook"): + if ( + not data + or file.kind not in ("tasks", "handlers", "playbook") + or file.failed() + ): return results # check at play level results.extend(self._get_field_matches(file=file, data=data)) @@ -117,7 +124,7 @@ class ValidateSchemaRule(AnsibleLintRule): message=msg, lineno=data.get("__line__", 1), lintable=file, - rule=ValidateSchemaRule(), + rule=self, details=ValidateSchemaRule.description, tag=f"schema[{file.kind}]", ), @@ -129,9 +136,13 @@ class ValidateSchemaRule(AnsibleLintRule): task: Task, file: Lintable | None = None, ) -> bool | str | MatchError | list[MatchError]: - results = [] + results: list[MatchError] = [] if not file: file = Lintable("", kind="tasks") + + if file.failed(): + return results + results.extend(self._get_field_matches(file=file, data=task.raw_task)) for key in pre_checks["task"]: if key in task.raw_task: @@ -141,7 +152,7 @@ class ValidateSchemaRule(AnsibleLintRule): MatchError( message=msg, lintable=file, - rule=ValidateSchemaRule(), + rule=self, details=ValidateSchemaRule.description, tag=f"schema[{tag}]", ), @@ -151,12 +162,15 @@ class ValidateSchemaRule(AnsibleLintRule): def matchyaml(self, file: Lintable) -> list[MatchError]: """Return JSON validation errors found as a list of MatchError(s).""" result: list[MatchError] = [] + + if file.failed(): + return result + if file.kind not in JSON_SCHEMAS: return result - errors = validate_file_schema(file) - if errors: - if errors[0].startswith("Failed to load YAML file"): + for error in validate_file_schema(file): + if error.startswith("Failed to load YAML file"): _logger.debug( "Ignored failure to load %s for schema validation, as !vault may cause it.", file, @@ -165,13 +179,14 @@ class ValidateSchemaRule(AnsibleLintRule): result.append( MatchError( - message=errors[0], + message=error, lintable=file, - rule=ValidateSchemaRule(), + rule=self, details=ValidateSchemaRule.description, tag=f"schema[{file.kind}]", ), ) + break if not result: result = super().matchyaml(file) @@ -183,7 +198,6 @@ if "pytest" in sys.modules: import pytest # pylint: disable=ungrouped-imports - from ansiblelint.config import options from ansiblelint.rules import RulesCollection from ansiblelint.runner import Runner @@ -191,27 +205,30 @@ if "pytest" in sys.modules: ("file", "expected_kind", "expected"), ( pytest.param( - "examples/collection/galaxy.yml", + "examples/.collection/galaxy.yml", "galaxy", - ["'GPL' is not one of"], + [r".*'GPL' is not one of.*https://"], id="galaxy", ), pytest.param( "examples/roles/invalid_requirements_schema/meta/requirements.yml", "requirements", - ["{'foo': 'bar'} is not valid under any of the given schemas"], + [ + # r".*{'foo': 'bar'} is not valid under any of the given schemas.*https://", + r".*{'foo': 'bar'} is not of type 'array'.*https://", + ], id="requirements", ), pytest.param( "examples/roles/invalid_meta_schema/meta/main.yml", "meta", - ["False is not of type 'string'"], + [r".*False is not of type 'string'.*https://"], id="meta", ), pytest.param( "examples/playbooks/vars/invalid_vars_schema.yml", "vars", - ["'123' does not match any of the regexes"], + [r".* '123' does not match any of the regexes.*https://"], id="vars", ), pytest.param( @@ -223,14 +240,23 @@ if "pytest" in sys.modules: pytest.param( "examples/ee_broken/execution-environment.yml", "execution-environment", - ["{'foo': 'bar'} is not valid under any of the given schemas"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], id="execution-environment-broken", ), - ("examples/meta/runtime.yml", "meta-runtime", []), + pytest.param( + "examples/meta/runtime.yml", + "meta-runtime", + [], + id="meta-runtime", + ), pytest.param( "examples/broken_collection_meta_runtime/meta/runtime.yml", "meta-runtime", - ["Additional properties are not allowed ('foo' was unexpected)"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], id="meta-runtime-broken", ), pytest.param( @@ -242,7 +268,9 @@ if "pytest" in sys.modules: pytest.param( "examples/inventory/broken_dev_inventory.yml", "inventory", - ["Additional properties are not allowed ('foo' was unexpected)"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], id="inventory-broken", ), pytest.param( @@ -260,7 +288,17 @@ if "pytest" in sys.modules: pytest.param( "examples/broken/.ansible-lint", "ansible-lint-config", - ["Additional properties are not allowed ('foo' was unexpected)"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], + id="ansible-lint-config-broken", + ), + pytest.param( + "examples/broken_supported_ansible_also/.ansible-lint", + "ansible-lint-config", + [ + r".*supported_ansible_also True is not of type 'array'.*https://", + ], id="ansible-lint-config-broken", ), pytest.param( @@ -272,7 +310,9 @@ if "pytest" in sys.modules: pytest.param( "examples/broken/ansible-navigator.yml", "ansible-navigator-config", - ["Additional properties are not allowed ('ansible' was unexpected)"], + [ + r".*Additional properties are not allowed \('ansible' was unexpected\).*https://", + ], id="ansible-navigator-config-broken", ), pytest.param( @@ -284,20 +324,25 @@ if "pytest" in sys.modules: pytest.param( "examples/roles/broken_argument_specs/meta/argument_specs.yml", "role-arg-spec", - ["Additional properties are not allowed ('foo' was unexpected)"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], id="role-arg-spec-broken", ), pytest.param( "examples/changelogs/changelog.yaml", "changelog", - ["Additional properties are not allowed ('foo' was unexpected)"], + [ + r".*Additional properties are not allowed \('foo' was unexpected\).*https://", + ], id="changelog", ), pytest.param( "examples/rulebooks/rulebook-fail.yml", "rulebook", [ - "Additional properties are not allowed ('that_should_not_be_here' was unexpected)", + # r".*Additional properties are not allowed \('that_should_not_be_here' was unexpected\).*https://", + r".*'sss' is not of type 'object'.*https://", ], id="rulebook", ), @@ -324,19 +369,24 @@ if "pytest" in sys.modules: ), ), ) - def test_schema(file: str, expected_kind: str, expected: list[str]) -> None: + def test_schema( + file: str, + expected_kind: str, + expected: list[str], + config_options: Options, + ) -> None: """Validate parsing of ansible output.""" lintable = Lintable(file) assert lintable.kind == expected_kind - rules = RulesCollection(options=options) + rules = RulesCollection(options=config_options) rules.register(ValidateSchemaRule()) results = Runner(lintable, rules=rules).run() assert len(results) == len(expected), results for idx, result in enumerate(results): assert result.filename.endswith(file) - assert expected[idx] in result.message + assert re.match(expected[idx], result.message) assert result.tag == f"schema[{expected_kind}]" @pytest.mark.parametrize( @@ -356,12 +406,13 @@ if "pytest" in sys.modules: expected_kind: str, expected_tag: str, count: int, + config_options: Options, ) -> None: """Validate ability to detect schema[moves].""" lintable = Lintable(file) assert lintable.kind == expected_kind - rules = RulesCollection(options=options) + rules = RulesCollection(options=config_options) rules.register(ValidateSchemaRule()) results = Runner(lintable, rules=rules).run() diff --git a/src/ansiblelint/rules/syntax_check.md b/src/ansiblelint/rules/syntax_check.md index e8197a5..566fa33 100644 --- a/src/ansiblelint/rules/syntax_check.md +++ b/src/ansiblelint/rules/syntax_check.md @@ -9,7 +9,7 @@ You can exclude these files from linting, but it is better to make sure they can be loaded by Ansible. This is often achieved by editing the inventory file and/or `ansible.cfg` so ansible can load required variables. -If undefined variables cause the failure, you can use the jinja `default()` +If undefined variables cause the failure, you can use the Jinja `default()` filter to provide fallback values, like in the example below. This rule is among the few `unskippable` rules that cannot be added to @@ -20,9 +20,32 @@ fixtures that are invalid on purpose. One of the most common sources of errors is a failure to assert the presence of various variables at the beginning of the playbook. -This rule can produce messages like below: +This rule can produce messages like: -- `syntax-check[empty-playbook]` is raised when a playbook file has no content. +- `syntax-check[empty-playbook]`: Empty playbook, nothing to do +- `syntax-check[malformed]`: A malformed block was encountered while loading a block +- `syntax-check[missing-file]`: Unable to retrieve file contents ... Could not find or access ... +- `syntax-check[unknown-module]`: couldn't resolve module/action +- `syntax-check[specific]`: for other errors not mentioned above. + +## syntax-check[unknown-module] + +The linter relies on ansible-core code to load the ansible code and it will +produce a syntax error if the code refers to ansible content that is not +installed. You must ensure that all collections and roles used inside your +repository are listed inside a [`requirements.yml`](https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-roles-and-collections-from-the-same-requirements-yml-file) file, so the linter can +install them when they are missing. + +Valid location for `requirements.yml` are: + +- `requirements.yml` +- `roles/requirements.yml` +- `collections/requirements.yml` +- `tests/requirements.yml` +- `tests/integration/requirements.yml` +- `tests/unit/requirements.yml` + +Note: If requirements are test related then they should be inside `tests/`. ## Problematic code diff --git a/src/ansiblelint/rules/syntax_check.py b/src/ansiblelint/rules/syntax_check.py index c6a4c5e..9b072f6 100644 --- a/src/ansiblelint/rules/syntax_check.py +++ b/src/ansiblelint/rules/syntax_check.py @@ -1,4 +1,5 @@ """Rule definition for ansible syntax check.""" + from __future__ import annotations import re @@ -15,6 +16,8 @@ class KnownError: regex: re.Pattern[str] +# Order matters, we only report the first matching pattern, the one at the end +# is used to match generic or less specific patterns. OUTPUT_PATTERNS = ( KnownError( tag="missing-file", @@ -25,9 +28,9 @@ OUTPUT_PATTERNS = ( ), ), KnownError( - tag="specific", + tag="no-file", regex=re.compile( - r"^ERROR! (?P<title>[^\n]*)\n\nThe error appears to be in '(?P<filename>[\w\/\.\-]+)': line (?P<line>\d+), column (?P<column>\d+)", + r"^ERROR! (?P<title>No file specified for [^\n]*)", re.MULTILINE | re.S | re.DOTALL, ), ), @@ -45,6 +48,28 @@ OUTPUT_PATTERNS = ( re.MULTILINE | re.S | re.DOTALL, ), ), + KnownError( + tag="unknown-module", + regex=re.compile( + r"^ERROR! (?P<title>couldn't resolve module/action [^\n]*)\n\nThe error appears to be in '(?P<filename>[\w\/\.\-]+)': line (?P<line>\d+), column (?P<column>\d+)", + re.MULTILINE | re.S | re.DOTALL, + ), + ), + KnownError( + tag="specific", + regex=re.compile( + r"^ERROR! (?P<title>[^\n]*)\n\nThe error appears to be in '(?P<filename>[\w\/\.\-]+)': line (?P<line>\d+), column (?P<column>\d+)", + re.MULTILINE | re.S | re.DOTALL, + ), + ), + # "ERROR! the role 'this_role_is_missing' was not found in ROLE_INCLUDE_PATHS\n\nThe error appears to be in 'FILE_PATH': line 5, column 7, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n roles:\n - this_role_is_missing\n ^ here\n" + KnownError( + tag="specific", + regex=re.compile( + r"^ERROR! (?P<title>the role '.*' was not found in[^\n]*)'(?P<filename>[\w\/\.\-]+)': line (?P<line>\d+), column (?P<column>\d+)", + re.MULTILINE | re.S | re.DOTALL, + ), + ), ) diff --git a/src/ansiblelint/rules/var_naming.md b/src/ansiblelint/rules/var_naming.md index 3386a0c..e4034f0 100644 --- a/src/ansiblelint/rules/var_naming.md +++ b/src/ansiblelint/rules/var_naming.md @@ -22,7 +22,7 @@ Possible errors messages: - `var-naming[no-jinja]`: Variables names must not contain jinja2 templating. - `var-naming[pattern]`: Variables names should match ... regex. - `var-naming[no-role-prefix]`: Variables names from within roles should use - `role_name_` as a prefix. + `role_name_` as a prefix. Underlines are accepted before the prefix. - `var-naming[no-reserved]`: Variables names must not be Ansible reserved names. - `var-naming[read-only]`: This special variable is read-only. diff --git a/src/ansiblelint/rules/var_naming.py b/src/ansiblelint/rules/var_naming.py index 389530d..14a4c40 100644 --- a/src/ansiblelint/rules/var_naming.py +++ b/src/ansiblelint/rules/var_naming.py @@ -1,4 +1,5 @@ """Implementation of var-naming rule.""" + from __future__ import annotations import keyword @@ -9,13 +10,19 @@ from typing import TYPE_CHECKING, Any from ansible.parsing.yaml.objects import AnsibleUnicode from ansible.vars.reserved import get_reserved_names -from ansiblelint.config import options -from ansiblelint.constants import ANNOTATION_KEYS, LINE_NUMBER_KEY, RC +from ansiblelint.config import Options, options +from ansiblelint.constants import ( + ANNOTATION_KEYS, + LINE_NUMBER_KEY, + PLAYBOOK_ROLE_KEYWORDS, + RC, +) from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable from ansiblelint.rules import AnsibleLintRule, RulesCollection from ansiblelint.runner import Runner from ansiblelint.skip_utils import get_rule_skips_from_line +from ansiblelint.text import has_jinja, is_fqcn_or_name from ansiblelint.utils import parse_yaml_from_file if TYPE_CHECKING: @@ -160,10 +167,15 @@ class VariableNamingRule(AnsibleLintRule): rule=self, ) - if prefix and not ident.startswith(f"{prefix}_"): + if ( + prefix + and not ident.lstrip("_").startswith(f"{prefix}_") + and not has_jinja(prefix) + and is_fqcn_or_name(prefix) + ): return MatchError( tag="var-naming[no-role-prefix]", - message="Variables names from within roles should use role_name_ as a prefix.", + message=f"Variables names from within roles should use {prefix}_ as a prefix.", rule=self, ) return None @@ -187,6 +199,37 @@ class VariableNamingRule(AnsibleLintRule): else our_vars[LINE_NUMBER_KEY] ) raw_results.append(match_error) + roles = data.get("roles", []) + for role in roles: + if isinstance(role, AnsibleUnicode): + continue + role_fqcn = role.get("role", role.get("name")) + prefix = role_fqcn.split("/" if "/" in role_fqcn else ".")[-1] + for key in list(role.keys()): + if key not in PLAYBOOK_ROLE_KEYWORDS: + match_error = self.get_var_naming_matcherror(key, prefix=prefix) + if match_error: + match_error.filename = str(file.path) + match_error.message += f" (vars: {key})" + match_error.lineno = ( + key.ansible_pos[1] + if isinstance(key, AnsibleUnicode) + else role[LINE_NUMBER_KEY] + ) + raw_results.append(match_error) + + our_vars = role.get("vars", {}) + for key in our_vars: + match_error = self.get_var_naming_matcherror(key, prefix=prefix) + if match_error: + match_error.filename = str(file.path) + match_error.message += f" (vars: {key})" + match_error.lineno = ( + key.ansible_pos[1] + if isinstance(key, AnsibleUnicode) + else our_vars[LINE_NUMBER_KEY] + ) + raw_results.append(match_error) if raw_results: lines = file.content.splitlines() for match in raw_results: @@ -266,7 +309,8 @@ class VariableNamingRule(AnsibleLintRule): if str(file.kind) == "vars" and file.data: meta_data = parse_yaml_from_file(str(file.path)) for key in meta_data: - match_error = self.get_var_naming_matcherror(key) + prefix = file.role if file.role else "" + match_error = self.get_var_naming_matcherror(key, prefix=prefix) if match_error: match_error.filename = filename match_error.lineno = key.ansible_pos[1] @@ -298,13 +342,21 @@ if "pytest" in sys.modules: @pytest.mark.parametrize( ("file", "expected"), ( - pytest.param("examples/playbooks/rule-var-naming-fail.yml", 7, id="0"), + pytest.param( + "examples/playbooks/var-naming/rule-var-naming-fail.yml", + 7, + id="0", + ), pytest.param("examples/Taskfile.yml", 0, id="1"), ), ) - def test_invalid_var_name_playbook(file: str, expected: int) -> None: + def test_invalid_var_name_playbook( + file: str, + expected: int, + config_options: Options, + ) -> None: """Test rule matches.""" - rules = RulesCollection(options=options) + rules = RulesCollection(options=config_options) rules.register(VariableNamingRule()) results = Runner(Lintable(file), rules=rules).run() assert len(results) == expected @@ -337,6 +389,40 @@ if "pytest" in sys.modules: assert result.tag == expected_errors[idx][0] assert result.lineno == expected_errors[idx][1] + def test_var_naming_with_role_prefix( + default_rules_collection: RulesCollection, + ) -> None: + """Test rule matches.""" + results = Runner( + Lintable("examples/roles/role_vars_prefix_detection"), + rules=default_rules_collection, + ).run() + assert len(results) == 2 + for result in results: + assert result.tag == "var-naming[no-role-prefix]" + + def test_var_naming_with_role_prefix_plays( + default_rules_collection: RulesCollection, + ) -> None: + """Test rule matches.""" + results = Runner( + Lintable("examples/playbooks/role_vars_prefix_detection.yml"), + rules=default_rules_collection, + exclude_paths=["examples/roles/role_vars_prefix_detection"], + ).run() + expected_errors = ( + ("var-naming[no-role-prefix]", 9), + ("var-naming[no-role-prefix]", 12), + ("var-naming[no-role-prefix]", 15), + ("var-naming[no-role-prefix]", 25), + ("var-naming[no-role-prefix]", 32), + ("var-naming[no-role-prefix]", 45), + ) + assert len(results) == len(expected_errors) + for idx, result in enumerate(results): + assert result.tag == expected_errors[idx][0] + assert result.lineno == expected_errors[idx][1] + def test_var_naming_with_pattern() -> None: """Test rule matches.""" role_path = "examples/roles/var_naming_pattern/tasks/main.yml" @@ -364,7 +450,7 @@ if "pytest" in sys.modules: def test_var_naming_with_include_role_import_role() -> None: """Test with include role and import role.""" - role_path = "examples/test_collection/roles/my_role/tasks/main.yml" + role_path = "examples/.test_collection/roles/my_role/tasks/main.yml" result = run_ansible_lint(role_path) assert result.returncode == RC.SUCCESS assert "var-naming" not in result.stdout diff --git a/src/ansiblelint/rules/yaml.md b/src/ansiblelint/rules/yaml.md index 8dc56eb..654f80e 100644 --- a/src/ansiblelint/rules/yaml.md +++ b/src/ansiblelint/rules/yaml.md @@ -1,6 +1,8 @@ # yaml -This rule checks YAML syntax and is an implementation of `yamllint`. +This rule checks YAML syntax by using [yamllint] library but with a +[specific default configuration](#yamllint-configuration), one that is +compatible with both, our internal reformatter (`--fix`) and also [prettier]. You can disable YAML syntax violations by adding `yaml` to the `skip_list` in your Ansible-lint configuration as follows: @@ -53,6 +55,7 @@ Some of the detailed error codes that you might see are: - `yaml[empty-lines]` - _too many blank lines (...> ...)_ - `yaml[indentation]` - _Wrong indentation: expected ... but found ..._ - `yaml[key-duplicates]` - _Duplication of key "..." in mapping_ +- `yaml[line-length]` - _Line too long (... > ... characters)_ - `yaml[new-line-at-end-of-file]` - _No new line character at the end of file_ - `yaml[octal-values]`: forbidden implicit or explicit [octal](#octals) value - `yaml[syntax]` - YAML syntax is broken @@ -72,6 +75,13 @@ for it does check these. If for some reason, you do not want to follow our defaults, you can create a `.yamllint` file in your project and this will take precedence over our defaults. +## Additional Information for Multiline Strings + +Adhering to yaml[line-length] rule, for writing multiline strings we recommend +using Block Style Indicator: literal style indicated by a pipe (|) or folded +style indicated by a right angle bracket (>), instead of escaping the newlines +with backslashes. Reference [guide] for writing multiple line strings in yaml. + ## Problematic code ```yaml @@ -91,7 +101,53 @@ foo2: "0o777" # <-- Explicitly quoting octal is less risky. bar: ... # Correct comment indentation. ``` +## Yamllint configuration + +If you decide to add a custom yamllint config to your project, ansible-lint +might refuse to run if it detects that some of your options are incompatible and +ask you to correct them. When this happens, you will see a message like the one +below: + +``` +CRITICAL Found incompatible custom yamllint configuration (.yamllint), please either remove the file or edit it to comply with: + - comments.min-spaces-from-content must be 1 + - braces.min-spaces-inside must be 0 + - braces.max-spaces-inside must be 1 + - octal-values.forbid-implicit-octal must be true + - octal-values.forbid-explicit-octal must be true + +Read https://ansible.readthedocs.io/projects/lint/rules/yaml/ for more details regarding why we have these requirements. +``` + +!!! warning + + [Auto-fix](../autofix.md) functionality will change **inline comment indentation to one + character instead of two**, which is the default of [yamllint]. The reason + for this decision was to keep reformatting compatibility + with [prettier], which is the most popular reformatter. + + ```yaml title=".yamllint" + rules: + comments: + min-spaces-from-content: 1 # prettier compatibility + ``` + + There is no need to create this yamllint config file, but if you also + run yamllint yourself, you might want to create it to make it behave + the same way as ansible-lint. + +Below you can find the default yamllint configuration that our linter will use +when there is no custom file present. + +```yaml +{!../src/ansiblelint/data/.yamllint!} +``` + [1.1]: https://yaml.org/spec/1.1/ [1.2.0]: https://yaml.org/spec/1.2.0/ [1.2.2]: https://yaml.org/spec/1.2.2/ [yaml specification]: https://yaml.org/ +[guide]: + https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html#yaml-basics +[prettier]: https://prettier.io/ +[yamllint]: https://yamllint.readthedocs.io/en/stable/ diff --git a/src/ansiblelint/rules/yaml_rule.py b/src/ansiblelint/rules/yaml_rule.py index 4da4d41..3ec5b59 100644 --- a/src/ansiblelint/rules/yaml_rule.py +++ b/src/ansiblelint/rules/yaml_rule.py @@ -1,27 +1,29 @@ """Implementation of yaml linting rule (yamllint integration).""" + from __future__ import annotations import logging import sys -from collections.abc import Iterable +from collections.abc import Iterable, MutableMapping, MutableSequence from typing import TYPE_CHECKING from yamllint.linter import run as run_yamllint from ansiblelint.constants import LINE_NUMBER_KEY, SKIPPED_RULES_KEY from ansiblelint.file_utils import Lintable -from ansiblelint.rules import AnsibleLintRule +from ansiblelint.rules import AnsibleLintRule, TransformMixin from ansiblelint.yaml_utils import load_yamllint_config if TYPE_CHECKING: from typing import Any + from ansiblelint.config import Options from ansiblelint.errors import MatchError _logger = logging.getLogger(__name__) -class YamllintRule(AnsibleLintRule): +class YamllintRule(AnsibleLintRule, TransformMixin): """Violations reported by yamllint.""" id = "yaml" @@ -73,6 +75,12 @@ class YamllintRule(AnsibleLintRule): self.severity = "VERY_LOW" if problem.level == "error": self.severity = "MEDIUM" + # Ignore truthy violation with github workflows ("on:" keys) + if problem.rule == "truthy" and file.path.parent.parts[-2:] == ( + ".github", + "workflows", + ): + continue matches.append( self.create_matcherror( # yamllint does return lower-case sentences @@ -85,6 +93,22 @@ class YamllintRule(AnsibleLintRule): ) return matches + def transform( + self: YamllintRule, + match: MatchError, + lintable: Lintable, + data: MutableMapping[str, Any] | MutableSequence[Any] | str, + ) -> None: + """Transform yaml. + + :param match: MatchError instance + :param lintable: Lintable instance + :param data: data to transform + """ + # This method does nothing because the YAML reformatting is implemented + # in data dumper. Still presence of this method helps us with + # documentation generation. + def _combine_skip_rules(data: Any) -> set[str]: """Return a consolidated list of skipped rules.""" @@ -107,7 +131,7 @@ def _fetch_skips(data: Any, collector: dict[int, set[str]]) -> dict[int, set[str collector[data.get(LINE_NUMBER_KEY)].update(rules) if isinstance(data, Iterable) and not isinstance(data, str): if isinstance(data, dict): - for _entry, value in data.items(): + for value in data.values(): _fetch_skips(value, collector) else: # must be some kind of list for entry in data: @@ -128,7 +152,6 @@ if "pytest" in sys.modules: import pytest # pylint: disable=ungrouped-imports - from ansiblelint.config import options from ansiblelint.rules import RulesCollection from ansiblelint.runner import Runner @@ -180,15 +203,26 @@ if "pytest" in sys.modules: [], id="rule-yaml-pass", ), + pytest.param( + "examples/yamllint/.github/workflows/ci.yml", + "yaml", + [], + id="rule-yaml-github-workflow", + ), ), ) @pytest.mark.filterwarnings("ignore::ansible_compat.runtime.AnsibleWarning") - def test_yamllint(file: str, expected_kind: str, expected: list[str]) -> None: + def test_yamllint( + file: str, + expected_kind: str, + expected: list[str], + config_options: Options, + ) -> None: """Validate parsing of ansible output.""" lintable = Lintable(file) assert lintable.kind == expected_kind - rules = RulesCollection(options=options) + rules = RulesCollection(options=config_options) rules.register(YamllintRule()) results = Runner(lintable, rules=rules).run() diff --git a/src/ansiblelint/runner.py b/src/ansiblelint/runner.py index 9d3500d..f487329 100644 --- a/src/ansiblelint/runner.py +++ b/src/ansiblelint/runner.py @@ -1,8 +1,10 @@ """Runner implementation.""" + from __future__ import annotations import json import logging +import math import multiprocessing import multiprocessing.pool import os @@ -12,7 +14,9 @@ import tempfile import warnings from dataclasses import dataclass from fnmatch import fnmatch +from functools import cache from pathlib import Path +from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any from ansible.errors import AnsibleError @@ -23,31 +27,24 @@ from ansible_compat.runtime import AnsibleWarning import ansiblelint.skip_utils import ansiblelint.utils -from ansiblelint._internal.rules import ( - BaseRule, - LoadingFailureRule, - RuntimeErrorRule, - WarningRule, -) from ansiblelint.app import App, get_app from ansiblelint.constants import States from ansiblelint.errors import LintWarning, MatchError, WarnSource from ansiblelint.file_utils import Lintable, expand_dirs_in_lintables from ansiblelint.logger import timed_info -from ansiblelint.rules.syntax_check import OUTPUT_PATTERNS, AnsibleSyntaxCheckRule +from ansiblelint.rules.syntax_check import OUTPUT_PATTERNS from ansiblelint.text import strip_ansi_escape from ansiblelint.utils import ( PLAYBOOK_DIR, - _include_children, - _roles_children, - _taskshandlers_children, + HandleChildren, + parse_examples_from_plugin, template, ) if TYPE_CHECKING: - from collections.abc import Generator - from typing import Callable + from collections.abc import Callable, Generator + from ansiblelint._internal.rules import BaseRule from ansiblelint.config import Options from ansiblelint.constants import FileType from ansiblelint.rules import RulesCollection @@ -77,11 +74,13 @@ class Runner: verbosity: int = 0, checked_files: set[Lintable] | None = None, project_dir: str | None = None, + _skip_ansible_syntax_check: bool = False, ) -> None: """Initialize a Runner instance.""" self.rules = rules self.lintables: set[Lintable] = set() self.project_dir = os.path.abspath(project_dir) if project_dir else None + self.skip_ansible_syntax_check = _skip_ansible_syntax_check if skip_list is None: skip_list = [] @@ -107,6 +106,8 @@ class Runner: checked_files = set() self.checked_files = checked_files + self.app = get_app(cached=True) + def _update_exclude_paths(self, exclude_paths: list[str]) -> None: if exclude_paths: # These will be (potentially) relative paths @@ -172,19 +173,19 @@ class Runner: if isinstance(warn.source, WarnSource): match = MatchError( message=warn.source.message or warn.category.__name__, - rule=WarningRule(), - filename=warn.source.filename.filename, + rule=self.rules["warning"], + lintable=Lintable(warn.source.filename.filename), tag=warn.source.tag, lineno=warn.source.lineno, ) else: filename = warn.source match = MatchError( - message=warn.message - if isinstance(warn.message, str) - else "?", - rule=WarningRule(), - filename=str(filename), + message=( + warn.message if isinstance(warn.message, str) else "?" + ), + rule=self.rules["warning"], + lintable=Lintable(str(filename)), ) matches.append(match) continue @@ -215,7 +216,7 @@ class Runner: lintable=lintable, message=str(lintable.exc), details=str(lintable.exc.__cause__), - rule=LoadingFailureRule(), + rule=self.rules["load-failure"], tag=f"load-failure[{lintable.exc.__class__.__name__.lower()}]", ), ) @@ -226,60 +227,63 @@ class Runner: MatchError( lintable=lintable, message="File or directory not found.", - rule=LoadingFailureRule(), + rule=self.rules["load-failure"], tag="load-failure[not-found]", ), ) # -- phase 1 : syntax check in parallel -- - app = get_app(offline=True) + if not self.skip_ansible_syntax_check: + # app = get_app(cached=True) - def worker(lintable: Lintable) -> list[MatchError]: - # pylint: disable=protected-access - return self._get_ansible_syntax_check_matches( - lintable=lintable, - app=app, - ) + def worker(lintable: Lintable) -> list[MatchError]: + return self._get_ansible_syntax_check_matches( + lintable=lintable, + app=self.app, + ) - for lintable in self.lintables: - if lintable.kind not in ("playbook", "role") or lintable.stop_processing: - continue - files.append(lintable) + for lintable in self.lintables: + if ( + lintable.kind not in ("playbook", "role") + or lintable.stop_processing + ): + continue + files.append(lintable) - # avoid resource leak warning, https://github.com/python/cpython/issues/90549 - # pylint: disable=unused-variable - global_resource = multiprocessing.Semaphore() # noqa: F841 + # avoid resource leak warning, https://github.com/python/cpython/issues/90549 + # pylint: disable=unused-variable + global_resource = multiprocessing.Semaphore() # noqa: F841 - pool = multiprocessing.pool.ThreadPool(processes=multiprocessing.cpu_count()) - return_list = pool.map(worker, files, chunksize=1) - pool.close() - pool.join() - for data in return_list: - matches.extend(data) + pool = multiprocessing.pool.ThreadPool(processes=threads()) + return_list = pool.map(worker, files, chunksize=1) + pool.close() + pool.join() + for data in return_list: + matches.extend(data) + + matches = self._filter_excluded_matches(matches) - matches = self._filter_excluded_matches(matches) # -- phase 2 --- - if not matches: - # do our processing only when ansible syntax check passed in order - # to avoid causing runtime exceptions. Our processing is not as - # resilient to be able process garbage. - matches.extend(self._emit_matches(files)) + # do our processing only when ansible syntax check passed in order + # to avoid causing runtime exceptions. Our processing is not as + # resilient to be able process garbage. + matches.extend(self._emit_matches(files)) - # remove duplicates from files list - files = [value for n, value in enumerate(files) if value not in files[:n]] + # remove duplicates from files list + files = [value for n, value in enumerate(files) if value not in files[:n]] - for file in self.lintables: - if file in self.checked_files or not file.kind: - continue - _logger.debug( - "Examining %s of type %s", - ansiblelint.file_utils.normpath(file.path), - file.kind, - ) + for file in self.lintables: + if file in self.checked_files or not file.kind or file.failed(): + continue + _logger.debug( + "Examining %s of type %s", + ansiblelint.file_utils.normpath(file.path), + file.kind, + ) - matches.extend( - self.rules.run(file, tags=set(self.tags), skip_list=self.skip_list), - ) + matches.extend( + self.rules.run(file, tags=set(self.tags), skip_list=self.skip_list), + ) # update list of checked files self.checked_files.update(self.lintables) @@ -296,7 +300,12 @@ class Runner: app: App, ) -> list[MatchError]: """Run ansible syntax check and return a list of MatchError(s).""" - default_rule: BaseRule = AnsibleSyntaxCheckRule() + try: + default_rule: BaseRule = self.rules["syntax-check"] + except ValueError: + # if syntax-check is not loaded, we do not perform any syntax check, + # that might happen during testing + return [] fh = None results = [] if lintable.kind not in ("playbook", "role"): @@ -341,6 +350,9 @@ class Runner: # https://github.com/paramiko/paramiko/issues/2038 env = app.runtime.environ.copy() env["PYTHONWARNINGS"] = "ignore" + # Avoid execution failure if user customized any_unparsed_is_failed setting + # https://github.com/ansible/ansible-lint/issues/3650 + env["ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED"] = "False" run = subprocess.run( cmd, @@ -357,6 +369,7 @@ class Runner: filename = lintable lineno = 1 column = None + ignore_rc = False stderr = strip_ansi_escape(run.stderr) stdout = strip_ansi_escape(run.stdout) @@ -376,25 +389,40 @@ class Runner: details = groups.get("details", "") lineno = int(groups.get("line", 1)) - if "filename" in groups: + if ( + "filename" in groups + and str(lintable.path.absolute()) != groups["filename"] + and lintable.filename != groups["filename"] + ): + # avoids creating a new lintable object if the filename + # is matching as this might prevent Lintable.failed() + # feature from working well. filename = Lintable(groups["filename"]) else: filename = lintable column = int(groups.get("column", 1)) - results.append( - MatchError( - message=title, - lintable=filename, - lineno=lineno, - column=column, - rule=rule, - details=details, - tag=f"{rule.id}[{pattern.tag}]", - ), - ) - if not results: - rule = RuntimeErrorRule() + if ( + pattern.tag in ("unknown-module", "specific") + and app.options.nodeps + ): + ignore_rc = True + else: + results.append( + MatchError( + message=title, + lintable=filename, + lineno=lineno, + column=column, + rule=rule, + details=details, + tag=f"{rule.id}[{pattern.tag}]", + ), + ) + break + + if not results and not ignore_rc: + rule = self.rules["internal-error"] message = ( f"Unexpected error code {run.returncode} from " f"execution of: {' '.join(cmd)}" @@ -427,6 +455,9 @@ class Runner: visited: set[Lintable] = set() while visited != self.lintables: for lintable in self.lintables - visited: + visited.add(lintable) + if not lintable.path.exists(): + continue try: children = self.find_children(lintable) for child in children: @@ -437,11 +468,13 @@ class Runner: except MatchError as exc: if not exc.filename: # pragma: no branch exc.filename = str(lintable.path) - exc.rule = LoadingFailureRule() + exc.rule = self.rules["load-failure"] yield exc except AttributeError: - yield MatchError(lintable=lintable, rule=LoadingFailureRule()) - visited.add(lintable) + yield MatchError( + lintable=lintable, + rule=self.rules["load-failure"], + ) def find_children(self, lintable: Lintable) -> list[Lintable]: """Traverse children of a single file or folder.""" @@ -452,21 +485,25 @@ class Runner: add_all_plugin_dirs(playbook_dir or ".") if lintable.kind == "role": playbook_ds = AnsibleMapping({"roles": [{"role": str(lintable.path)}]}) + elif lintable.kind == "plugin": + return self.plugin_children(lintable) elif lintable.kind not in ("playbook", "tasks"): return [] else: try: playbook_ds = ansiblelint.utils.parse_yaml_from_file(str(lintable.path)) except AnsibleError as exc: - raise SystemExit(exc) from exc + msg = f"Loading {lintable.filename} caused an {type(exc).__name__} exception: {exc}, file was ignored." + logging.exception(msg) + return [] results = [] # playbook_ds can be an AnsibleUnicode string, which we consider invalid if isinstance(playbook_ds, str): - raise MatchError(lintable=lintable, rule=LoadingFailureRule()) + raise MatchError(lintable=lintable, rule=self.rules["load-failure"]) for item in ansiblelint.utils.playbook_items(playbook_ds): # if lintable.kind not in ["playbook"]: for child in self.play_children( - lintable.path.parent, + lintable, item, lintable.kind, playbook_dir, @@ -487,35 +524,40 @@ class Runner: if path != path_str: child.path = Path(path) child.name = child.path.name - results.append(child) return results def play_children( self, - basedir: Path, + lintable: Lintable, item: tuple[str, Any], parent_type: FileType, playbook_dir: str, ) -> list[Lintable]: """Flatten the traversed play tasks.""" # pylint: disable=unused-argument - delegate_map: dict[str, Callable[[str, Any, Any, FileType], list[Lintable]]] = { - "tasks": _taskshandlers_children, - "pre_tasks": _taskshandlers_children, - "post_tasks": _taskshandlers_children, - "block": _taskshandlers_children, - "include": _include_children, - "ansible.builtin.include": _include_children, - "import_playbook": _include_children, - "ansible.builtin.import_playbook": _include_children, - "roles": _roles_children, - "dependencies": _roles_children, - "handlers": _taskshandlers_children, - "include_tasks": _include_children, - "ansible.builtin.include_tasks": _include_children, - "import_tasks": _include_children, - "ansible.builtin.import_tasks": _include_children, + basedir = lintable.path.parent + handlers = HandleChildren(self.rules, app=self.app) + + delegate_map: dict[ + str, + Callable[[Lintable, Any, Any, FileType], list[Lintable]], + ] = { + "tasks": handlers.taskshandlers_children, + "pre_tasks": handlers.taskshandlers_children, + "post_tasks": handlers.taskshandlers_children, + "block": handlers.taskshandlers_children, + "include": handlers.include_children, + "ansible.builtin.include": handlers.include_children, + "import_playbook": handlers.include_children, + "ansible.builtin.import_playbook": handlers.include_children, + "roles": handlers.roles_children, + "dependencies": handlers.roles_children, + "handlers": handlers.taskshandlers_children, + "include_tasks": handlers.include_children, + "ansible.builtin.include_tasks": handlers.include_children, + "import_tasks": handlers.include_children, + "ansible.builtin.import_tasks": handlers.include_children, } (k, v) = item add_all_plugin_dirs(str(basedir.resolve())) @@ -527,11 +569,92 @@ class Runner: {"playbook_dir": PLAYBOOK_DIR or str(basedir.resolve())}, fail_on_undefined=False, ) - return delegate_map[k](str(basedir), k, v, parent_type) + return delegate_map[k](lintable, k, v, parent_type) return [] + def plugin_children(self, lintable: Lintable) -> list[Lintable]: + """Collect lintable sections from plugin file.""" + offset, content = parse_examples_from_plugin(lintable) + if not content: + # No examples, nothing to see here + return [] + examples = Lintable( + name=lintable.name, + content=content, + kind="yaml", + base_kind="text/yaml", + parent=lintable, + ) + examples.line_offset = offset -def _get_matches(rules: RulesCollection, options: Options) -> LintResult: + # pylint: disable=consider-using-with + examples.file = NamedTemporaryFile( + mode="w+", + suffix=f"_{lintable.path.name}.yaml", + ) + examples.file.write(content) + examples.file.flush() + examples.filename = examples.file.name + examples.path = Path(examples.file.name) + return [examples] + + +@cache +def threads() -> int: + """Determine how many threads to use. + + Inside containers we want to respect limits imposed. + + When present /sys/fs/cgroup/cpu.max can contain something like: + $ podman/docker run -it --rm --cpus 1.5 ubuntu:latest cat /sys/fs/cgroup/cpu.max + 150000 100000 + # "max 100000" is returned when no limits are set. + + See: https://github.com/python/cpython/issues/80235 + See: https://github.com/python/cpython/issues/70879 + """ + os_cpu_count = multiprocessing.cpu_count() + # Cgroup CPU bandwidth limit available in Linux since 2.6 kernel + + cpu_max_fname = "/sys/fs/cgroup/cpu.max" + cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + if os.path.exists(cpu_max_fname): + # cgroup v2 + # https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html + with open(cpu_max_fname, encoding="utf-8") as fh: + cpu_quota_us, cpu_period_us = fh.read().strip().split() + elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname): + # cgroup v1 + # https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management + with open(cfs_quota_fname, encoding="utf-8") as fh: + cpu_quota_us = fh.read().strip() + with open(cfs_period_fname, encoding="utf-8") as fh: + cpu_period_us = fh.read().strip() + else: + # No Cgroup CPU bandwidth limit (e.g. non-Linux platform) + cpu_quota_us = "max" + cpu_period_us = "100000" # unused, for consistency with default values + + if cpu_quota_us == "max": + # No active Cgroup quota on a Cgroup-capable platform + return os_cpu_count + cpu_quota_us_int = int(cpu_quota_us) + cpu_period_us_int = int(cpu_period_us) + if cpu_quota_us_int > 0 and cpu_period_us_int > 0: + return math.ceil(cpu_quota_us_int / cpu_period_us_int) + # Setting a negative cpu_quota_us value is a valid way to disable + # cgroup CPU bandwidth limits + return os_cpu_count + + +def get_matches(rules: RulesCollection, options: Options) -> LintResult: + """Get matches for given rules and options. + + :param rules: Rules to use for linting. + :param options: Options to use for linting. + :returns: LintResult containing matches and checked files. + """ lintables = ansiblelint.utils.get_lintables(opts=options, args=options.lintables) for rule in rules: @@ -551,6 +674,7 @@ def _get_matches(rules: RulesCollection, options: Options) -> LintResult: verbosity=options.verbosity, checked_files=checked_files, project_dir=options.project_dir, + _skip_ansible_syntax_check=options._skip_ansible_syntax_check, # noqa: SLF001 ) matches.extend(runner.run()) diff --git a/src/ansiblelint/schemas/__main__.py b/src/ansiblelint/schemas/__main__.py index e3ec8ae..e216c0b 100644 --- a/src/ansiblelint/schemas/__main__.py +++ b/src/ansiblelint/schemas/__main__.py @@ -1,4 +1,5 @@ """Module containing cached JSON schemas.""" + import json import logging import os @@ -68,7 +69,10 @@ def refresh_schemas(min_age_seconds: int = 3600 * 24) -> int: raise RuntimeError(msg) path = Path(__file__).parent.resolve() / f"{kind}.json" _logger.debug("Refreshing %s schema ...", kind) - request = Request(url) + if not url.startswith(("http:", "https:")): + msg = f"Unexpected url schema: {url}" + raise ValueError(msg) + request = Request(url) # noqa: S310 etag = data.get("etag", "") if etag: request.add_header("If-None-Match", f'"{data.get("etag")}"') @@ -108,7 +112,6 @@ def refresh_schemas(min_age_seconds: int = 3600 * 24) -> int: get_schema.cache_clear() else: store_file.touch() - changed = 1 return changed diff --git a/src/ansiblelint/schemas/__store__.json b/src/ansiblelint/schemas/__store__.json index d4bcdca..d66d675 100644 --- a/src/ansiblelint/schemas/__store__.json +++ b/src/ansiblelint/schemas/__store__.json @@ -1,10 +1,10 @@ { "ansible-lint-config": { - "etag": "0ec39ba1ca9c20aea463f7f536c6903c88288f47c1b2b2b3d53b527c293f8cc3", + "etag": "a0bb8004fad70bab34fad94a45b2698125127142ec6b2c8900976aa2bd96a86c", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible-lint-config.json" }, "ansible-navigator-config": { - "etag": "dd0f0dea68266ae61e5a8d6aed0a1279fdee16f2da4911bc27970241df80f798", + "etag": "431f1a81acc74fe1112d5839551105bc2fa4e0314d811699eb525dae4fe3760d", "url": "https://raw.githubusercontent.com/ansible/ansible-navigator/main/src/ansible_navigator/data/ansible-navigator.json" }, "changelog": { @@ -12,19 +12,19 @@ "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/changelog.json" }, "execution-environment": { - "etag": "f3abb1716134227ccd667607840dd7bdebfd02a8980603df031282126dc78264", + "etag": "2e1b1d02460fb93892252439e9634d9574dfdd37aea82af32f4622dacd5990b5", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/execution-environment.json" }, "galaxy": { - "etag": "61f38feb51dc7eaff43ab22f3759b3a5202776ee75ee4204f07135282817f724", + "etag": "4224ac235cc5657bf77b5834cea48b4d573cc8b666694f788590e213adfb8113", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/galaxy.json" }, "inventory": { - "etag": "3dcd4890bf31e634a7c4f6138286a42b4985393f210f7ffaa840c2127876aa55", + "etag": "b52c251a121e2e807928db7b4e09338babde9e74a50d0f74e8908f6e230d101d", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/inventory.json" }, "meta": { - "etag": "0f376059285181985711b4271a6ff34a8dde662b9fc221d09bdcd64e4fbf86bf", + "etag": "fdff861b226b13b711dd7f94301ed5becd6dc5d8d4e872f909d4a3d8133d600a", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/meta.json" }, "meta-runtime": { @@ -32,31 +32,31 @@ "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/meta-runtime.json" }, "molecule": { - "etag": "3456b2e5aaa02fde359ff147cff81d01a37c07f5e10542b6b8b61aaaf8c756a6", + "etag": "3b625438c28e884ac42a14c09ca542fc3e1b4466abaf47d0c28646e0857d3fb5", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/molecule.json" }, "playbook": { - "etag": "acbd5edfc66279f8c3f6f8a99d0874669a254983ace5e4a2cce6105489ab3e21", + "etag": "4f8cbba62fcf8a1fa6e8ef5e42696aec5b0876487478df83a7ffdf8bdbb4abcf", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/playbook.json" }, "requirements": { - "etag": "93c6ccd1f79f58134795b85f9b1193d6e18417dd01a9d1f37d9f247562a1e6fe", + "etag": "5ae3a6058ac626a341338c760db7cef7f02a8911c7293c7e129dbc6b0f8bb86d", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/requirements.json" }, "role-arg-spec": { - "etag": "498a6f716c7e99bd474ae9e7d34b3f43fbf2aad750f769392fc8e29fa590be6c", + "etag": "e41a42e1ca634a9eb2edbc4a180f404bdc71e17aafa464e6651387c08152bbc5", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/role-arg-spec.json" }, "rulebook": { - "etag": "f0bbd0ecd656b2298febccc6da0ecf4a7bd239cc112b9de8292c1f50bad612e0", + "etag": "baba5774a46fcc2bc8c4a8c2f25b49df64a0856e415dbf601b0559f215e55968", "url": "https://raw.githubusercontent.com/ansible/ansible-rulebook/main/ansible_rulebook/schema/ruleset_schema.json" }, "tasks": { - "etag": "f9fbc0855680d1321fa3902181131d73838d922362d8dfb85a4f59402240cc07", + "etag": "9f3b54cf5cc432d57c9691fb3108a7f37996ab0875e2abb66eda0aa62437dcdc", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/tasks.json" }, "vars": { - "etag": "5d6c2c22a58f2b48c2a8d8d129f2516e4f17ffc78a2c9ba045eb5ede0ff749d7", + "etag": "73feaa77561d1d5b0bebe6cd66d499a28d67037055ac6d746139a38c9d28ca04", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json" } } diff --git a/src/ansiblelint/schemas/ansible-lint-config.json b/src/ansiblelint/schemas/ansible-lint-config.json index f7d50e4..7f53ffd 100644 --- a/src/ansiblelint/schemas/ansible-lint-config.json +++ b/src/ansiblelint/schemas/ansible-lint-config.json @@ -17,6 +17,7 @@ "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible-lint-config.json", "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, + "description": "https://ansible.readthedocs.io/projects/lint/configuring/", "examples": [ ".ansible-lint", ".config/ansible-lint.yml", @@ -60,6 +61,11 @@ "title": "Loop Var Prefix", "type": "string" }, + "max_block_depth": { + "title": "Maximum Block Depth", + "type": "integer", + "default": 20 + }, "mock_modules": { "items": { "type": "string" @@ -242,6 +248,13 @@ "title": "Strict", "type": "boolean" }, + "supported_ansible_also": { + "items": { + "type": "string" + }, + "title": "Add supported ansible versions", + "type": "array" + }, "tags": { "items": { "type": "string" diff --git a/src/ansiblelint/schemas/ansible-navigator-config.json b/src/ansiblelint/schemas/ansible-navigator-config.json index e81a878..d528267 100644 --- a/src/ansiblelint/schemas/ansible-navigator-config.json +++ b/src/ansiblelint/schemas/ansible-navigator-config.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, + "description": "See https://ansible.readthedocs.io/projects/navigator/settings/", "properties": { "ansible-navigator": { "additionalProperties": false, @@ -9,7 +10,7 @@ "additionalProperties": false, "properties": { "cmdline": { - "description": "Extra parameters passed to the corresponding command", + "description": "Extra parameters passed to the underlying ansible command (e.g. ansible-playbook, ansible-doc, etc)", "type": "string" }, "config": { @@ -524,7 +525,7 @@ "required": [ "ansible-navigator" ], - "title": "ansible-navigator settings v3", + "title": "ansible-navigator settings v24", "type": "object", - "version": "3" + "version": "24" } diff --git a/src/ansiblelint/schemas/ansible.json b/src/ansiblelint/schemas/ansible.json index 94846d0..9423f7a 100644 --- a/src/ansiblelint/schemas/ansible.json +++ b/src/ansiblelint/schemas/ansible.json @@ -1,5 +1,10 @@ { "$defs": { + "removed-include-module": { + "markdownDescription": "See [include module](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_module.html)", + "not": {}, + "title": "Replace 'include' with either 'ansible.builtin.include_tasks' or 'ansible.builtin.import_tasks'" + }, "ansible.builtin.import_playbook": { "additionalProperties": false, "oneOf": [ @@ -163,7 +168,7 @@ }, "ignore_unreachable": { "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean" }, "module_defaults": { "title": "Module Defaults" @@ -348,8 +353,8 @@ "type": "boolean" }, "gather_facts": { - "title": "Gather Facts", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Gather Facts" }, "gather_subset": { "items": { @@ -523,11 +528,11 @@ }, "ignore_unreachable": { "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean" }, "max_fail_percentage": { "title": "Max Fail Percentage", - "type": "number" + "$ref": "#/$defs/templated-integer" }, "module_defaults": { "title": "Module Defaults" @@ -540,15 +545,23 @@ "$ref": "#/$defs/templated-boolean" }, "order": { - "enum": [ - "default", - "sorted", - "reverse_sorted", - "reverse_inventory", - "shuffle" + "oneOf": [ + { + "enum": [ + "inventory", + "reverse_inventory", + "reverse_sorted", + "shuffle", + "sorted" + ], + "type": "string" + }, + { + "$ref": "#/$defs/full-jinja" + } ], "title": "Order", - "type": "string" + "markdownDescription": "See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#ordering-execution-based-on-inventory" }, "port": { "$ref": "#/$defs/templated-integer", @@ -720,7 +733,7 @@ }, "ignore_unreachable": { "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean" }, "module_defaults": { "title": "Module Defaults" @@ -831,6 +844,15 @@ "title": "Action", "type": "string" }, + "ansible.builtin.include": { + "$ref": "#/$defs/removed-include-module" + }, + "include": { + "$ref": "#/$defs/removed-include-module" + }, + "ansible.legacy.include": { + "$ref": "#/$defs/removed-include-module" + }, "any_errors_fatal": { "$ref": "#/$defs/templated-boolean", "title": "Any Errors Fatal" @@ -914,7 +936,7 @@ }, "ignore_unreachable": { "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean" }, "listen": { "anyOf": [ @@ -1196,6 +1218,7 @@ "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json", "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, + "description": "https://docs.ansible.com/ansible/latest/reference_appendices/playbooks_keywords.html", "examples": [], "title": "Ansible Schemas Bundle 22.4", "type": ["array", "object"] diff --git a/src/ansiblelint/schemas/execution-environment.json b/src/ansiblelint/schemas/execution-environment.json index 4720a93..7d44ab3 100644 --- a/src/ansiblelint/schemas/execution-environment.json +++ b/src/ansiblelint/schemas/execution-environment.json @@ -302,7 +302,8 @@ }, "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/execution-environment.json", "$schema": "http://json-schema.org/draft-07/schema", - "description": "See \nV1: https://docs.ansible.com/automation-controller/latest/html/userguide/ee_reference.html\nV3: https://ansible-builder.readthedocs.io/en/latest/definition/", + "description": "See https://ansible-builder.readthedocs.io/en/latest/definition/ for V3 or https://docs.ansible.com/automation-controller/latest/html/userguide/ee_reference.html for older V1 format.\n", + "documentation_url": "https://ansible.readthedocs.io/projects/builder/en/latest/definition/", "examples": ["execution-environment.yml"], "oneOf": [{ "$ref": "#/$defs/v3" }, { "$ref": "#/$defs/v1" }], "title": "Ansible Execution Environment Schema v1/v3" diff --git a/src/ansiblelint/schemas/galaxy.json b/src/ansiblelint/schemas/galaxy.json index 6381f28..ae03445 100644 --- a/src/ansiblelint/schemas/galaxy.json +++ b/src/ansiblelint/schemas/galaxy.json @@ -13,6 +13,7 @@ "description": "An enumeration.", "enum": [ "0BSD", + "389-exception", "AAL", "ADSL", "AFL-1.1", @@ -26,6 +27,7 @@ "AGPL-3.0-or-later", "AMDPLPA", "AML", + "AML-glslang", "AMPAS", "ANTLR-PD", "ANTLR-PD-fallback", @@ -35,10 +37,14 @@ "APSL-1.1", "APSL-1.2", "APSL-2.0", + "ASWF-Digital-Assets-1.0", + "ASWF-Digital-Assets-1.1", "Abstyles", "AdaCore-doc", "Adobe-2006", + "Adobe-Display-PostScript", "Adobe-Glyph", + "Adobe-Utopia", "Afmparse", "Aladdin", "Apache-1.0", @@ -50,13 +56,21 @@ "Artistic-1.0-Perl", "Artistic-1.0-cl8", "Artistic-2.0", + "Asterisk-exception", + "Autoconf-exception-2.0", + "Autoconf-exception-3.0", + "Autoconf-exception-generic", + "Autoconf-exception-generic-3.0", + "Autoconf-exception-macro", "BSD-1-Clause", "BSD-2-Clause", + "BSD-2-Clause-Darwin", "BSD-2-Clause-Patent", "BSD-2-Clause-Views", "BSD-3-Clause", "BSD-3-Clause-Attribution", "BSD-3-Clause-Clear", + "BSD-3-Clause-HP", "BSD-3-Clause-LBNL", "BSD-3-Clause-Modification", "BSD-3-Clause-No-Military-License", @@ -64,6 +78,9 @@ "BSD-3-Clause-No-Nuclear-License-2014", "BSD-3-Clause-No-Nuclear-Warranty", "BSD-3-Clause-Open-MPI", + "BSD-3-Clause-Sun", + "BSD-3-Clause-acpica", + "BSD-3-Clause-flex", "BSD-4-Clause", "BSD-4-Clause-Shortened", "BSD-4-Clause-UC", @@ -71,20 +88,29 @@ "BSD-4.3TAHOE", "BSD-Advertising-Acknowledgement", "BSD-Attribution-HPND-disclaimer", + "BSD-Inferno-Nettverk", "BSD-Protection", "BSD-Source-Code", + "BSD-Source-beginning-file", + "BSD-Systemics", + "BSD-Systemics-W3Works", "BSL-1.0", "BUSL-1.1", "Baekmuk", "Bahyph", "Barr", "Beerware", + "Bison-exception-1.24", + "Bison-exception-2.2", "BitTorrent-1.0", "BitTorrent-1.1", "Bitstream-Charter", "Bitstream-Vera", "BlueOak-1.0.0", + "Boehm-GC", + "Bootloader-exception", "Borceux", + "Brian-Gladman-2-Clause", "Brian-Gladman-3-Clause", "C-UDA-1.0", "CAL-1.0", @@ -96,6 +122,7 @@ "CC-BY-2.5-AU", "CC-BY-3.0", "CC-BY-3.0-AT", + "CC-BY-3.0-AU", "CC-BY-3.0-DE", "CC-BY-3.0-IGO", "CC-BY-3.0-NL", @@ -138,6 +165,7 @@ "CC-BY-SA-3.0", "CC-BY-SA-3.0-AT", "CC-BY-SA-3.0-DE", + "CC-BY-SA-3.0-IGO", "CC-BY-SA-4.0", "CC-PDDC", "CC0-1.0", @@ -159,7 +187,9 @@ "CERN-OHL-S-2.0", "CERN-OHL-W-2.0", "CFITSIO", + "CLISP-exception-2.0", "CMU-Mach", + "CMU-Mach-nodoc", "CNRI-Jython", "CNRI-Python", "CNRI-Python-GPL-Compatible", @@ -169,19 +199,26 @@ "CPOL-1.02", "CUA-OPL-1.0", "Caldera", + "Caldera-no-preamble", "ClArtistic", + "Classpath-exception-2.0", "Clips", "Community-Spec-1.0", "Condor-1.1", "Cornell-Lossless-JPEG", + "Cronyx", "Crossword", "CrystalStacker", "Cube", "D-FSL-1.0", + "DEC-3-Clause", "DL-DE-BY-2.0", + "DL-DE-ZERO-2.0", "DOC", "DRL-1.0", + "DRL-1.1", "DSDP", + "DigiRule-FOSS-exception", "Dotseqn", "ECL-1.0", "ECL-2.0", @@ -198,16 +235,27 @@ "Entessa", "ErlPL-1.1", "Eurosym", + "FBM", "FDK-AAC", + "FLTK-exception", "FSFAP", + "FSFAP-no-warranty-disclaimer", "FSFUL", "FSFULLR", "FSFULLRWD", "FTL", "Fair", + "Fawkes-Runtime-exception", + "Ferguson-Twofish", + "Font-exception-2.0", "Frameworx-1.0", "FreeBSD-DOC", "FreeImage", + "Furuseth", + "GCC-exception-2.0", + "GCC-exception-2.0-note", + "GCC-exception-3.1", + "GCR-docs", "GD", "GFDL-1.1-invariants-only", "GFDL-1.1-invariants-or-later", @@ -229,20 +277,43 @@ "GFDL-1.3-or-later", "GL2PS", "GLWTPL", + "GNAT-exception", + "GNOME-examples-exception", + "GNU-compiler-exception", "GPL-1.0-only", "GPL-1.0-or-later", "GPL-2.0-only", "GPL-2.0-or-later", + "GPL-3.0-interface-exception", + "GPL-3.0-linking-exception", + "GPL-3.0-linking-source-exception", "GPL-3.0-only", "GPL-3.0-or-later", + "GPL-CC-1.0", + "GStreamer-exception-2005", + "GStreamer-exception-2008", "Giftware", "Glide", "Glulxe", + "Gmsh-exception", "Graphics-Gems", "HP-1986", + "HP-1989", "HPND", + "HPND-DEC", + "HPND-Fenneberg-Livingston", + "HPND-INRIA-IMAG", + "HPND-Kevlin-Henney", + "HPND-MIT-disclaimer", "HPND-Markus-Kuhn", + "HPND-Pbmplus", + "HPND-UC", + "HPND-doc", + "HPND-doc-sell", "HPND-export-US", + "HPND-export-US-modify", + "HPND-sell-MIT-disclaimer-xserver", + "HPND-sell-regexpr", "HPND-sell-variant", "HPND-sell-variant-MIT-disclaimer", "HTMLTIDY", @@ -256,9 +327,11 @@ "IPA", "IPL-1.0", "ISC", + "ISC-Veillard", "ImageMagick", "Imlib2", "Info-ZIP", + "Inner-Net-2.0", "Intel", "Intel-ACPI", "Interbase-1.0", @@ -267,7 +340,9 @@ "JSON", "Jam", "JasPer-2.0", + "Kastrup", "Kazlib", + "KiCad-libraries-exception", "Knuth-CTAN", "LAL-1.2", "LAL-1.3", @@ -275,10 +350,14 @@ "LGPL-2.0-or-later", "LGPL-2.1-only", "LGPL-2.1-or-later", + "LGPL-3.0-linking-exception", "LGPL-3.0-only", "LGPL-3.0-or-later", "LGPLLR", + "LLGPL", + "LLVM-exception", "LOOP", + "LPD-document", "LPL-1.0", "LPL-1.02", "LPPL-1.0", @@ -288,24 +367,36 @@ "LPPL-1.3c", "LZMA-SDK-9.11-to-9.20", "LZMA-SDK-9.22", + "LZMA-exception", "Latex2e", + "Latex2e-translated-notice", "Leptonica", "LiLiQ-P-1.1", "LiLiQ-R-1.1", "LiLiQ-Rplus-1.1", "Libpng", + "Libtool-exception", "Linux-OpenIB", + "Linux-man-pages-1-para", "Linux-man-pages-copyleft", + "Linux-man-pages-copyleft-2-para", + "Linux-man-pages-copyleft-var", + "Linux-syscall-note", + "Lucida-Bitmap-Fonts", "MIT", "MIT-0", "MIT-CMU", + "MIT-Festival", "MIT-Modern-Variant", "MIT-Wu", "MIT-advertising", "MIT-enna", "MIT-feh", "MIT-open-group", + "MIT-testregex", "MITNFA", + "MMIXware", + "MPEG-SSG", "MPL-1.0", "MPL-1.1", "MPL-2.0", @@ -314,8 +405,11 @@ "MS-PL", "MS-RL", "MTLL", + "Mackerras-3-Clause", + "Mackerras-3-Clause-acknowledgment", "MakeIndex", "Martin-Birgmeier", + "McPhee-slideshow", "Minpack", "MirOS", "Motosoto", @@ -332,6 +426,7 @@ "NICTA-1.0", "NIST-PD", "NIST-PD-fallback", + "NIST-Software", "NLOD-1.0", "NLOD-2.0", "NLPL", @@ -350,7 +445,9 @@ "Noweb", "O-UDA-1.0", "OCCT-PL", + "OCCT-exception-1.0", "OCLC-2.0", + "OCaml-LGPL-linking-exception", "ODC-By-1.0", "ODbL-1.0", "OFFIS", @@ -383,8 +480,10 @@ "OLDAP-2.6", "OLDAP-2.7", "OLDAP-2.8", + "OLFL-1.3", "OML", "OPL-1.0", + "OPL-UK-3.0", "OPUBL-1.0", "OSET-PL-2.1", "OSL-1.0", @@ -392,14 +491,20 @@ "OSL-2.0", "OSL-2.1", "OSL-3.0", + "OpenJDK-assembly-exception-1.0", "OpenPBS-2.3", "OpenSSL", + "OpenSSL-standalone", + "OpenVision", + "PADL", "PDDL-1.0", "PHP-3.0", "PHP-3.01", + "PS-or-PDF-font-exception-20170817", "PSF-2.0", "Parity-6.0.0", "Parity-7.0.0", + "Pixar", "Plexus", "PolyForm-Noncommercial-1.0.0", "PolyForm-Small-Business-1.0.0", @@ -408,7 +513,11 @@ "Python-2.0.1", "QPL-1.0", "QPL-1.0-INRIA-2004", + "QPL-1.0-INRIA-2004-exception", "Qhull", + "Qt-GPL-exception-1.0", + "Qt-LGPL-exception-1.1", + "Qwt-exception-1.0", "RHeCos-1.1", "RPL-1.1", "RPL-1.5", @@ -417,22 +526,31 @@ "RSCPL", "Rdisc", "Ruby", + "SANE-exception", "SAX-PD", + "SAX-PD-2.0", "SCEA", "SGI-B-1.0", "SGI-B-1.1", "SGI-B-2.0", + "SGI-OpenGL", + "SGP4", "SHL-0.5", "SHL-0.51", + "SHL-2.0", + "SHL-2.1", "SISSL", "SISSL-1.2", + "SL", "SMLNJ", "SMPPL", "SNIA", "SPL-1.0", "SSH-OpenSSH", "SSH-short", + "SSLeay-standalone", "SSPL-1.0", + "SWI-exception", "SWL", "Saxpath", "SchemeReport", @@ -440,29 +558,42 @@ "Sendmail-8.23", "SimPL-2.0", "Sleepycat", + "Soundex", "Spencer-86", "Spencer-94", "Spencer-99", "SugarCRM-1.1.3", + "Sun-PPP", "SunPro", + "Swift-exception", "Symlinks", "TAPR-OHL-1.0", "TCL", "TCP-wrappers", + "TGPPL-1.0", "TMate", "TORQUE-1.1", "TOSL", "TPDL", "TPL-1.0", "TTWL", + "TTYP0", "TU-Berlin-1.0", "TU-Berlin-2.0", + "TermReadKey", + "Texinfo-exception", + "UBDL-exception", "UCAR", "UCL-1.0", + "UMich-Merit", "UPL-1.0", + "URT-RLE", + "Unicode-3.0", "Unicode-DFS-2015", "Unicode-DFS-2016", "Unicode-TOU", + "Universal-FOSS-exception-1.0", + "UnixCrypt", "Unlicense", "VOSTROM", "VSL-1.0", @@ -472,12 +603,16 @@ "W3C-20150513", "WTFPL", "Watcom-1.0", + "Widget-Workshop", "Wsuipa", + "WxWindows-exception-3.1", "X11", "X11-distribute-modifications-variant", "XFree86-1.1", "XSkat", + "Xdebug-1.03", "Xerox", + "Xfig", "Xnet", "YPL-1.0", "YPL-1.1", @@ -485,35 +620,67 @@ "ZPL-2.0", "ZPL-2.1", "Zed", + "Zeeff", "Zend-2.0", "Zimbra-1.3", "Zimbra-1.4", "Zlib", + "bcrypt-Solar-Designer", "blessing", "bzip2-1.0.6", + "check-cvs", "checkmk", "copyleft-next-0.3.0", "copyleft-next-0.3.1", + "cryptsetup-OpenSSL-exception", "curl", "diffmark", + "dtoa", "dvipdfm", + "eCos-exception-2.0", "eGenix", "etalab-2.0", + "fmt-exception", + "freertos-exception-2.0", + "fwlw", "gSOAP-1.3b", + "gnu-javamail-exception", "gnuplot", + "gtkbook", + "hdparm", + "i2p-gpl-java-exception", "iMatix", "libpng-2.0", + "libpri-OpenH323-exception", "libselinux-1.0", "libtiff", "libutil-David-Nugent", + "lsof", + "magaz", + "mailprio", + "metamail", + "mif-exception", "mpi-permissive", "mpich2", "mplus", + "openvpn-openssl-exception", + "pnmstitch", "psfrag", "psutils", + "python-ldap", + "radvd", "snprintf", + "softSurfer", + "ssh-keyscan", + "stunnel-exception", + "swrule", + "u-boot-exception-2.0", + "ulem", + "vsftpd-openssl-exception", "w3m", + "x11vnc-openssl-exception", "xinetd", + "xkeyboard-config-Zinoviev", "xlock", "xpp", "zlib-acknowledgement" @@ -525,6 +692,7 @@ "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, "examples": ["galaxy.yml"], + "markdownDescription": "https://docs.ansible.com/ansible/latest/dev_guide/collections_galaxy_meta.html", "properties": { "authors": { "items": { diff --git a/src/ansiblelint/schemas/inventory.json b/src/ansiblelint/schemas/inventory.json index 80333ce..06cf2ca 100644 --- a/src/ansiblelint/schemas/inventory.json +++ b/src/ansiblelint/schemas/inventory.json @@ -45,7 +45,7 @@ "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/inventory.json", "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": true, - "description": "Ansible Inventory Schema", + "description": "See https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html", "examples": [ "inventory.yaml", "inventory.yml", diff --git a/src/ansiblelint/schemas/main.py b/src/ansiblelint/schemas/main.py index 590aea3..45b0c48 100644 --- a/src/ansiblelint/schemas/main.py +++ b/src/ansiblelint/schemas/main.py @@ -1,8 +1,11 @@ """Module containing cached JSON schemas.""" + from __future__ import annotations import json import logging +import re +import typing from typing import TYPE_CHECKING import jsonschema @@ -18,20 +21,95 @@ if TYPE_CHECKING: from ansiblelint.file_utils import Lintable +def find_best_deep_match( + errors: jsonschema.ValidationError, +) -> jsonschema.ValidationError: + """Return the deepest schema validation error.""" + + def iter_validation_error( + err: jsonschema.ValidationError, + ) -> typing.Iterator[jsonschema.ValidationError]: + if err.context: + for e in err.context: + yield e + yield from iter_validation_error(e) + + return max(iter_validation_error(errors), key=_deep_match_relevance) + + def validate_file_schema(file: Lintable) -> list[str]: """Return list of JSON validation errors found.""" + schema = {} if file.kind not in JSON_SCHEMAS: return [f"Unable to find JSON Schema '{file.kind}' for '{file.path}' file."] try: # convert yaml to json (keys are converted to strings) yaml_data = yaml_load_safe(file.content) json_data = json.loads(json.dumps(yaml_data)) - jsonschema.validate( - instance=json_data, - schema=_schema_cache[file.kind], - ) + schema = _schema_cache[file.kind] + + validator = jsonschema.validators.validator_for(schema) + v = validator(schema) + try: + error = next(v.iter_errors(json_data)) + except StopIteration: + return [] + if error.context: + error = find_best_deep_match(error) + # determine if we want to use our own messages embedded into schemas inside title/markdownDescription fields + if "not" in error.schema and len(error.schema["not"]) == 0: + message = error.schema["title"] + schema = error.schema + else: + message = f"{error.json_path} {error.message}" + + documentation_url = "" + for json_schema in (error.schema, schema): + for k in ("description", "markdownDescription"): + if k in json_schema: + # Find standalone URLs and also markdown urls. + match = re.search( + r"\[.*?\]\((?P<url>https?://[^\s]+)\)|(?P<url2>https?://[^\s]+)", + json_schema[k], + ) + if match: + documentation_url = next( + x for x in match.groups() if x is not None + ) + break + if documentation_url: + break + if documentation_url: + if not message.endswith("."): + message += "." + message += f" See {documentation_url}" except yaml.constructor.ConstructorError as exc: return [f"Failed to load YAML file '{file.path}': {exc.problem}"] except ValidationError as exc: - return [exc.message] - return [] + message = exc.message + documentation_url = "" + for k in ("description", "markdownDescription"): + if k in schema: + # Find standalone URLs and also markdown urls. + match = re.search( + r"\[.*?\]\((https?://[^\s]+)\)|https?://[^\s]+", + schema[k], + ) + if match: + documentation_url = match.groups()[0] + break + if documentation_url: + if not message.endswith("."): + message += "." + message += f" See {documentation_url}" + return [message] + return [message] + + +def _deep_match_relevance(error: jsonschema.ValidationError) -> tuple[bool | int, ...]: + validator = error.validator + return ( + validator not in ("anyOf", "oneOf"), # type: ignore[comparison-overlap] + len(error.absolute_path), + -len(error.path), + ) diff --git a/src/ansiblelint/schemas/meta.json b/src/ansiblelint/schemas/meta.json index 384d113..8971817 100644 --- a/src/ansiblelint/schemas/meta.json +++ b/src/ansiblelint/schemas/meta.json @@ -110,6 +110,25 @@ "title": "ArchLinuxPlatformModel", "type": "object" }, + "AstraLinuxPlatformModel": { + "properties": { + "name": { + "const": "Astra Linux", + "title": "Name", + "type": "string" + }, + "versions": { + "default": "all", + "items": { + "enum": ["1.8", "1.7", "1.6", "2.12", "all"], + "type": "string" + }, + "type": "array" + } + }, + "title": "AstraLinuxPlatformModel", + "type": "object" + }, "ClearLinuxPlatformModel": { "properties": { "name": { @@ -168,6 +187,7 @@ "sid", "squeeze", "stretch", + "trixie", "wheezy", "all" ], @@ -267,7 +287,14 @@ "versions": { "default": "all", "items": { - "enum": ["ascii", "beowulf", "ceres", "jessie", "all"], + "enum": [ + "ascii", + "beowulf", + "chimaera", + "daedalus", + "jessie", + "all" + ], "type": "string" }, "type": "array" @@ -286,7 +313,7 @@ "versions": { "default": "all", "items": { - "enum": ["5.2", "5.4", "all"], + "enum": ["5.2", "5.4", "5.6", "5.8", "6.0", "6.2", "6.4", "all"], "type": "string" }, "type": "array" @@ -348,6 +375,8 @@ "36", "37", "38", + "39", + "40", "all" ], "type": "string" @@ -503,7 +532,7 @@ "namespace": { "markdownDescription": "Used by molecule and ansible-lint to compute FQRN for roles outside collections", "minLength": 2, - "pattern": "^[a-z][a-z0-9_]+$", + "pattern": "^[a-z][a-z0-9_-]+$", "title": "Namespace Name", "type": "string" }, @@ -838,7 +867,15 @@ "versions": { "default": "all", "items": { - "enum": ["17.01", "18.06", "19.07", "21.02", "22.03", "all"], + "enum": [ + "17.01", + "18.06", + "19.07", + "21.02", + "22.03", + "23.05", + "all" + ], "type": "string" }, "type": "array" @@ -879,6 +916,7 @@ "8.8", "9.0", "9.1", + "9.2", "all" ], "type": "string" @@ -908,6 +946,39 @@ "title": "PAN-OSPlatformModel", "type": "object" }, + "RockyLinuxPlatformModel": { + "properties": { + "name": { + "const": "Rocky", + "title": "Name", + "type": "string" + }, + "versions": { + "default": "all", + "items": { + "enum": [ + "8.0", + "8.1", + "8.2", + "8.3", + "8.4", + "8.5", + "8.6", + "8.7", + "8.8", + "9.0", + "9.1", + "9.2", + "all" + ], + "type": "string" + }, + "type": "array" + } + }, + "title": "RockyLinuxPlatformModel", + "type": "object" + }, "SLESPlatformModel": { "properties": { "name": { @@ -1038,7 +1109,6 @@ "artful", "bionic", "cosmic", - "cuttlefish", "disco", "eoan", "focal", @@ -1046,7 +1116,11 @@ "hirsute", "impish", "jammy", + "kinetic", "lucid", + "lunar", + "mantic", + "noble", "maverick", "natty", "oneiric", @@ -1199,6 +1273,8 @@ "Mojave", "Monterey", "Sierra", + "Sonoma", + "Ventura", "all" ], "type": "string" @@ -1372,6 +1448,9 @@ "$ref": "#/$defs/PAN-OSPlatformModel" }, { + "$ref": "#/$defs/RockyLinuxPlatformModel" + }, + { "$ref": "#/$defs/SLESPlatformModel" }, { @@ -1447,6 +1526,7 @@ }, "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/meta.json", "$schema": "http://json-schema.org/draft-07/schema", + "description": "https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_reuse_roles.html#using-role-dependencies", "examples": ["meta/main.yml"], "properties": { "additionalProperties": false, @@ -1459,7 +1539,14 @@ }, "dependencies": { "items": { - "$ref": "#/$defs/DependencyModel" + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/DependencyModel" + } + ] }, "title": "Dependencies", "type": "array" diff --git a/src/ansiblelint/schemas/molecule.json b/src/ansiblelint/schemas/molecule.json index d957f08..21f1610 100644 --- a/src/ansiblelint/schemas/molecule.json +++ b/src/ansiblelint/schemas/molecule.json @@ -512,6 +512,7 @@ "$id": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/molecule.json", "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, + "description": "https://ansible.readthedocs.io/projects/molecule/configuration/", "examples": ["molecule/*/molecule.yml"], "properties": { "dependency": { diff --git a/src/ansiblelint/schemas/playbook.json b/src/ansiblelint/schemas/playbook.json index 983033f..f4d315b 100644 --- a/src/ansiblelint/schemas/playbook.json +++ b/src/ansiblelint/schemas/playbook.json @@ -171,8 +171,8 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" }, "module_defaults": { "title": "Module Defaults" @@ -363,8 +363,8 @@ "type": "boolean" }, "gather_facts": { - "title": "Gather Facts", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Gather Facts" }, "gather_subset": { "items": { @@ -537,12 +537,12 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" }, "max_fail_percentage": { - "title": "Max Fail Percentage", - "type": "number" + "$ref": "#/$defs/templated-integer", + "title": "Max Fail Percentage" }, "module_defaults": { "title": "Module Defaults" @@ -555,15 +555,23 @@ "$ref": "#/$defs/templated-boolean" }, "order": { - "enum": [ - "default", - "sorted", - "reverse_sorted", - "reverse_inventory", - "shuffle" + "markdownDescription": "See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#ordering-execution-based-on-inventory", + "oneOf": [ + { + "enum": [ + "inventory", + "reverse_inventory", + "reverse_sorted", + "shuffle", + "sorted" + ], + "type": "string" + }, + { + "$ref": "#/$defs/full-jinja" + } ], - "title": "Order", - "type": "string" + "title": "Order" }, "port": { "$ref": "#/$defs/templated-integer", @@ -740,8 +748,8 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" }, "module_defaults": { "title": "Module Defaults" @@ -796,6 +804,11 @@ "title": "play-role", "type": "object" }, + "removed-include-module": { + "markdownDescription": "See [include module](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_module.html)", + "not": {}, + "title": "Replace 'include' with either 'ansible.builtin.include_tasks' or 'ansible.builtin.import_tasks'" + }, "tags": { "anyOf": [ { @@ -847,6 +860,12 @@ "title": "Action", "type": "string" }, + "ansible.builtin.include": { + "$ref": "#/$defs/removed-include-module" + }, + "ansible.legacy.include": { + "$ref": "#/$defs/removed-include-module" + }, "any_errors_fatal": { "$ref": "#/$defs/templated-boolean", "title": "Any Errors Fatal" @@ -929,8 +948,11 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" + }, + "include": { + "$ref": "#/$defs/removed-include-module" }, "listen": { "anyOf": [ diff --git a/src/ansiblelint/schemas/requirements.json b/src/ansiblelint/schemas/requirements.json index dc7ded6..ef8d2a4 100644 --- a/src/ansiblelint/schemas/requirements.json +++ b/src/ansiblelint/schemas/requirements.json @@ -130,6 +130,7 @@ "$ref": "#/$defs/RequirementsV2Model" } ], + "description": "https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-roles-and-collections-from-the-same-requirements-yml-file", "examples": ["requirements.yml"], "title": "Ansible Requirements Schema" } diff --git a/src/ansiblelint/schemas/role-arg-spec.json b/src/ansiblelint/schemas/role-arg-spec.json index 433993e..111fbe5 100644 --- a/src/ansiblelint/schemas/role-arg-spec.json +++ b/src/ansiblelint/schemas/role-arg-spec.json @@ -1,5 +1,73 @@ { "$defs": { + "attribute": { + "additionalProperties": false, + "properties": { + "description": { + "description": "Detailed explanation of what this attribute does. It should be written in full sentences.", + "oneOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "details": { + "description": "Detailed explanation of what this attribute does. It should be written in full sentences.", + "oneOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "membership": { + "oneOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "platform": { + "oneOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "support": { + "enum": ["full", "partial", "none", "N/A"], + "type": "string" + }, + "version_added": { + "type": "string" + } + }, + "required": ["description", "support"], + "title": "Attribute" + }, "datatype": { "enum": [ "str", @@ -38,6 +106,12 @@ "entry_point": { "additionalProperties": false, "properties": { + "attributes": { + "additionalProperties": { + "$ref": "#/$defs/attribute" + }, + "type": "object" + }, "author": { "oneOf": [ { @@ -64,6 +138,9 @@ } ] }, + "examples": { + "type": "string" + }, "options": { "additionalProperties": { "$ref": "#/$defs/option" @@ -146,30 +223,27 @@ "title": "Entry Point", "type": "object" }, + "full-jinja": { + "pattern": "^\\{[\\{%](.|[\r\n])*[\\}%]\\}$", + "type": "string" + }, "option": { "additionalProperties": false, - "aliases": { - "items": { + "markdownDescription": "See [argument-spec](https://docs.ansible.com/ansible/latest/dev_guide/developing_program_flow_modules.html#argument-spec)", + "properties": { + "apply_defaults": { "type": "string" }, - "type": "array" - }, - "apply_defaults": { - "type": "string" - }, - "deprecated_aliases": { - "items": { - "$ref": "#/$defs/deprecated_alias" - }, - "type": "array" - }, - "markdownDescription": "xxx", - "options": { - "$ref": "#/$defs/option" - }, - "properties": { "choices": { - "type": "array" + "oneOf": [ + { + "type": "array" + }, + { + "$ref": "#/$defs/full-jinja", + "type": "string" + } + ] }, "default": { "default": "None" @@ -213,6 +287,70 @@ "default": false, "type": "boolean" }, + "mutually_exclusive": { + "type": "array", + "items": { + "items": { + "type": "string" + } + } + }, + "required_together": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required_one_of": { + "type": "array", + "items": { + "items": { + "type": "string" + } + } + }, + "required_if": { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + {}, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "boolean" + } + ], + "minItems": 3, + "maxItems": 4 + } + }, + "required_by": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, "type": { "$ref": "#/$defs/datatype", "markdownDescription": "See [argument-spec](https://docs.ansible.com/ansible/latest/dev_guide/developing_program_flow_modules.html#argument-spec" @@ -237,7 +375,7 @@ "$schema": "http://json-schema.org/draft-07/schema", "additionalProperties": false, "examples": ["meta/argument_specs.yml"], - "markdownDescription": "Add entry point, usually `main`.\nSee [role-argument-validation](https://docs.ansible.com/ansible/latest/user_guide/playbooks_reuse_roles.html#role-argument-validation)", + "markdownDescription": "See [role-argument-validation](https://docs.ansible.com/ansible/latest/user_guide/playbooks_reuse_roles.html#role-argument-validation)", "properties": { "argument_specs": { "additionalProperties": { diff --git a/src/ansiblelint/schemas/rulebook.json b/src/ansiblelint/schemas/rulebook.json index 6c441cd..6321f08 100644 --- a/src/ansiblelint/schemas/rulebook.json +++ b/src/ansiblelint/schemas/rulebook.json @@ -1,6 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/ansible/ansible-rulebook/main/ansible_rulebook/schema/ruleset_schema.json", + "title": "Ansible Rulebook", + "description": "See https://ansible.readthedocs.io/projects/rulebook/en/stable/rulebooks.html", "type": "array", "items": { "$ref": "#/$defs/ruleset" @@ -25,12 +27,19 @@ "type": "boolean", "default": false }, + "match_multiple_rules": { + "type": "boolean", + "default": false + }, "name": { "type": "string" }, "execution_strategy": { "type": "string", - "enum": ["sequential", "parallel"], + "enum": [ + "parallel", + "sequential" + ], "default": "sequential" }, "sources": { @@ -47,6 +56,7 @@ } }, "required": [ + "name", "hosts", "sources", "rules" @@ -147,6 +157,9 @@ "type": "string" }, { + "type": "boolean" + }, + { "$ref": "#/$defs/all-condition" }, { @@ -171,6 +184,9 @@ "$ref": "#/$defs/run-job-template-action" }, { + "$ref": "#/$defs/run-workflow-template-action" + }, + { "$ref": "#/$defs/post-event-action" }, { @@ -190,6 +206,9 @@ }, { "$ref": "#/$defs/shutdown-action" + }, + { + "$ref": "#/$defs/pg-notify-action" } ] } @@ -206,6 +225,9 @@ "$ref": "#/$defs/run-job-template-action" }, { + "$ref": "#/$defs/run-workflow-template-action" + }, + { "$ref": "#/$defs/post-event-action" }, { @@ -225,6 +247,9 @@ }, { "$ref": "#/$defs/shutdown-action" + }, + { + "$ref": "#/$defs/pg-notify-action" } ] } @@ -342,9 +367,6 @@ "run_module": { "type": "object", "properties": { - "copy_files": { - "type": "boolean" - }, "name": { "type": "string" }, @@ -376,7 +398,10 @@ "type": "number" }, "module_args": { - "type": "object" + "type": [ + "object", + "string" + ] }, "extra_vars": { "type": "object" @@ -442,6 +467,91 @@ ], "additionalProperties": false }, + "run-workflow-template-action": { + "type": "object", + "properties": { + "run_workflow_template": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "job_args": { + "type": "object" + }, + "post_events": { + "type": "boolean" + }, + "set_facts": { + "type": "boolean" + }, + "ruleset": { + "type": "string" + }, + "var_root": { + "type": "string" + }, + "retry": { + "type": "boolean" + }, + "retries": { + "type": "integer" + }, + "delay": { + "type": "integer" + } + }, + "required": [ + "name", + "organization" + ], + "additionalProperties": false + } + }, + "required": [ + "run_workflow_template" + ], + "additionalProperties": false + }, + "pg-notify-action": { + "type": "object", + "properties": { + "pg_notify": { + "type": "object", + "properties": { + "dsn": { + "type": "string" + }, + "channel": { + "type": "string" + }, + "event": { + "type": [ + "string", + "object" + ] + }, + "remove_meta": { + "type": "boolean", + "default": false + } + }, + "required": [ + "dsn", + "channel", + "event" + ], + "additionalProperties": false + } + }, + "required": [ + "pg_notify" + ], + "additionalProperties": false + }, "post-event-action": { "type": "object", "properties": { diff --git a/src/ansiblelint/schemas/tasks.json b/src/ansiblelint/schemas/tasks.json index ec7f85d..d6efec8 100644 --- a/src/ansiblelint/schemas/tasks.json +++ b/src/ansiblelint/schemas/tasks.json @@ -123,8 +123,8 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" }, "module_defaults": { "title": "Module Defaults" @@ -238,6 +238,11 @@ "markdownDescription": "Use for protecting sensitive data. See [no_log](https://docs.ansible.com/ansible/latest/reference_appendices/logging.html)", "title": "no_log" }, + "removed-include-module": { + "markdownDescription": "See [include module](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_module.html)", + "not": {}, + "title": "Replace 'include' with either 'ansible.builtin.include_tasks' or 'ansible.builtin.import_tasks'" + }, "tags": { "anyOf": [ { @@ -289,6 +294,12 @@ "title": "Action", "type": "string" }, + "ansible.builtin.include": { + "$ref": "#/$defs/removed-include-module" + }, + "ansible.legacy.include": { + "$ref": "#/$defs/removed-include-module" + }, "any_errors_fatal": { "$ref": "#/$defs/templated-boolean", "title": "Any Errors Fatal" @@ -371,8 +382,11 @@ "$ref": "#/$defs/ignore_errors" }, "ignore_unreachable": { - "title": "Ignore Unreachable", - "type": "boolean" + "$ref": "#/$defs/templated-boolean", + "title": "Ignore Unreachable" + }, + "include": { + "$ref": "#/$defs/removed-include-module" }, "listen": { "anyOf": [ diff --git a/src/ansiblelint/schemas/vars.json b/src/ansiblelint/schemas/vars.json index c0b66e8..44acb10 100644 --- a/src/ansiblelint/schemas/vars.json +++ b/src/ansiblelint/schemas/vars.json @@ -17,6 +17,7 @@ "type": "null" } ], + "description": "https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html", "examples": [ "playbooks/vars/*.yml", "vars/*.yml", diff --git a/src/ansiblelint/skip_utils.py b/src/ansiblelint/skip_utils.py index f2f6177..e1a3a8f 100644 --- a/src/ansiblelint/skip_utils.py +++ b/src/ansiblelint/skip_utils.py @@ -199,7 +199,7 @@ def _append_skipped_rules( ruamel_tasks = _get_tasks_from_blocks(ruamel_task_blocks) # append skipped_rules for each task - for ruamel_task, pyyaml_task in zip(ruamel_tasks, pyyaml_tasks): + for ruamel_task, pyyaml_task in zip(ruamel_tasks, pyyaml_tasks, strict=False): # ignore empty tasks if not pyyaml_task and not ruamel_task: continue @@ -240,7 +240,7 @@ def _get_tasks_from_blocks(task_blocks: Sequence[Any]) -> Generator[Any, None, N if not task or not is_nested_task(task): return for k in NESTED_TASK_KEYS: - if k in task and task[k]: + if task.get(k): if hasattr(task[k], "get"): continue for subtask in task[k]: @@ -279,16 +279,14 @@ def _get_rule_skips_from_yaml( yaml_comment_obj_strings.append(str(obj.ca.items)) if isinstance(obj, dict): for val in obj.values(): - if isinstance(val, (dict, list)): + if isinstance(val, dict | list): traverse_yaml(val) elif isinstance(obj, list): for element in obj: - if isinstance(element, (dict, list)): + if isinstance(element, dict | list): traverse_yaml(element) - else: - return - if isinstance(yaml_input, (dict, list)): + if isinstance(yaml_input, dict | list): traverse_yaml(yaml_input) rule_id_list = [] diff --git a/src/ansiblelint/stats.py b/src/ansiblelint/stats.py index 67320b8..79475d2 100644 --- a/src/ansiblelint/stats.py +++ b/src/ansiblelint/stats.py @@ -1,4 +1,5 @@ """Module hosting functionality about reporting.""" + from __future__ import annotations from dataclasses import dataclass, field diff --git a/src/ansiblelint/testing/__init__.py b/src/ansiblelint/testing/__init__.py index e7f6c1b..9c5463f 100644 --- a/src/ansiblelint/testing/__init__.py +++ b/src/ansiblelint/testing/__init__.py @@ -1,4 +1,5 @@ """Test utils for ansible-lint.""" + from __future__ import annotations import os @@ -13,7 +14,6 @@ from ansiblelint.app import get_app if TYPE_CHECKING: # https://github.com/PyCQA/pylint/issues/3240 - # pylint: disable=unsubscriptable-object CompletedProcess = subprocess.CompletedProcess[Any] from ansiblelint.errors import MatchError from ansiblelint.rules import RulesCollection @@ -156,4 +156,5 @@ def run_ansible_lint( cwd=cwd, env=_env, text=True, + encoding="utf-8", ) diff --git a/src/ansiblelint/testing/fixtures.py b/src/ansiblelint/testing/fixtures.py index 814a076..05e1ad7 100644 --- a/src/ansiblelint/testing/fixtures.py +++ b/src/ansiblelint/testing/fixtures.py @@ -5,21 +5,19 @@ file: pytest_plugins = ['ansiblelint.testing'] """ + from __future__ import annotations -import copy from typing import TYPE_CHECKING import pytest -from ansiblelint.config import Options, options +from ansiblelint.config import Options from ansiblelint.constants import DEFAULT_RULESDIR from ansiblelint.rules import RulesCollection from ansiblelint.testing import RunFromText if TYPE_CHECKING: - from collections.abc import Iterator - from _pytest.fixtures import SubRequest @@ -29,13 +27,12 @@ if TYPE_CHECKING: def fixture_default_rules_collection() -> RulesCollection: """Return default rule collection.""" assert DEFAULT_RULESDIR.is_dir() - # For testing we want to manually enable opt-in rules - test_options = copy.deepcopy(options) - test_options.enable_list = ["no-same-owner"] + config_options = Options() + config_options.enable_list = ["no-same-owner"] # That is instantiated very often and do want to avoid ansible-galaxy # install errors due to concurrency. - test_options.offline = True - return RulesCollection(rulesdirs=[DEFAULT_RULESDIR], options=test_options) + config_options.offline = True + return RulesCollection(rulesdirs=[DEFAULT_RULESDIR], options=config_options) @pytest.fixture() @@ -45,9 +42,10 @@ def default_text_runner(default_rules_collection: RulesCollection) -> RunFromTex @pytest.fixture() -def rule_runner(request: SubRequest, config_options: Options) -> RunFromText: +def rule_runner(request: SubRequest) -> RunFromText: """Return runner for a specific rule class.""" rule_class = request.param + config_options = Options() config_options.enable_list.append(rule_class().id) collection = RulesCollection(options=config_options) collection.register(rule_class()) @@ -55,9 +53,6 @@ def rule_runner(request: SubRequest, config_options: Options) -> RunFromText: @pytest.fixture(name="config_options") -def fixture_config_options() -> Iterator[Options]: +def fixture_config_options() -> Options: """Return configuration options that will be restored after testrun.""" - global options # pylint: disable=global-statement,invalid-name # noqa: PLW0603 - original_options = copy.deepcopy(options) - yield options - options = original_options + return Options() diff --git a/src/ansiblelint/text.py b/src/ansiblelint/text.py index 038fde1..3510f75 100644 --- a/src/ansiblelint/text.py +++ b/src/ansiblelint/text.py @@ -1,4 +1,5 @@ """Text utils.""" + from __future__ import annotations import re @@ -6,6 +7,7 @@ from functools import cache RE_HAS_JINJA = re.compile(r"{[{%#].*[%#}]}", re.DOTALL) RE_HAS_GLOB = re.compile("[][*?]") +RE_IS_FQCN_OR_NAME = re.compile(r"^\w+(\.\w+\.\w+)?$") def strip_ansi_escape(data: str | bytes) -> str: @@ -47,3 +49,9 @@ def has_jinja(value: str) -> bool: def has_glob(value: str) -> bool: """Return true if a string looks like having a glob pattern.""" return bool(isinstance(value, str) and RE_HAS_GLOB.search(value)) + + +@cache +def is_fqcn_or_name(value: str) -> bool: + """Return true if a string seems to be a module/filter old name or a fully qualified one.""" + return bool(isinstance(value, str) and RE_IS_FQCN_OR_NAME.search(value)) diff --git a/src/ansiblelint/transformer.py b/src/ansiblelint/transformer.py index 3716ef9..c610704 100644 --- a/src/ansiblelint/transformer.py +++ b/src/ansiblelint/transformer.py @@ -1,8 +1,10 @@ +# cspell:ignore classinfo """Transformer implementation.""" + from __future__ import annotations import logging -from typing import TYPE_CHECKING, Union, cast +from typing import TYPE_CHECKING, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -20,7 +22,6 @@ __all__ = ["Transformer"] _logger = logging.getLogger(__name__) -# pylint: disable=too-few-public-methods class Transformer: """Transformer class marshals transformations. @@ -33,6 +34,17 @@ class Transformer: pre-requisite for the planned rule-specific transforms. """ + DUMP_MSG = "Rewriting yaml file:" + FIX_NA_MSG = "Rule specific fix not available for:" + FIX_NE_MSG = "Rule specific fix not enabled for:" + FIX_APPLY_MSG = "Applying rule specific fix for:" + FIX_FAILED_MSG = "Rule specific fix failed for:" + FIX_ISSUE_MSG = ( + "Please file an issue for this with the task or playbook that caused the error." + ) + FIX_APPLIED_MSG = "Rule specific fix applied for:" + FIX_NOT_APPLIED_MSG = "Rule specific fix not applied for:" + def __init__(self, result: LintResult, options: Options): """Initialize a Transformer instance.""" self.write_set = self.effective_write_set(options.write_list) @@ -44,8 +56,8 @@ class Transformer: self.matches_per_file: dict[Lintable, list[MatchError]] = { file: [] for file in result.files } - - for match in self.matches: + not_ignored = [match for match in self.matches if not match.ignored] + for match in not_ignored: try: lintable = lintables[match.filename] except KeyError: @@ -93,10 +105,13 @@ class Transformer: # We need a fresh YAML() instance for each load because ruamel.yaml # stores intermediate state during load which could affect loading # any other files. (Based on suggestion from ruamel.yaml author) - yaml = FormattedYAML() + yaml = FormattedYAML( + # Ansible only uses YAML 1.1, but others files should use newer 1.2 (ruamel.yaml defaults to 1.2) + version=(1, 1) if file.is_owned_by_ansible() else None, + ) - ruamel_data = yaml.loads(data) - if not isinstance(ruamel_data, (CommentedMap, CommentedSeq)): + ruamel_data = yaml.load(data) + if not isinstance(ruamel_data, CommentedMap | CommentedSeq): # This is an empty vars file or similar which loads as None. # It is not safe to write this file or data-loss is likely. # Only maps and sequences can preserve comments. Skip it. @@ -110,6 +125,7 @@ class Transformer: self._do_transforms(file, ruamel_data or data, file_is_yaml, matches) if file_is_yaml: + _logger.debug("%s %s, version=%s", self.DUMP_MSG, file, yaml.version) # noinspection PyUnboundLocalVariable file.content = yaml.dumps(ruamel_data) @@ -125,17 +141,19 @@ class Transformer: ) -> None: """Do Rule-Transforms handling any last-minute MatchError inspections.""" for match in sorted(matches): + match_id = f"{match.tag}/{match.match_type} {match.filename}:{match.lineno}" if not isinstance(match.rule, TransformMixin): + logging.debug("%s %s", self.FIX_NA_MSG, match_id) continue if self.write_set != {"all"}: rule = cast(AnsibleLintRule, match.rule) rule_definition = set(rule.tags) rule_definition.add(rule.id) if rule_definition.isdisjoint(self.write_set): - # rule transform not requested. Skip it. + logging.debug("%s %s", self.FIX_NE_MSG, match_id) continue if file_is_yaml and not match.yaml_path: - data = cast(Union[CommentedMap, CommentedSeq], data) + data = cast(CommentedMap | CommentedSeq, data) if match.match_type == "play": match.yaml_path = get_path_to_play(file, match.lineno, data) elif match.task or file.kind in ( @@ -144,4 +162,16 @@ class Transformer: "playbook", ): match.yaml_path = get_path_to_task(file, match.lineno, data) - match.rule.transform(match, file, data) + + logging.debug("%s %s", self.FIX_APPLY_MSG, match_id) + try: + match.rule.transform(match, file, data) + except Exception as exc: # pylint: disable=broad-except + _logger.error("%s %s", self.FIX_FAILED_MSG, match_id) # noqa: TRY400 + _logger.exception(exc) # noqa: TRY401 + _logger.error(self.FIX_ISSUE_MSG) # noqa: TRY400 + continue + if match.fixed: + _logger.debug("%s %s", self.FIX_APPLIED_MSG, match_id) + else: + _logger.error("%s %s", self.FIX_NOT_APPLIED_MSG, match_id) diff --git a/src/ansiblelint/utils.py b/src/ansiblelint/utils.py index 9cb97aa..3d0e535 100644 --- a/src/ansiblelint/utils.py +++ b/src/ansiblelint/utils.py @@ -22,26 +22,35 @@ """Generic utility helpers.""" from __future__ import annotations +import ast import contextlib import inspect import logging import os import re -from collections.abc import Generator, ItemsView, Iterator, Mapping, Sequence +from collections.abc import ItemsView, Iterable, Iterator, Mapping, Sequence from dataclasses import _MISSING_TYPE, dataclass, field -from functools import cache +from functools import cache, lru_cache from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any +import ruamel.yaml.parser import yaml from ansible.errors import AnsibleError, AnsibleParserError from ansible.module_utils.parsing.convert_bool import boolean from ansible.parsing.dataloader import DataLoader from ansible.parsing.mod_args import ModuleArgsParser +from ansible.parsing.plugin_docs import read_docstring +from ansible.parsing.splitter import split_args from ansible.parsing.yaml.constructor import AnsibleConstructor, AnsibleMapping from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence -from ansible.plugins.loader import add_all_plugin_dirs +from ansible.plugins.loader import ( + PluginLoadContext, + action_loader, + add_all_plugin_dirs, + module_loader, +) from ansible.template import Templar from ansible.utils.collection_loader import AnsibleCollectionConfig from yaml.composer import Composer @@ -51,7 +60,7 @@ from ansiblelint._internal.rules import ( AnsibleParserErrorRule, RuntimeErrorRule, ) -from ansiblelint.app import get_app +from ansiblelint.app import App, get_app from ansiblelint.config import Options, options from ansiblelint.constants import ( ANNOTATION_KEYS, @@ -67,8 +76,10 @@ from ansiblelint.constants import ( from ansiblelint.errors import MatchError from ansiblelint.file_utils import Lintable, discover_lintables from ansiblelint.skip_utils import is_nested_task -from ansiblelint.text import removeprefix +from ansiblelint.text import has_jinja, removeprefix +if TYPE_CHECKING: + from ansiblelint.rules import RulesCollection # ansible-lint doesn't need/want to know about encrypted secrets, so we pass a # string as the password to enable such yaml files to be opened and parsed # successfully. @@ -164,7 +175,6 @@ def ansible_template( for _i in range(10): try: templated = templar.template(varname, **kwargs) - return templated except AnsibleError as exc: if lookup_error in exc.message: return varname @@ -186,16 +196,14 @@ def ansible_template( _logger.warning(err) raise - # pylint: disable=protected-access - templar.environment.filters._delegatee[ # noqa: SLF001 - missing_filter - ] = mock_filter + templar.environment.filters._delegatee[missing_filter] = mock_filter # fmt: skip # noqa: SLF001 # Record the mocked filter so we can warn the user if missing_filter not in options.mock_filters: _logger.debug("Mocking missing filter %s", missing_filter) options.mock_filters.append(missing_filter) continue raise + return templated return None @@ -210,26 +218,23 @@ BLOCK_NAME_TO_ACTION_TYPE_MAP = { } -def tokenize(line: str) -> tuple[str, list[str], dict[str, str]]: +def tokenize(value: str) -> tuple[list[str], dict[str, str]]: """Parse a string task invocation.""" - tokens = line.lstrip().split(" ") - if tokens[0] == "-": - tokens = tokens[1:] - if tokens[0] == "action:" or tokens[0] == "local_action:": - tokens = tokens[1:] - command = tokens[0].replace(":", "") - - args = [] - kwargs = {} - non_kv_found = False - for arg in tokens[1:]: - if "=" in arg and not non_kv_found: - key_value = arg.split("=", 1) - kwargs[key_value[0]] = key_value[1] + # We do not try to tokenize something very simple because it would fail to + # work for a case like: task_include: path with space.yml + if value and "=" not in value: + return ([value], {}) + + parts = split_args(value) + args: list[str] = [] + kwargs: dict[str, str] = {} + for part in parts: + if "=" not in part: + args.append(part) else: - non_kv_found = True - args.append(arg) - return (command, args, kwargs) + k, v = part.split("=", 1) + kwargs[k] = v + return (args, kwargs) def playbook_items(pb_data: AnsibleBaseYAMLObject) -> ItemsView: # type: ignore[type-arg] @@ -278,106 +283,179 @@ def template( return value -def _include_children( - basedir: str, - k: str, - v: Any, - parent_type: FileType, -) -> list[Lintable]: - # handle special case include_tasks: name=filename.yml - if k in INCLUSION_ACTION_NAMES and isinstance(v, dict) and "file" in v: - v = v["file"] - - # we cannot really parse any jinja2 in includes, so we ignore them - if not v or "{{" in v: - return [] - - if "import_playbook" in k and COLLECTION_PLAY_RE.match(v): - # Any import_playbooks from collections should be ignored as ansible - # own syntax check will handle them. - return [] - - # handle include: filename.yml tags=blah - # pylint: disable=unused-variable - (command, args, kwargs) = tokenize(f"{k}: {v}") - - result = path_dwim(basedir, args[0]) - while basedir not in ["", "/"]: - if os.path.exists(result): - break - basedir = os.path.dirname(basedir) - result = path_dwim(basedir, args[0]) - - return [Lintable(result, kind=parent_type)] - - -def _taskshandlers_children( - basedir: str, - k: str, - v: None | Any, - parent_type: FileType, -) -> list[Lintable]: - results: list[Lintable] = [] - if v is None: - raise MatchError( - message="A malformed block was encountered while loading a block.", - rule=RuntimeErrorRule(), - ) - for task_handler in v: - # ignore empty tasks, `-` - if not task_handler: - continue +@dataclass +class HandleChildren: + """Parse task, roles and children.""" + + rules: RulesCollection = field(init=True, repr=False) + app: App + + def include_children( # pylint: disable=too-many-return-statements + self, + lintable: Lintable, + k: str, + v: Any, + parent_type: FileType, + ) -> list[Lintable]: + """Include children.""" + basedir = str(lintable.path.parent) + # import_playbook only accepts a string as argument (no dict syntax) + if k in ( + "import_playbook", + "ansible.builtin.import_playbook", + ) and not isinstance(v, str): + return [] + + # handle special case include_tasks: name=filename.yml + if k in INCLUSION_ACTION_NAMES and isinstance(v, dict) and "file" in v: + v = v["file"] + + # we cannot really parse any jinja2 in includes, so we ignore them + if not v or "{{" in v: + return [] + + if k in ("import_playbook", "ansible.builtin.import_playbook"): + included = Path(basedir) / v + if self.app.runtime.has_playbook(v, basedir=Path(basedir)): + if included.exists(): + return [Lintable(included, kind=parent_type)] + return [] + msg = f"Failed to find {v} playbook." + logging.error(msg) + return [] + + # handle include: filename.yml tags=blah + (args, kwargs) = tokenize(v) + + if args: + file = args[0] + elif "file" in kwargs: + file = kwargs["file"] + else: + return [] - with contextlib.suppress(LookupError): - children = _get_task_handler_children_for_tasks_or_playbooks( - task_handler, - basedir, - k, - parent_type, + result = path_dwim(basedir, file) + while basedir not in ["", "/"]: + if os.path.exists(result): + break + basedir = os.path.dirname(basedir) + result = path_dwim(basedir, file) + + return [Lintable(result, kind=parent_type)] + + def taskshandlers_children( + self, + lintable: Lintable, + k: str, + v: None | Any, + parent_type: FileType, + ) -> list[Lintable]: + """TasksHandlers Children.""" + basedir = str(lintable.path.parent) + results: list[Lintable] = [] + if v is None or isinstance(v, int | str): + raise MatchError( + message="A malformed block was encountered while loading a block.", + rule=RuntimeErrorRule(), ) - results.append(children) - continue + for task_handler in v: + # ignore empty tasks, `-` + if not task_handler: + continue - if any(x in task_handler for x in ROLE_IMPORT_ACTION_NAMES): - task_handler = normalize_task_v2(task_handler) - _validate_task_handler_action_for_role(task_handler["action"]) - results.extend( - _roles_children( + with contextlib.suppress(LookupError): + children = _get_task_handler_children_for_tasks_or_playbooks( + task_handler, basedir, k, - [task_handler["action"].get("name")], parent_type, - main=task_handler["action"].get("tasks_from", "main"), - ), - ) - continue + ) + results.append(children) + continue - if "block" not in task_handler: - continue + if any(x in task_handler for x in ROLE_IMPORT_ACTION_NAMES): + task_handler = normalize_task_v2( + Task(task_handler, filename=str(lintable.path)), + ) + self._validate_task_handler_action_for_role(task_handler["action"]) + name = task_handler["action"].get("name") + if has_jinja(name): + # we cannot deal with dynamic imports + continue + results.extend( + self.roles_children(lintable, k, [name], parent_type), + ) + continue - results.extend( - _taskshandlers_children(basedir, k, task_handler["block"], parent_type), - ) - if "rescue" in task_handler: - results.extend( - _taskshandlers_children( - basedir, - k, - task_handler["rescue"], - parent_type, + if "block" not in task_handler: + continue + + for elem in ("block", "rescue", "always"): + if elem in task_handler: + results.extend( + self.taskshandlers_children( + lintable, + k, + task_handler[elem], + parent_type, + ), + ) + + return results + + def _validate_task_handler_action_for_role(self, th_action: dict[str, Any]) -> None: + """Verify that the task handler action is valid for role include.""" + module = th_action["__ansible_module__"] + + if "name" not in th_action: + raise MatchError( + message=f"Failed to find required 'name' key in {module!s}", + rule=self.rules.rules[0], + lintable=Lintable( + ( + self.rules.options.lintables[0] + if self.rules.options.lintables + else "." + ), ), ) - if "always" in task_handler: - results.extend( - _taskshandlers_children( - basedir, - k, - task_handler["always"], - parent_type, - ), + + if not isinstance(th_action["name"], str): + raise MatchError( + message=f"Value assigned to 'name' key on '{module!s}' is not a string.", + rule=self.rules.rules[1], ) - return results + def roles_children( + self, + lintable: Lintable, + k: str, + v: Sequence[Any], + parent_type: FileType, + ) -> list[Lintable]: + """Roles children.""" + # pylint: disable=unused-argument # parent_type) + basedir = str(lintable.path.parent) + results: list[Lintable] = [] + if not v or not isinstance(v, Iterable): + # typing does not prevent junk from being passed in + return results + for role in v: + if isinstance(role, dict): + if "role" in role or "name" in role: + if "tags" not in role or "skip_ansible_lint" not in role["tags"]: + results.extend( + _look_for_role_files( + basedir, + role.get("role", role.get("name")), + ), + ) + elif k != "dependencies": + msg = f'role dict {role} does not contain a "role" or "name" key' + raise SystemExit(msg) + else: + results.extend(_look_for_role_files(basedir, role)) + return results def _get_task_handler_children_for_tasks_or_playbooks( @@ -393,13 +471,27 @@ def _get_task_handler_children_for_tasks_or_playbooks( for task_handler_key in INCLUSION_ACTION_NAMES: with contextlib.suppress(KeyError): # ignore empty tasks - if not task_handler: # pragma: no branch + if not task_handler or isinstance(task_handler, str): # pragma: no branch continue - file_name = task_handler[task_handler_key] - if isinstance(file_name, Mapping) and file_name.get("file", None): - file_name = file_name["file"] + file_name = "" + action_args = task_handler[task_handler_key] + if isinstance(action_args, str): + (args, kwargs) = tokenize(action_args) + if len(args) == 1: + file_name = args[0] + elif kwargs.get("file", None): + file_name = kwargs["file"] + else: + # ignore invalid data (syntax check will outside the scope) + continue + + if isinstance(action_args, Mapping) and action_args.get("file", None): + file_name = action_args["file"] + if not file_name: + # ignore invalid data (syntax check will outside the scope) + continue f = path_dwim(basedir, file_name) while basedir not in ["", "/"]: if os.path.exists(f): @@ -411,50 +503,6 @@ def _get_task_handler_children_for_tasks_or_playbooks( raise LookupError(msg) -def _validate_task_handler_action_for_role(th_action: dict[str, Any]) -> None: - """Verify that the task handler action is valid for role include.""" - module = th_action["__ansible_module__"] - - if "name" not in th_action: - raise MatchError(message=f"Failed to find required 'name' key in {module!s}") - - if not isinstance(th_action["name"], str): - raise MatchError( - message=f"Value assigned to 'name' key on '{module!s}' is not a string.", - ) - - -def _roles_children( - basedir: str, - k: str, - v: Sequence[Any], - parent_type: FileType, # noqa: ARG001 - main: str = "main", -) -> list[Lintable]: - # pylint: disable=unused-argument # parent_type) - results: list[Lintable] = [] - if not v: - # typing does not prevent junk from being passed in - return results - for role in v: - if isinstance(role, dict): - if "role" in role or "name" in role: - if "tags" not in role or "skip_ansible_lint" not in role["tags"]: - results.extend( - _look_for_role_files( - basedir, - role.get("role", role.get("name")), - main=main, - ), - ) - elif k != "dependencies": - msg = f'role dict {role} does not contain a "role" or "name" key' - raise SystemExit(msg) - else: - results.extend(_look_for_role_files(basedir, role, main=main)) - return results - - def _rolepath(basedir: str, role: str) -> str | None: role_path = None @@ -469,7 +517,7 @@ def _rolepath(basedir: str, role: str) -> str | None: path_dwim(basedir, os.path.join("..", role)), ] - for loc in get_app(offline=True).runtime.config.default_roles_path: + for loc in get_app(cached=True).runtime.config.default_roles_path: loc = os.path.expanduser(loc) possible_paths.append(path_dwim(loc, role)) @@ -486,12 +534,7 @@ def _rolepath(basedir: str, role: str) -> str | None: return role_path -def _look_for_role_files( - basedir: str, - role: str, - main: str | None = "main", # noqa: ARG001 -) -> list[Lintable]: - # pylint: disable=unused-argument # main +def _look_for_role_files(basedir: str, role: str) -> list[Lintable]: role_path = _rolepath(basedir, role) if not role_path: # pragma: no branch return [] @@ -539,13 +582,14 @@ def _extract_ansible_parsed_keys_from_task( return result -def normalize_task_v2(task: dict[str, Any]) -> dict[str, Any]: +def normalize_task_v2(task: Task) -> dict[str, Any]: """Ensure tasks have a normalized action key and strings are converted to python objects.""" + raw_task = task.raw_task result: dict[str, Any] = {} ansible_parsed_keys = ("action", "local_action", "args", "delegate_to") - if is_nested_task(task): - _extract_ansible_parsed_keys_from_task(result, task, ansible_parsed_keys) + if is_nested_task(raw_task): + _extract_ansible_parsed_keys_from_task(result, raw_task, ansible_parsed_keys) # Add dummy action for block/always/rescue statements result["action"] = { "__ansible_module__": "block/always/rescue", @@ -554,7 +598,7 @@ def normalize_task_v2(task: dict[str, Any]) -> dict[str, Any]: return result - sanitized_task = _sanitize_task(task) + sanitized_task = _sanitize_task(raw_task) mod_arg_parser = ModuleArgsParser(sanitized_task) try: @@ -562,12 +606,11 @@ def normalize_task_v2(task: dict[str, Any]) -> dict[str, Any]: skip_action_validation=options.skip_action_validation, ) except AnsibleParserError as exc: - # pylint: disable=raise-missing-from raise MatchError( rule=AnsibleParserErrorRule(), message=exc.message, - filename=task.get(FILENAME_KEY, "Unknown"), - lineno=task.get(LINE_NUMBER_KEY, 0), + lintable=Lintable(task.filename or ""), + lineno=raw_task.get(LINE_NUMBER_KEY, 1), ) from exc # denormalize shell -> command conversion @@ -577,13 +620,13 @@ def normalize_task_v2(task: dict[str, Any]) -> dict[str, Any]: _extract_ansible_parsed_keys_from_task( result, - task, + raw_task, (*ansible_parsed_keys, action), ) if not isinstance(action, str): msg = f"Task actions can only be strings, got {action}" - raise RuntimeError(msg) + raise TypeError(msg) action_unnormalized = action # convert builtin fqn calls to short forms because most rules know only # about short calls but in the future we may switch the normalization to do @@ -599,17 +642,6 @@ def normalize_task_v2(task: dict[str, Any]) -> dict[str, Any]: return result -def normalize_task(task: dict[str, Any], filename: str) -> dict[str, Any]: - """Unify task-like object structures.""" - ansible_action_type = task.get("__ansible_action_type__", "task") - if "__ansible_action_type__" in task: - del task["__ansible_action_type__"] - task = normalize_task_v2(task) - task[FILENAME_KEY] = filename - task["__ansible_action_type__"] = ansible_action_type - return task - - def task_to_str(task: dict[str, Any]) -> str: """Make a string identifier for the given task.""" name = task.get("name") @@ -634,7 +666,7 @@ def task_to_str(task: dict[str, Any]) -> str: _raw_params = action.get("_raw_params", []) if isinstance(_raw_params, list): for item in _raw_params: - args.append(str(item)) + args.extend(str(item)) else: args.append(_raw_params) @@ -698,7 +730,11 @@ class Task(dict[str, Any]): @property def name(self) -> str | None: """Return the name of the task.""" - return self.raw_task.get("name", None) + name = self.raw_task.get("name", None) + if name is not None and not isinstance(name, str): + msg = "Task name can only be a string." + raise RuntimeError(msg) + return name @property def action(self) -> str: @@ -706,7 +742,7 @@ class Task(dict[str, Any]): action_name = self.normalized_task["action"]["__ansible_module_original__"] if not isinstance(action_name, str): msg = "Task actions can only be strings." - raise RuntimeError(msg) + raise TypeError(msg) return action_name @property @@ -729,10 +765,7 @@ class Task(dict[str, Any]): """Return the name of the task.""" if not hasattr(self, "_normalized_task"): try: - self._normalized_task = normalize_task( - self.raw_task, - filename=self.filename, - ) + self._normalized_task = self._normalize_task() except MatchError as err: self.error = err # When we cannot normalize it, we just use the raw task instead @@ -740,15 +773,35 @@ class Task(dict[str, Any]): self._normalized_task = self.raw_task if isinstance(self._normalized_task, _MISSING_TYPE): msg = "Task was not normalized" - raise RuntimeError(msg) + raise TypeError(msg) return self._normalized_task + def _normalize_task(self) -> dict[str, Any]: + """Unify task-like object structures.""" + ansible_action_type = self.raw_task.get("__ansible_action_type__", "task") + if "__ansible_action_type__" in self.raw_task: + del self.raw_task["__ansible_action_type__"] + task = normalize_task_v2(self) + task[FILENAME_KEY] = self.filename + task["__ansible_action_type__"] = ansible_action_type + return task + @property def skip_tags(self) -> list[str]: """Return the list of tags to skip.""" skip_tags: list[str] = self.raw_task.get(SKIPPED_RULES_KEY, []) return skip_tags + def is_handler(self) -> bool: + """Return true for tasks that are handlers.""" + is_handler_file = False + if isinstance(self._normalized_task, dict): + file_name = str(self._normalized_task["action"].get(FILENAME_KEY, None)) + if file_name: + paths = file_name.split("/") + is_handler_file = "handlers" in paths + return is_handler_file if is_handler_file else ".handlers[" in self.position + def __repr__(self) -> str: """Return a string representation of the task.""" return f"Task('{self.name}' [{self.position}])" @@ -761,7 +814,7 @@ class Task(dict[str, Any]): """Allow access as task[...].""" return self.normalized_task[index] - def __iter__(self) -> Generator[str, None, None]: + def __iter__(self) -> Iterator[str]: """Provide support for 'key in task'.""" yield from (f for f in self.normalized_task) @@ -857,7 +910,7 @@ def parse_yaml_linenumbers( node = Composer.compose_node(loader, parent, index) if not isinstance(node, yaml.nodes.Node): msg = "Unexpected yaml data." - raise RuntimeError(msg) + raise TypeError(msg) node.__line__ = line + 1 # type: ignore[attr-defined] return node @@ -870,9 +923,7 @@ def parse_yaml_linenumbers( if hasattr(node, "__line__"): mapping[LINE_NUMBER_KEY] = node.__line__ else: - mapping[ - LINE_NUMBER_KEY - ] = mapping._line_number # pylint: disable=protected-access # noqa: SLF001 + mapping[LINE_NUMBER_KEY] = mapping._line_number # noqa: SLF001 mapping[FILENAME_KEY] = lintable.path return mapping @@ -895,8 +946,9 @@ def parse_yaml_linenumbers( yaml.parser.ParserError, yaml.scanner.ScannerError, yaml.constructor.ConstructorError, + ruamel.yaml.parser.ParserError, ) as exc: - msg = "Failed to load YAML file" + msg = f"Failed to load YAML file: {lintable.path}" raise RuntimeError(msg) from exc if len(result) == 0: @@ -975,7 +1027,6 @@ def is_playbook(filename: str) -> bool: return False -# pylint: disable=too-many-statements def get_lintables( opts: Options = options, args: list[str] | None = None, @@ -1018,3 +1069,49 @@ def _extend_with_roles(lintables: list[Lintable]) -> None: def convert_to_boolean(value: Any) -> bool: """Use Ansible to convert something to a boolean.""" return bool(boolean(value)) + + +def parse_examples_from_plugin(lintable: Lintable) -> tuple[int, str]: + """Parse yaml inside plugin EXAMPLES string. + + Store a line number offset to realign returned line numbers later + """ + offset = 1 + parsed = ast.parse(lintable.content) + for child in parsed.body: + if isinstance(child, ast.Assign): + label = child.targets[0] + if isinstance(label, ast.Name) and label.id == "EXAMPLES": + offset = child.lineno - 1 + break + + docs = read_docstring(str(lintable.path)) + examples = docs["plainexamples"] + + # Ignore the leading newline and lack of document start + # as including those in EXAMPLES would be weird. + return offset, (f"---{examples}" if examples else "") + + +@lru_cache +def load_plugin(name: str) -> PluginLoadContext: + """Return loaded ansible plugin/module.""" + loaded_module = action_loader.find_plugin_with_context( + name, + ignore_deprecated=True, + check_aliases=True, + ) + if not loaded_module.resolved: + loaded_module = module_loader.find_plugin_with_context( + name, + ignore_deprecated=True, + check_aliases=True, + ) + if not loaded_module.resolved and name.startswith("ansible.builtin."): + # fallback to core behavior of using legacy + loaded_module = module_loader.find_plugin_with_context( + name.replace("ansible.builtin.", "ansible.legacy."), + ignore_deprecated=True, + check_aliases=True, + ) + return loaded_module diff --git a/src/ansiblelint/version.py b/src/ansiblelint/version.py index a65c3cf..80a0f7d 100644 --- a/src/ansiblelint/version.py +++ b/src/ansiblelint/version.py @@ -1,4 +1,5 @@ """Ansible-lint version information.""" + try: from ._version import version as __version__ except ImportError: # pragma: no cover diff --git a/src/ansiblelint/yaml_utils.py b/src/ansiblelint/yaml_utils.py index cc7e9ef..a1b963d 100644 --- a/src/ansiblelint/yaml_utils.py +++ b/src/ansiblelint/yaml_utils.py @@ -1,4 +1,5 @@ """Utility helpers to simplify working with yaml-based data.""" + # pylint: disable=too-many-lines from __future__ import annotations @@ -6,21 +7,23 @@ import functools import logging import os import re -from collections.abc import Iterator, Sequence +from collections.abc import Callable, Iterator, Sequence from io import StringIO from pathlib import Path from re import Pattern -from typing import TYPE_CHECKING, Any, Callable, Union, cast +from typing import TYPE_CHECKING, Any, cast import ruamel.yaml.events from ruamel.yaml.comments import CommentedMap, CommentedSeq, Format +from ruamel.yaml.composer import ComposerError from ruamel.yaml.constructor import RoundTripConstructor from ruamel.yaml.emitter import Emitter, ScalarAnalysis # Module 'ruamel.yaml' does not explicitly export attribute 'YAML'; implicit reexport disabled # To make the type checkers happy, we import from ruamel.yaml.main instead. from ruamel.yaml.main import YAML -from ruamel.yaml.scalarint import ScalarInt +from ruamel.yaml.parser import ParserError +from ruamel.yaml.scalarint import HexInt, ScalarInt from yamllint.config import YamlLintConfig from ansiblelint.constants import ( @@ -32,7 +35,8 @@ from ansiblelint.utils import Task if TYPE_CHECKING: # noinspection PyProtectedMember - from ruamel.yaml.comments import LineCol # pylint: disable=ungrouped-imports + from ruamel.yaml.comments import LineCol + from ruamel.yaml.compat import StreamTextType from ruamel.yaml.nodes import ScalarNode from ruamel.yaml.representer import RoundTripRepresenter from ruamel.yaml.tokens import CommentToken @@ -41,28 +45,18 @@ if TYPE_CHECKING: _logger = logging.getLogger(__name__) -YAMLLINT_CONFIG = """ -extends: default -rules: - comments: - # https://github.com/prettier/prettier/issues/6780 - min-spaces-from-content: 1 - # https://github.com/adrienverge/yamllint/issues/384 - comments-indentation: false - document-start: disable - # 160 chars was the default used by old E204 rule, but - # you can easily change it or disable in your .yamllint file. - line-length: - max: 160 - # We are adding an extra space inside braces as that's how prettier does it - # and we are trying not to fight other linters. - braces: - min-spaces-inside: 0 # yamllint defaults to 0 - max-spaces-inside: 1 # yamllint defaults to 0 - octal-values: - forbid-implicit-octal: true # yamllint defaults to false - forbid-explicit-octal: true # yamllint defaults to false -""" + +class CustomYamlLintConfig(YamlLintConfig): # type: ignore[misc] + """Extension of YamlLintConfig.""" + + def __init__( + self, + content: str | None = None, + file: str | Path | None = None, + ) -> None: + """Initialize config.""" + super().__init__(content, file) + self.incompatible = "" def deannotate(data: Any) -> Any: @@ -80,10 +74,10 @@ def deannotate(data: Any) -> Any: return data -@functools.lru_cache(maxsize=1) -def load_yamllint_config() -> YamlLintConfig: +def load_yamllint_config() -> CustomYamlLintConfig: """Load our default yamllint config and any customized override file.""" - config = YamlLintConfig(content=YAMLLINT_CONFIG) + config = CustomYamlLintConfig(file=Path(__file__).parent / "data" / ".yamllint") + config.incompatible = "" # if we detect local yamllint config we use it but raise a warning # as this is likely to get out of sync with our internal config. for path in [ @@ -100,10 +94,65 @@ def load_yamllint_config() -> YamlLintConfig: "internal yamllint config.", file, ) - config_override = YamlLintConfig(file=str(file)) - config_override.extend(config) - config = config_override + custom_config = CustomYamlLintConfig(file=str(file)) + custom_config.extend(config) + config = custom_config break + + # Look for settings incompatible with our reformatting + checks: list[tuple[str, str | int | bool]] = [ + ( + "comments.min-spaces-from-content", + 1, + ), + ( + "comments-indentation", + False, + ), + ( + "braces.min-spaces-inside", + 0, + ), + ( + "braces.max-spaces-inside", + 1, + ), + ( + "octal-values.forbid-implicit-octal", + True, + ), + ( + "octal-values.forbid-explicit-octal", + True, + ), + # ( + # "key-duplicates.forbid-duplicated-merge-keys", # v1.34.0+ + # True, + # ), + # ( + # "quoted-strings.quote-type", "double", + # ), + # ( + # "quoted-strings.required", "only-when-needed", + # ), + ] + errors = [] + for setting, expected_value in checks: + v = config.rules + for key in setting.split("."): + if not isinstance(v, dict): # pragma: no cover + break + if key not in v: # pragma: no cover + break + v = v[key] + if v != expected_value: + msg = f"{setting} must be {str(expected_value).lower()}" + errors.append(msg) + if errors: + nl = "\n" + msg = f"Found incompatible custom yamllint configuration ({file}), please either remove the file or edit it to comply with:{nl} - {(nl + ' - ').join(errors)}.{nl}{nl}Read https://ansible.readthedocs.io/projects/lint/rules/yaml/ for more details regarding why we have these requirements. Fix mode will not be available." + config.incompatible = msg + _logger.debug("Effective yamllint rules used: %s", config.rules) return config @@ -196,7 +245,7 @@ def _nested_items_path( """ # we have to cast each convert_to_tuples assignment or mypy complains # that both assignments (for dict and list) do not have the same type - convert_to_tuples_type = Callable[[], Iterator[tuple[Union[str, int], Any]]] + convert_to_tuples_type = Callable[[], Iterator[tuple[str | int, Any]]] if isinstance(data_collection, dict): convert_data_collection_to_tuples = cast( convert_to_tuples_type, @@ -214,7 +263,7 @@ def _nested_items_path( if key in (*ANNOTATION_KEYS, *ignored_keys): continue yield key, value, parent_path - if isinstance(value, (dict, list)): + if isinstance(value, dict | list): yield from _nested_items_path( data_collection=value, parent_path=[*parent_path, key], @@ -232,7 +281,7 @@ def get_path_to_play( raise ValueError(msg) if lintable.kind != "playbook" or not isinstance(ruamel_data, CommentedSeq): return [] - lc: LineCol # lc uses 0-based counts # pylint: disable=invalid-name + lc: LineCol # lc uses 0-based counts # lineno is 1-based. Convert to 0-based. line_index = lineno - 1 @@ -245,10 +294,10 @@ def get_path_to_play( else: next_play_line_index = None - lc = play.lc # pylint: disable=invalid-name + lc = play.lc if not isinstance(lc.line, int): msg = f"expected lc.line to be an int, got {lc.line!r}" - raise RuntimeError(msg) + raise TypeError(msg) if lc.line == line_index: return [play_index] if play_index > 0 and prev_play_line_index < line_index < lc.line: @@ -300,6 +349,10 @@ def _get_path_to_task_in_playbook( else: next_play_line_index = None + # We clearly haven't found the right spot yet if a following play starts on an earlier line. + if next_play_line_index and lineno > next_play_line_index: + continue + play_keys = list(play.keys()) for tasks_keyword in PLAYBOOK_TASK_KEYWORDS: if not play.get(tasks_keyword): @@ -381,7 +434,7 @@ def _get_path_to_task_in_tasks_block( if not isinstance(task.lc.line, int): msg = f"expected task.lc.line to be an int, got {task.lc.line!r}" - raise RuntimeError(msg) + raise TypeError(msg) if task.lc.line == line_index: return [task_index] if task_index > 0 and prev_task_line_index < line_index < task.lc.line: @@ -418,6 +471,8 @@ def _get_path_to_task_in_nested_tasks_block( continue next_task_key = task_keys_by_index.get(task_index + 1, None) if next_task_key is not None: + if task.lc.data[next_task_key][2] < lineno: + continue next_task_key_line_index = task.lc.data[next_task_key][0] else: next_task_key_line_index = None @@ -461,7 +516,6 @@ class OctalIntYAML11(ScalarInt): v = format(data, "o") anchor = data.yaml_anchor(any=True) # noinspection PyProtectedMember - # pylint: disable=protected-access return representer.insert_underscore( "0", v, @@ -498,7 +552,9 @@ class CustomConstructor(RoundTripConstructor): value_s = value_su.replace("_", "") if value_s[0] in "+-": value_s = value_s[1:] - if value_s[0] == "0": + if value_s[0:2] == "0x": + ret = HexInt(ret, width=len(value_s) - 2) + elif value_s[0] == "0": # got an octal in YAML 1.1 ret = OctalIntYAML11( ret, @@ -582,15 +638,33 @@ class FormattedEmitter(Emitter): """Select how to quote scalars if needed.""" style = super().choose_scalar_style() if ( - style == "" # noqa: PLC1901 + style == "" and self.event.value.startswith("0") and len(self.event.value) > 1 ): - if self.event.tag == "tag:yaml.org,2002:int" and self.event.implicit[0]: - # ensures that "0123" string does not lose its quoting + # We have an as-yet unquoted token that starts with "0" (but is not itself the digit 0). + # It could be: + # - hexadecimal like "0xF1"; comes tagged as int. Should continue unquoted to continue as an int. + # - octal like "0666" or "0o755"; comes tagged as str. **Should** be quoted to be cross-YAML compatible. + # - string like "0.0.0.0" and "00-header". Should not be quoted, unless it has a quote in it. + if ( + self.event.value.startswith("0x") + and self.event.tag == "tag:yaml.org,2002:int" + and self.event.implicit[0] + ): + # hexadecimal + self.event.tag = "tag:yaml.org,2002:str" + return "" + try: + int(self.event.value, 8) + except ValueError: + pass + # fallthrough to string + else: + # octal self.event.tag = "tag:yaml.org,2002:str" self.event.implicit = (True, True, True) - return '"' + return '"' if style != "'": # block scalar, double quoted, etc. return style @@ -598,6 +672,17 @@ class FormattedEmitter(Emitter): return "'" return self.preferred_quote + def increase_indent( + self, + flow: bool = False, # noqa: FBT002 + sequence: bool | None = None, + indentless: bool = False, # noqa: FBT002 + ) -> None: + super().increase_indent(flow, sequence, indentless) + # If our previous node was a sequence and we are still trying to indent, don't + if self.indents.last_seq(): + self.indent = self.column + 1 + def write_indicator( self, indicator: str, # ruamel.yaml typehint is wrong. This is a string. @@ -620,6 +705,9 @@ class FormattedEmitter(Emitter): and not self._in_empty_flow_map ): indicator = (" " * spaces_inside) + "}" + # Indicator sometimes comes with embedded spaces we need to squish + if indicator == " -" and self.indents.last_seq(): + indicator = "-" super().write_indicator(indicator, need_whitespace, whitespace, indention) # if it is the start of a flow mapping, and it's not time # to wrap the lines, insert a space. @@ -691,16 +779,21 @@ class FormattedEmitter(Emitter): and not value.strip() and not isinstance( self.event, - ( - ruamel.yaml.events.CollectionEndEvent, - ruamel.yaml.events.DocumentEndEvent, - ruamel.yaml.events.StreamEndEvent, - ), + ruamel.yaml.events.CollectionEndEvent + | ruamel.yaml.events.DocumentEndEvent + | ruamel.yaml.events.StreamEndEvent + | ruamel.yaml.events.MappingStartEvent, ) ): # drop pure whitespace pre comments # does not apply to End events since they consume one of the newlines. value = "" + elif ( + pre + and not value.strip() + and isinstance(self.event, ruamel.yaml.events.MappingStartEvent) + ): + value = self._re_repeat_blank_lines.sub("", value) elif pre: # preserve content in pre comment with at least one newline, # but no extra blank lines. @@ -727,13 +820,25 @@ class FormattedEmitter(Emitter): class FormattedYAML(YAML): """A YAML loader/dumper that handles ansible content better by default.""" - def __init__( + default_config = { + "explicit_start": True, + "explicit_end": False, + "width": 160, + "indent_sequences": True, + "preferred_quote": '"', + "min_spaces_inside": 0, + "max_spaces_inside": 1, + } + + def __init__( # pylint: disable=too-many-arguments self, *, typ: str | None = None, pure: bool = False, output: Any = None, plug_ins: list[str] | None = None, + version: tuple[int, int] | None = None, + config: dict[str, bool | int | str] | None = None, ): """Return a configured ``ruamel.yaml.YAML`` instance. @@ -793,15 +898,18 @@ class FormattedYAML(YAML): tasks: - name: Task """ - # Default to reading/dumping YAML 1.1 (ruamel.yaml defaults to 1.2) - self._yaml_version_default: tuple[int, int] = (1, 1) - self._yaml_version: str | tuple[int, int] = self._yaml_version_default - + if version: + if isinstance(version, str): + x, y = version.split(".", maxsplit=1) + version = (int(x), int(y)) + self._yaml_version_default: tuple[int, int] = version + self._yaml_version: tuple[int, int] = self._yaml_version_default super().__init__(typ=typ, pure=pure, output=output, plug_ins=plug_ins) # NB: We ignore some mypy issues because ruamel.yaml typehints are not great. - config = self._defaults_from_yamllint_config() + if not config: + config = self._defaults_from_yamllint_config() # these settings are derived from yamllint config self.explicit_start: bool = config["explicit_start"] # type: ignore[assignment] @@ -854,15 +962,8 @@ class FormattedYAML(YAML): @staticmethod def _defaults_from_yamllint_config() -> dict[str, bool | int | str]: """Extract FormattedYAML-relevant settings from yamllint config if possible.""" - config = { - "explicit_start": True, - "explicit_end": False, - "width": 160, - "indent_sequences": True, - "preferred_quote": '"', - "min_spaces_inside": 0, - "max_spaces_inside": 1, - } + config = FormattedYAML.default_config + for rule, rule_config in load_yamllint_config().rules.items(): if not rule_config: # rule disabled @@ -895,10 +996,10 @@ class FormattedYAML(YAML): elif quote_type == "double": config["preferred_quote"] = '"' - return cast(dict[str, Union[bool, int, str]], config) + return cast(dict[str, bool | int | str], config) - @property # type: ignore[override] - def version(self) -> str | tuple[int, int]: + @property + def version(self) -> tuple[int, int] | None: """Return the YAML version used to parse or dump. Ansible uses PyYAML which only supports YAML 1.1. ruamel.yaml defaults to 1.2. @@ -906,19 +1007,25 @@ class FormattedYAML(YAML): We can relax the version requirement once ansible uses a version of PyYAML that includes this PR: https://github.com/yaml/pyyaml/pull/555 """ - return self._yaml_version + if hasattr(self, "_yaml_version"): + return self._yaml_version + return None @version.setter - def version(self, value: str | tuple[int, int] | None) -> None: + def version(self, value: tuple[int, int] | None) -> None: """Ensure that yaml version uses our default value. The yaml Reader updates this value based on the ``%YAML`` directive in files. So, if a file does not include the directive, it sets this to None. But, None effectively resets the parsing version to YAML 1.2 (ruamel's default). """ - self._yaml_version = value if value is not None else self._yaml_version_default + if value is not None: + self._yaml_version = value + elif hasattr(self, "_yaml_version_default"): + self._yaml_version = self._yaml_version_default + # We do nothing if the object did not have a previous default version defined - def loads(self, stream: str) -> Any: + def load(self, stream: Path | StreamTextType) -> Any: """Load YAML content from a string while avoiding known ruamel.yaml issues.""" if not isinstance(stream, str): msg = f"expected a str but got {type(stream)}" @@ -928,10 +1035,18 @@ class FormattedYAML(YAML): # https://sourceforge.net/p/ruamel-yaml/tickets/460/ text, preamble_comment = self._pre_process_yaml(stream) - data = self.load(stream=text) + try: + data = super().load(stream=text) + except ComposerError: + data = self.load_all(stream=text) + except ParserError: + data = None + _logger.error( # noqa: TRY400 + "Invalid yaml, verify the file contents and try again.", + ) if preamble_comment is not None and isinstance( data, - (CommentedMap, CommentedSeq), + CommentedMap | CommentedSeq, ): data.preamble_comment = preamble_comment # type: ignore[union-attr] # Because data can validly also be None for empty documents, we cannot @@ -948,15 +1063,20 @@ class FormattedYAML(YAML): stream.write(preamble_comment) self.dump(data, stream) text = stream.getvalue() - return self._post_process_yaml(text) + strip_version_directive = hasattr(self, "_yaml_version_default") + return self._post_process_yaml( + text, + strip_version_directive=strip_version_directive, + strip_explicit_start=not self.explicit_start, + ) def _prevent_wrapping_flow_style(self, data: Any) -> None: - if not isinstance(data, (CommentedMap, CommentedSeq)): + if not isinstance(data, CommentedMap | CommentedSeq): return for key, value, parent_path in nested_items_path(data): - if not isinstance(value, (CommentedMap, CommentedSeq)): + if not isinstance(value, CommentedMap | CommentedSeq): continue - fa: Format = value.fa # pylint: disable=invalid-name + fa: Format = value.fa if fa.flow_style(): predicted_indent = self._predict_indent_length(parent_path, key) predicted_width = len(str(value)) @@ -1036,7 +1156,12 @@ class FormattedYAML(YAML): return text, "".join(preamble_comments) or None @staticmethod - def _post_process_yaml(text: str) -> str: + def _post_process_yaml( + text: str, + *, + strip_version_directive: bool = False, + strip_explicit_start: bool = False, + ) -> str: """Handle known issues with ruamel.yaml dumping. Make sure there's only one newline at the end of the file. @@ -1048,6 +1173,14 @@ class FormattedYAML(YAML): Make sure null list items don't end in a space. """ + # remove YAML directive + if strip_version_directive and text.startswith("%YAML"): + text = text.split("\n", 1)[1] + + # remove explicit document start + if strip_explicit_start and text.startswith("---"): + text = text.split("\n", 1)[1] + text = text.rstrip("\n") + "\n" lines = text.splitlines(keepends=True) @@ -1092,9 +1225,9 @@ class FormattedYAML(YAML): def clean_json( obj: Any, - func: Callable[[str], Any] = lambda key: key.startswith("__") - if isinstance(key, str) - else False, + func: Callable[[str], Any] = lambda key: ( + key.startswith("__") if isinstance(key, str) else False + ), ) -> Any: """Remove all keys matching the condition from a nested JSON-like object. |