Merging upstream version 24.6.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 85 insertions, 69 deletions

diff --git a/src/ansible_compat/config.py b/src/ansible_compat/config.py
index 6bed01b..9435540 100644
--- a/src/ansible_compat/config.py
+++ b/src/ansible_compat/config.py
@@ -8,7 +8,7 @@ import os
 import re
 import subprocess
 from collections import UserDict
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from packaging.version import Version
 
@@ -16,6 +16,9 @@ from ansible_compat.constants import ANSIBLE_MIN_VERSION
 from ansible_compat.errors import InvalidPrerequisiteError, MissingAnsibleError
 from ansible_compat.ports import cache
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 # do not use lru_cache here, as environment can change between calls
 def ansible_collections_path() -> str:
@@ -397,35 +400,49 @@ class AnsibleConfig(UserDict[str, object]):  # pylint: disable=too-many-ancestor
         self,
         config_dump: str | None = None,
         data: dict[str, object] | None = None,
+        cache_dir: Path | None = None,
     ) -> None:
         """Load config dictionary."""
         super().__init__()
 
+        self.cache_dir = cache_dir
         if data:
             self.data = copy.deepcopy(data)
-            return
-
-        if not config_dump:
-            env = os.environ.copy()
-            # Avoid possible ANSI garbage
-            env["ANSIBLE_FORCE_COLOR"] = "0"
-            config_dump = subprocess.check_output(
-                ["ansible-config", "dump"],  # noqa: S603
-                universal_newlines=True,
-                env=env,
-            )
+        else:
+            if not config_dump:
+                env = os.environ.copy()
+                # Avoid possible ANSI garbage
+                env["ANSIBLE_FORCE_COLOR"] = "0"
+                config_dump = subprocess.check_output(
+                    ["ansible-config", "dump"],  # noqa: S603
+                    universal_newlines=True,
+                    env=env,
+                )
 
-        for match in re.finditer(
-            r"^(?P<key>[A-Za-z0-9_]+).* = (?P<value>.*)$",
-            config_dump,
-            re.MULTILINE,
-        ):
-            key = match.groupdict()["key"]
-            value = match.groupdict()["value"]
-            try:
-                self[key] = ast.literal_eval(value)
-            except (NameError, SyntaxError, ValueError):
-                self[key] = value
+            for match in re.finditer(
+                r"^(?P<key>[A-Za-z0-9_]+).* = (?P<value>.*)$",
+                config_dump,
+                re.MULTILINE,
+            ):
+                key = match.groupdict()["key"]
+                value = match.groupdict()["value"]
+                try:
+                    self[key] = ast.literal_eval(value)
+                except (NameError, SyntaxError, ValueError):
+                    self[key] = value
+        # inject isolation collections paths into the config
+        if self.cache_dir:
+            cpaths = self.data["COLLECTIONS_PATHS"]
+            if cpaths and isinstance(cpaths, list):
+                cpaths.insert(
+                    0,
+                    f"{self.cache_dir}/collections",
+                )
+            else:  # pragma: no cover
+                msg = f"Unexpected data type for COLLECTIONS_PATHS: {cpaths}"
+                raise RuntimeError(msg)
+        if data:
+            return
 
     def __getattribute__(self, attr_name: str) -> object:
         """Allow access of config options as attributes."""
diff --git a/src/ansible_compat/constants.py b/src/ansible_compat/constants.py
index f3d7866..f5335ab 100644
--- a/src/ansible_compat/constants.py
+++ b/src/ansible_compat/constants.py
@@ -14,7 +14,7 @@ REQUIREMENT_LOCATIONS = [
 ]
 
 # Minimal version of Ansible we support for runtime
-ANSIBLE_MIN_VERSION = "2.12"
+ANSIBLE_MIN_VERSION = "2.14"
 
 # Based on https://docs.ansible.com/ansible/latest/reference_appendices/config.html
 ANSIBLE_DEFAULT_ROLES_PATH = (
diff --git a/src/ansible_compat/runtime.py b/src/ansible_compat/runtime.py
index fbeaa98..9ed1853 100644
--- a/src/ansible_compat/runtime.py
+++ b/src/ansible_compat/runtime.py
@@ -1,5 +1,7 @@
 """Ansible runtime environment manager."""
 
+# pylint: disable=too-many-lines
+
 from __future__ import annotations
 
 import contextlib
@@ -23,7 +25,6 @@ from packaging.version import Version
 from ansible_compat.config import (
     AnsibleConfig,
     ansible_collections_path,
-    ansible_version,
     parse_ansible_version,
 )
 from ansible_compat.constants import (
@@ -128,9 +129,6 @@ class Plugins:  # pylint: disable=too-many-instance-attributes
             try:
                 result = super().__getattribute__(attr)
             except AttributeError as exc:
-                if ansible_version() < Version("2.14") and attr in {"filter", "test"}:
-                    msg = "Ansible version below 2.14 does not support retrieving filter and test plugins."
-                    raise RuntimeError(msg) from exc
                 proc = self.runtime.run(
                     ["ansible-doc", "--json", "-l", "-t", attr],
                 )
@@ -211,7 +209,7 @@ class Runtime:
 
         if isolated:
             self.cache_dir = get_cache_dir(self.project_dir)
-        self.config = AnsibleConfig()
+        self.config = AnsibleConfig(cache_dir=self.cache_dir)
 
         # Add the sys.path to the collection paths if not isolated
         self._add_sys_path_to_collection_paths()
@@ -231,7 +229,7 @@ class Runtime:
             msg: str,
             *,
             formatted: bool = False,  # noqa: ARG001
-        ) -> None:
+        ) -> None:  # pragma: no cover
             """Override ansible.utils.display.Display.warning to avoid printing warnings."""
             warnings.warn(
                 message=msg,
@@ -275,34 +273,50 @@ class Runtime:
         self.collections = OrderedDict()
         no_collections_msg = "None of the provided paths were usable"
 
-        proc = self.run(["ansible-galaxy", "collection", "list", "--format=json"])
+        # do not use --path because it does not allow multiple values
+        proc = self.run(
+            [
+                "ansible-galaxy",
+                "collection",
+                "list",
+                "--format=json",
+            ],
+        )
         if proc.returncode == RC_ANSIBLE_OPTIONS_ERROR and (
             no_collections_msg in proc.stdout or no_collections_msg in proc.stderr
-        ):
+        ):  # pragma: no cover
             _logger.debug("Ansible reported no installed collections at all.")
             return
         if proc.returncode != 0:
             _logger.error(proc)
             msg = f"Unable to list collections: {proc}"
             raise RuntimeError(msg)
-        data = json.loads(proc.stdout)
+        try:
+            data = json.loads(proc.stdout)
+        except json.decoder.JSONDecodeError as exc:
+            msg = f"Unable to parse galaxy output as JSON: {proc.stdout}"
+            raise RuntimeError(msg) from exc
         if not isinstance(data, dict):
             msg = f"Unexpected collection data, {data}"
             raise TypeError(msg)
         for path in data:
+            if not isinstance(data[path], dict):
+                msg = f"Unexpected collection data, {data[path]}"
+                raise TypeError(msg)
             for collection, collection_info in data[path].items():
-                if not isinstance(collection, str):
-                    msg = f"Unexpected collection data, {collection}"
-                    raise TypeError(msg)
                 if not isinstance(collection_info, dict):
                     msg = f"Unexpected collection data, {collection_info}"
                     raise TypeError(msg)
 
-                self.collections[collection] = Collection(
-                    name=collection,
-                    version=collection_info["version"],
-                    path=path,
-                )
+                if collection in self.collections:
+                    msg = f"Multiple versions of '{collection}' were found installed, only the first one will be used, {self.collections[collection].version} ({self.collections[collection].path})."
+                    logging.warning(msg)
+                else:
+                    self.collections[collection] = Collection(
+                        name=collection,
+                        version=collection_info["version"],
+                        path=path,
+                    )
 
     def _ensure_module_available(self) -> None:
         """Assure that Ansible Python module is installed and matching CLI version."""
@@ -378,6 +392,8 @@ class Runtime:
         # https://github.com/ansible/ansible-lint/issues/3522
         env["ANSIBLE_VERBOSE_TO_STDERR"] = "True"
 
+        env["ANSIBLE_COLLECTIONS_PATH"] = ":".join(self.config.collections_paths)
+
         for _ in range(self.max_retries + 1 if retry else 1):
             result = run_func(
                 args,
@@ -506,7 +522,7 @@ class Runtime:
             env={**self.environ, ansible_collections_path(): ":".join(cpaths)},
         )
         if process.returncode != 0:
-            msg = f"Command returned {process.returncode} code:\n{process.stdout}\n{process.stderr}"
+            msg = f"Command {' '.join(cmd)}, returned {process.returncode} code:\n{process.stdout}\n{process.stderr}"
             _logger.error(msg)
             raise InvalidPrerequisiteError(msg)
 
@@ -594,19 +610,10 @@ class Runtime:
                 )
             else:
                 cmd.extend(["-r", str(requirement)])
-                cpaths = self.config.collections_paths
-                if self.cache_dir:
-                    # we cannot use '-p' because it breaks galaxy ability to ignore already installed collections, so
-                    # we hack ansible_collections_path instead and inject our own path there.
-                    dest_path = f"{self.cache_dir}/collections"
-                    if dest_path not in cpaths:
-                        # pylint: disable=no-member
-                        cpaths.insert(0, dest_path)
                 _logger.info("Running %s", " ".join(cmd))
                 result = self.run(
                     cmd,
                     retry=retry,
-                    env={**os.environ, "ANSIBLE_COLLECTIONS_PATH": ":".join(cpaths)},
                 )
                 _logger.debug(result.stdout)
                 if result.returncode != 0:
@@ -743,12 +750,6 @@ class Runtime:
                 msg,
             )
 
-        if self.cache_dir:
-            # if we have a cache dir, we want to be use that would be preferred
-            # destination when installing a missing collection
-            # https://github.com/PyCQA/pylint/issues/4667
-            paths.insert(0, f"{self.cache_dir}/collections")  # pylint: disable=E1101
-
         for path in paths:
             collpath = Path(path) / "ansible_collections" / ns / coll
             if collpath.exists():
@@ -772,18 +773,16 @@ class Runtime:
                             _logger.fatal(msg)
                             raise InvalidPrerequisiteError(msg)
                     return found_version, collpath.resolve()
-                break
-        else:
-            if install:
-                self.install_collection(f"{name}:>={version}" if version else name)
-                return self.require_collection(
-                    name=name,
-                    version=version,
-                    install=False,
-                )
-            msg = f"Collection '{name}' not found in '{paths}'"
-            _logger.fatal(msg)
-            raise InvalidPrerequisiteError(msg)
+        if install:
+            self.install_collection(f"{name}:>={version}" if version else name)
+            return self.require_collection(
+                name=name,
+                version=version,
+                install=False,
+            )
+        msg = f"Collection '{name}' not found in '{paths}'"
+        _logger.fatal(msg)
+        raise InvalidPrerequisiteError(msg)
 
     def _prepare_ansible_paths(self) -> None:
         """Configure Ansible environment variables."""