Merging upstream version 2.17.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

21 files changed, 192 insertions, 36 deletions

diff --git a/PKG-INFO b/PKG-INFO
index 5cefb3a..b62b8dd 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ansible-core
-Version: 2.17.1
+Version: 2.17.2
 Summary: Radically simple IT automation
 Home-page: https://ansible.com/
 Author: Ansible, Inc.
diff --git a/changelogs/CHANGELOG-v2.17.rst b/changelogs/CHANGELOG-v2.17.rst
index 65bd411..1ac8c02 100644
--- a/changelogs/CHANGELOG-v2.17.rst
+++ b/changelogs/CHANGELOG-v2.17.rst
@@ -4,6 +4,26 @@ ansible-core 2.17 "Gallows Pole" Release Notes
 
 .. contents:: Topics
 
+v2.17.2
+=======
+
+Release Summary
+---------------
+
+| Release Date: 2024-07-15
+| `Porting Guide <https://docs.ansible.com/ansible-core/2.17/porting_guides/porting_guide_core_2.17.html>`__
+
+Bugfixes
+--------
+
+- Fix a traceback when an environment variable contains certain special characters (https://github.com/ansible/ansible/issues/83498)
+- dnf - reverted incomplete fix from 2.17.2rc1 (https://github.com/ansible/ansible/pull/83504)
+- dnf, dnf5 - fix for installing a set of packages by specifying them using a wildcard character (https://github.com/ansible/ansible/issues/83373)
+- linear strategy now provides a properly templated task name to the v2_runner_on_started callback event.
+- package_facts - ignore warnings sent by apk on stderr (https://github.com/ansible/ansible/issues/83501).
+- replace - Updated before/after example (https://github.com/ansible/ansible/issues/83390).
+- templating hostvars under native jinja will not cause serialization errors anymore.
+
 v2.17.1
 =======
 
diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml
index 3020a6f..c4da406 100644
--- a/changelogs/changelog.yaml
+++ b/changelogs/changelog.yaml
@@ -731,3 +731,56 @@ releases:
     - PowerShell-AddType-temp.yml
     - correct-callback-fqcn-old-style-action-invocation.yml
     release_date: '2024-06-10'
+  2.17.2:
+    changes:
+      release_summary: '| Release Date: 2024-07-15
+
+        | `Porting Guide <https://docs.ansible.com/ansible-core/2.17/porting_guides/porting_guide_core_2.17.html>`__
+
+        '
+    codename: Gallows Pole
+    fragments:
+    - 2.17.2_summary.yaml
+    release_date: '2024-07-15'
+  2.17.2rc1:
+    changes:
+      bugfixes:
+      - Fix a traceback when an environment variable contains certain special characters
+        (https://github.com/ansible/ansible/issues/83498)
+      - dnf, dnf5 - fix for installing a set of packages by specifying them using
+        a wildcard character (https://github.com/ansible/ansible/issues/83373)
+      - linear strategy now provides a properly templated task name to the v2_runner_on_started
+        callback event.
+      - replace - Updated before/after example (https://github.com/ansible/ansible/issues/83390).
+      - templating hostvars under native jinja will not cause serialization errors
+        anymore.
+      release_summary: '| Release Date: 2024-07-08
+
+        | `Porting Guide <https://docs.ansible.com/ansible-core/2.17/porting_guides/porting_guide_core_2.17.html>`__
+
+        '
+    codename: Gallows Pole
+    fragments:
+    - 2.17.2rc1_summary.yaml
+    - 83373-dnf5-wildcard.yml
+    - 83498-command-tb-env.yml
+    - hostvars_fix.yml
+    - linear_started_name.yml
+    - replace_regex.yml
+    release_date: '2024-07-08'
+  2.17.2rc2:
+    changes:
+      bugfixes:
+      - dnf - reverted incomplete fix from 2.17.2rc1 (https://github.com/ansible/ansible/pull/83504)
+      - package_facts - ignore warnings sent by apk on stderr (https://github.com/ansible/ansible/issues/83501).
+      release_summary: '| Release Date: 2024-07-09
+
+        | `Porting Guide <https://docs.ansible.com/ansible-core/2.17/porting_guides/porting_guide_core_2.17.html>`__
+
+        '
+    codename: Gallows Pole
+    fragments:
+    - 2.17.2rc2_summary.yaml
+    - dnf-revert.yaml
+    - package_facts_apk.yml
+    release_date: '2024-07-09'
diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml
index 9a5686d..b4c2ae5 100644
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@@ -1733,7 +1733,7 @@ INJECT_FACTS_AS_VARS:
   default: True
   description:
     - Facts are available inside the `ansible_facts` variable, this setting also pushes them as their own vars in the main namespace.
-    - Unlike inside the `ansible_facts` dictionary, these will have an `ansible_` prefix.
+    - Unlike inside the `ansible_facts` dictionary where the prefix `ansible_` is removed from fact names, these will have the exact names that are returned by the module.
   env: [{name: ANSIBLE_INJECT_FACT_VARS}]
   ini:
   - {key: inject_facts_as_vars, section: defaults}
diff --git a/lib/ansible/module_utils/ansible_release.py b/lib/ansible/module_utils/ansible_release.py
index 7ed9fdf..8d3f49b 100644
--- a/lib/ansible/module_utils/ansible_release.py
+++ b/lib/ansible/module_utils/ansible_release.py
@@ -17,6 +17,6 @@
 
 from __future__ import annotations
 
-__version__ = '2.17.1'
+__version__ = '2.17.2'
 __author__ = 'Ansible, Inc.'
 __codename__ = "Gallows Pole"
diff --git a/lib/ansible/modules/dnf.py b/lib/ansible/modules/dnf.py
index 593f006..2ab66fb 100644
--- a/lib/ansible/modules/dnf.py
+++ b/lib/ansible/modules/dnf.py
@@ -19,9 +19,15 @@ description:
 options:
   use_backend:
     description:
-      - By default, this module will select the backend based on the C(ansible_pkg_mgr) fact.
+      - Backend module to use.
     default: "auto"
-    choices: [ auto, yum, yum4, dnf4, dnf5 ]
+    choices:
+        auto: Automatically select the backend based on the C(ansible_facts.pkg_mgr) fact.
+        yum: Alias for V(auto) (see Notes)
+        dnf: M(ansible.builtin.dnf)
+        yum4: Alias for V(dnf)
+        dnf4: Alias for V(dnf)
+        dnf5: M(ansible.builtin.dnf5)
     type: str
     version_added: 2.15
   name:
@@ -287,6 +293,11 @@ notes:
     upstream dnf's API doesn't properly mark groups as installed, therefore upon
     removal the module is unable to detect that the group is installed
     (https://bugzilla.redhat.com/show_bug.cgi?id=1620324)
+  - While O(use_backend=yum) and the ability to call the action plugin as
+    M(ansible.builtin.yum) are provided for syntax compatibility, the YUM
+    backend was removed in ansible-core 2.17 because the required libraries are
+    not available for any supported version of Python. If you rely on this
+    functionality, use an older version of Ansible.
 requirements:
   - "python >= 2.6"
   - python-dnf
@@ -723,9 +734,14 @@ class DnfModule(YumDnf):
         self.module.exit_json(msg="", results=results)
 
     def _is_installed(self, pkg):
-        return bool(
-            dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).installed().run()
-        )
+        installed_query = dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).installed()
+        if dnf.util.is_glob_pattern(pkg):
+            available_query = dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).available()
+            return not (
+                {p.name for p in available_query} - {p.name for p in installed_query}
+            )
+        else:
+            return bool(installed_query)
 
     def _is_newer_version_installed(self, pkg_name):
         try:
@@ -1336,7 +1352,7 @@ def main():
     #   list=repos
     #   list=pkgspec
 
-    yumdnf_argument_spec['argument_spec']['use_backend'] = dict(default='auto', choices=['auto', 'yum', 'yum4', 'dnf4', 'dnf5'])
+    yumdnf_argument_spec['argument_spec']['use_backend'] = dict(default='auto', choices=['auto', 'dnf', 'yum', 'yum4', 'dnf4', 'dnf5'])
 
     module = AnsibleModule(
         **yumdnf_argument_spec
diff --git a/lib/ansible/modules/dnf5.py b/lib/ansible/modules/dnf5.py
index 7af1f4a..3a4fdfc 100644
--- a/lib/ansible/modules/dnf5.py
+++ b/lib/ansible/modules/dnf5.py
@@ -357,10 +357,23 @@ libdnf5 = None
 
 def is_installed(base, spec):
     settings = libdnf5.base.ResolveSpecSettings()
-    query = libdnf5.rpm.PackageQuery(base)
-    query.filter_installed()
-    match, nevra = query.resolve_pkg_spec(spec, settings, True)
-    return match
+    installed_query = libdnf5.rpm.PackageQuery(base)
+    installed_query.filter_installed()
+    match, nevra = installed_query.resolve_pkg_spec(spec, settings, True)
+
+    # FIXME use `is_glob_pattern` function when available:
+    # https://github.com/rpm-software-management/dnf5/issues/1563
+    glob_patterns = set("*[?")
+    if any(set(char) & glob_patterns for char in spec):
+        available_query = libdnf5.rpm.PackageQuery(base)
+        available_query.filter_available()
+        available_query.resolve_pkg_spec(spec, settings, True)
+
+        return not (
+            {p.get_name() for p in available_query} - {p.get_name() for p in installed_query}
+        )
+    else:
+        return match
 
 
 def is_newer_version_installed(base, spec):
diff --git a/lib/ansible/modules/package_facts.py b/lib/ansible/modules/package_facts.py
index 11a8f61..dd9badf 100644
--- a/lib/ansible/modules/package_facts.py
+++ b/lib/ansible/modules/package_facts.py
@@ -440,7 +440,7 @@ class APK(CLIMgr):
 
     def list_installed(self):
         rc, out, err = module.run_command([self._cli, 'info', '-v'])
-        if rc != 0 or err:
+        if rc != 0:
             raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
         return out.splitlines()
 
diff --git a/lib/ansible/modules/replace.py b/lib/ansible/modules/replace.py
index 2fee290..8e4b976 100644
--- a/lib/ansible/modules/replace.py
+++ b/lib/ansible/modules/replace.py
@@ -140,7 +140,7 @@ EXAMPLES = r'''
   ansible.builtin.replace:
     path: /etc/hosts
     after: '(?m)^<VirtualHost [*]>'
-    before: '(?m)^</VirtualHost>'
+    before: '</VirtualHost>'
     regexp: '^(.+)$'
     replace: '# \1'
 
diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py
index 56be533..49bbb01 100644
--- a/lib/ansible/plugins/shell/__init__.py
+++ b/lib/ansible/plugins/shell/__init__.py
@@ -211,7 +211,11 @@ class ShellBase(AnsiblePlugin):
             arg_path,
         ]
 
-        return f'{env_string}%s' % shlex.join(cps for cp in cmd_parts if cp and (cps := cp.strip()))
+        cleaned_up_cmd = shlex.join(
+            stripped_cmd_part for raw_cmd_part in cmd_parts
+            if raw_cmd_part and (stripped_cmd_part := raw_cmd_part.strip())
+        )
+        return ''.join((env_string, cleaned_up_cmd))
 
     def append_command(self, cmd, cmd_to_append):
         """Append an additional command if supported by the shell"""
diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py
index 29f94c4..d9e5d42 100644
--- a/lib/ansible/plugins/strategy/linear.py
+++ b/lib/ansible/plugins/strategy/linear.py
@@ -211,30 +211,21 @@ class StrategyModule(StrategyBase):
                                 skip_rest = True
                                 break
 
-                        run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False)
+                        run_once = action and getattr(action, 'BYPASS_HOST_LOOP', False) or templar.template(task.run_once)
+                        try:
+                            task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
+                        except Exception as e:
+                            display.debug(f"Failed to templalte task name ({task.name}), ignoring error and continuing: {e}")
 
                         if (task.any_errors_fatal or run_once) and not task.ignore_errors:
                             any_errors_fatal = True
 
                         if not callback_sent:
-                            display.debug("sending task start callback, copying the task so we can template it temporarily")
-                            saved_name = task.name
-                            display.debug("done copying, going to template now")
-                            try:
-                                task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
-                                display.debug("done templating")
-                            except Exception:
-                                # just ignore any errors during task name templating,
-                                # we don't care if it just shows the raw name
-                                display.debug("templating failed for some reason")
-                            display.debug("here goes the callback...")
                             if isinstance(task, Handler):
                                 self._tqm.send_callback('v2_playbook_on_handler_task_start', task)
                             else:
                                 self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
-                            task.name = saved_name
                             callback_sent = True
-                            display.debug("sending task start callback")
 
                         self._blocked_hosts[host.get_name()] = True
                         self._queue_task(host, task, task_vars, play_context)
diff --git a/lib/ansible/release.py b/lib/ansible/release.py
index 7ed9fdf..8d3f49b 100644
--- a/lib/ansible/release.py
+++ b/lib/ansible/release.py
@@ -17,6 +17,6 @@
 
 from __future__ import annotations
 
-__version__ = '2.17.1'
+__version__ = '2.17.2'
 __author__ = 'Ansible, Inc.'
 __codename__ = "Gallows Pole"
diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py
index bb0372e..6f8491d 100644
--- a/lib/ansible/vars/hostvars.py
+++ b/lib/ansible/vars/hostvars.py
@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
+from functools import cached_property
 
 from ansible import constants as C
 from ansible.template import Templar, AnsibleUndefined
@@ -114,9 +115,12 @@ class HostVarsVars(Mapping):
     def __init__(self, variables, loader):
         self._vars = variables
         self._loader = loader
+
+    @cached_property
+    def _templar(self):
         # NOTE: this only has access to the host's own vars,
         # so templates that depend on vars in other scopes will not work.
-        self._templar = Templar(variables=self._vars, loader=self._loader)
+        return Templar(variables=self._vars, loader=self._loader)
 
     def __getitem__(self, var):
         return self._templar.template(self._vars[var], fail_on_undefined=False, static_vars=C.INTERNAL_STATIC_VARS)
@@ -132,3 +136,10 @@ class HostVarsVars(Mapping):
 
     def __repr__(self):
         return repr(self._templar.template(self._vars, fail_on_undefined=False, static_vars=C.INTERNAL_STATIC_VARS))
+
+    def __getstate__(self):
+        ''' override serialization here to avoid
+            pickle issues with templar and Jinja native'''
+        state = self.__dict__.copy()
+        state.pop('_templar', None)
+        return state
diff --git a/lib/ansible_core.egg-info/PKG-INFO b/lib/ansible_core.egg-info/PKG-INFO
index 5cefb3a..b62b8dd 100644
--- a/lib/ansible_core.egg-info/PKG-INFO
+++ b/lib/ansible_core.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ansible-core
-Version: 2.17.1
+Version: 2.17.2
 Summary: Radically simple IT automation
 Home-page: https://ansible.com/
 Author: Ansible, Inc.
diff --git a/packaging/release.py b/packaging/release.py
index 95ee2c3..1b69631 100755
--- a/packaging/release.py
+++ b/packaging/release.py
@@ -751,12 +751,17 @@ def get_next_version(version: Version, /, final: bool = False, pre: str | None =
             pre = ""
         elif not pre and version.pre is not None:
             pre = f"{version.pre[0]}{version.pre[1]}"
+        elif not pre:
+            pre = "b1"  # when there is no existing pre and none specified, advance to b1
+
     elif version.is_postrelease:
         # The next version of a post release is the next pre-release *or* micro release component.
         if final:
             pre = ""
         elif not pre and version.pre is not None:
             pre = f"{version.pre[0]}{version.pre[1] + 1}"
+        elif not pre:
+            pre = "rc1"  # when there is no existing pre and none specified, advance to rc1
 
         if version.pre is None:
             micro = version.micro + 1
@@ -1041,7 +1046,7 @@ See the [full changelog]({{ changelog }}) for the changes included in this relea
 # Release Artifacts
 
 {%- for release in releases %}
-* {{ release.package_label }}: [{{ release.url|basename }}]({{ release.url }}) - {{ release.size }} bytes
+* {{ release.package_label }}: [{{ release.url|basename }}]({{ release.url }}) - &zwnj;{{ release.size }} bytes
   * {{ release.digest }} ({{ release.digest_algorithm }})
 {%- endfor %}
 """
@@ -1130,7 +1135,7 @@ command = CommandFramework(
     pre=dict(exclusive="version", help="increment version to the specified pre-release (aN, bN, rcN)"),
     final=dict(exclusive="version", action="store_true", help="increment version to the next final release"),
     commit=dict(help="commit to tag"),
-    mailto=dict(name="--no-mailto", action="store_false", help="write announcement to console instead of using a mailto: link"),
+    mailto=dict(name="--mailto", action="store_true", help="write announcement to mailto link instead of console"),
     validate=dict(name="--no-validate", action="store_false", help="disable validation of PyPI artifacts against local ones"),
     prompt=dict(name="--no-prompt", action="store_false", help="disable interactive prompt before publishing with twine"),
     allow_tag=dict(action="store_true", help="allow an existing release tag (for testing)"),
diff --git a/test/integration/targets/dnf/tasks/repo.yml b/test/integration/targets/dnf/tasks/repo.yml
index 7e34aed..d50535b 100644
--- a/test/integration/targets/dnf/tasks/repo.yml
+++ b/test/integration/targets/dnf/tasks/repo.yml
@@ -520,3 +520,25 @@
       dnf:
         name: epochone
         state: absent
+
+# https://github.com/ansible/ansible/issues/83373
+- name: test installing a set of packages by specifying them using a wildcard character
+  block:
+    - dnf:
+        name: provides_foo_a
+        state: present
+
+    - dnf:
+        name: provides_foo*
+        state: present
+      register: dnf_results
+
+    - assert:
+        that:
+          - dnf_results is changed
+          - "'Installed: provides_foo_b' in dnf_results['results'][0]"
+  always:
+    - name: Clean up
+      dnf:
+        name: provides_foo*
+        state: absent
diff --git a/test/integration/targets/dnf5/playbook.yml b/test/integration/targets/dnf5/playbook.yml
index 16dfd22..78b5f4d 100644
--- a/test/integration/targets/dnf5/playbook.yml
+++ b/test/integration/targets/dnf5/playbook.yml
@@ -2,7 +2,7 @@
   tasks:
     - block:
         - command: "dnf install -y 'dnf-command(copr)'"
-        - command: dnf copr enable -y rpmsoftwaremanagement/dnf5-unstable
+        - command: dnf copr enable -y rpmsoftwaremanagement/dnf-nightly
         - command: dnf install -y python3-libdnf5
 
         - include_role:
diff --git a/test/integration/targets/package_facts/tasks/main.yml b/test/integration/targets/package_facts/tasks/main.yml
index 12dfcf0..6d4b04d 100644
--- a/test/integration/targets/package_facts/tasks/main.yml
+++ b/test/integration/targets/package_facts/tasks/main.yml
@@ -65,6 +65,19 @@
         that: ansible_facts.packages is defined
   when: (ansible_os_family == "openSUSE Leap") or (ansible_os_family == "Suse")
 
+- name: Same as those above, but based on pkg_mgr
+  block:
+    - name: Gather package facts
+      package_facts:
+        manager: '{{ ansible_facts["pkg_mgr"] }}'
+
+    - name: check for ansible_facts.packages exists
+      assert:
+        that:
+          - ansible_facts.packages is defined
+          - ansible_facts.packages | length > 1
+  when: ansible_facts['os_family'] in ["openSUSE Leap", "Suse", "Debian"]
+
 # Check that auto detection works also
 - name: Gather package facts
   package_facts:
diff --git a/test/integration/targets/retry_task_name_in_callback/runme.sh b/test/integration/targets/retry_task_name_in_callback/runme.sh
index 5f636cd..415a018 100755
--- a/test/integration/targets/retry_task_name_in_callback/runme.sh
+++ b/test/integration/targets/retry_task_name_in_callback/runme.sh
@@ -11,3 +11,8 @@ EXPECTED_REGEX="^.*TASK.*18236 callback task template fix OUTPUT 2"
 ansible-playbook "$@" -i ../../inventory test.yml | tee "${OUTFILE}"
 echo "Grepping for ${EXPECTED_REGEX} in stdout."
 grep -e "${EXPECTED_REGEX}" "${OUTFILE}"
+
+# check variables are interpolated in 'started'
+UNTEMPLATED_STARTED="^.*\[started .*{{.*}}.*$"
+echo "Checking we dont have untemplated started in stdout."
+grep -e "${UNTEMPLATED_STARTED}" "${OUTFILE}" || exit 0
diff --git a/test/integration/targets/shell/tasks/command-building.yml b/test/integration/targets/shell/tasks/command-building.yml
index bd45261..d22f674 100644
--- a/test/integration/targets/shell/tasks/command-building.yml
+++ b/test/integration/targets/shell/tasks/command-building.yml
@@ -28,6 +28,7 @@
         ANSIBLE_REMOTE_TMP: '{{ atd }}'
         ANSIBLE_NOCOLOR: "1"
         ANSIBLE_FORCE_COLOR: "0"
+        TEST: "foo%D"
       register: command_building
       delegate_to: localhost
 
diff --git a/test/integration/targets/template/runme.sh b/test/integration/targets/template/runme.sh
index e814110..b37467a 100755
--- a/test/integration/targets/template/runme.sh
+++ b/test/integration/targets/template/runme.sh
@@ -55,3 +55,5 @@ do
 	ANSIBLE_CONFIG="./${badcfg}.cfg" ansible-config dump --only-changed
 done
 
+# ensure we picle hostvarscorrectly with native https://github.com/ansible/ansible/issues/83503
+ANSIBLE_JINJA2_NATIVE=1 ansible -m debug -a "msg={{ groups.all | map('extract', hostvars) }}" -i testhost, all -c local -v "$@"