Adding upstream version 2.16.5.upstream/2.16.5

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 111 insertions, 154 deletions

diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index c45cfe3..05aab63 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -45,8 +45,8 @@ from ansible.errors import (
     AnsibleOptionsError,
     AnsibleUndefinedVariable,
 )
-from ansible.module_utils.six import string_types, text_type
-from ansible.module_utils._text import to_native, to_text, to_bytes
+from ansible.module_utils.six import string_types
+from ansible.module_utils.common.text.converters import to_native, to_text, to_bytes
 from ansible.module_utils.common.collections import is_sequence
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
 from ansible.template.native_helpers import ansible_native_concat, ansible_eval_concat, ansible_concat
@@ -55,7 +55,7 @@ from ansible.template.vars import AnsibleJ2Vars
 from ansible.utils.display import Display
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.native_jinja import NativeJinjaText
-from ansible.utils.unsafe_proxy import wrap_var, AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText
+from ansible.utils.unsafe_proxy import to_unsafe_text, wrap_var, AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText
 
 display = Display()
 
@@ -103,9 +103,9 @@ def generate_ansible_template_vars(path, fullpath=None, dest_path=None):
     managed_str = managed_default.format(
         host=temp_vars['template_host'],
         uid=temp_vars['template_uid'],
-        file=temp_vars['template_path'],
+        file=temp_vars['template_path'].replace('%', '%%'),
     )
-    temp_vars['ansible_managed'] = to_text(time.strftime(to_native(managed_str), time.localtime(os.path.getmtime(b_path))))
+    temp_vars['ansible_managed'] = to_unsafe_text(time.strftime(to_native(managed_str), time.localtime(os.path.getmtime(b_path))))
 
     return temp_vars
 
@@ -130,7 +130,7 @@ def _escape_backslashes(data, jinja_env):
     backslashes inside of a jinja2 expression.
 
     """
-    if '\\' in data and '{{' in data:
+    if '\\' in data and jinja_env.variable_start_string in data:
         new_data = []
         d2 = jinja_env.preprocess(data)
         in_var = False
@@ -153,6 +153,39 @@ def _escape_backslashes(data, jinja_env):
     return data
 
 
+def _create_overlay(data, overrides, jinja_env):
+    if overrides is None:
+        overrides = {}
+
+    try:
+        has_override_header = data.startswith(JINJA2_OVERRIDE)
+    except (TypeError, AttributeError):
+        has_override_header = False
+
+    if overrides or has_override_header:
+        overlay = jinja_env.overlay(**overrides)
+    else:
+        overlay = jinja_env
+
+    # Get jinja env overrides from template
+    if has_override_header:
+        eol = data.find('\n')
+        line = data[len(JINJA2_OVERRIDE):eol]
+        data = data[eol + 1:]
+        for pair in line.split(','):
+            if ':' not in pair:
+                raise AnsibleError("failed to parse jinja2 override '%s'."
+                                   " Did you use something different from colon as key-value separator?" % pair.strip())
+            (key, val) = pair.split(':', 1)
+            key = key.strip()
+            if hasattr(overlay, key):
+                setattr(overlay, key, ast.literal_eval(val.strip()))
+            else:
+                display.warning(f"Could not find Jinja2 environment setting to override: '{key}'")
+
+    return data, overlay
+
+
 def is_possibly_template(data, jinja_env):
     """Determines if a string looks like a template, by seeing if it
     contains a jinja2 start delimiter. Does not guarantee that the string
@@ -532,7 +565,7 @@ class AnsibleEnvironment(NativeEnvironment):
     '''
     context_class = AnsibleContext
     template_class = AnsibleJ2Template
-    concat = staticmethod(ansible_eval_concat)
+    concat = staticmethod(ansible_eval_concat)  # type: ignore[assignment]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -547,7 +580,7 @@ class AnsibleEnvironment(NativeEnvironment):
 
 
 class AnsibleNativeEnvironment(AnsibleEnvironment):
-    concat = staticmethod(ansible_native_concat)
+    concat = staticmethod(ansible_native_concat)  # type: ignore[assignment]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -559,14 +592,7 @@ class Templar:
     The main class for templating, with the main entry-point of template().
     '''
 
-    def __init__(self, loader, shared_loader_obj=None, variables=None):
-        if shared_loader_obj is not None:
-            display.deprecated(
-                "The `shared_loader_obj` option to `Templar` is no longer functional, "
-                "ansible.plugins.loader is used directly instead.",
-                version='2.16',
-            )
-
+    def __init__(self, loader, variables=None):
         self._loader = loader
         self._available_variables = {} if variables is None else variables
 
@@ -580,9 +606,6 @@ class Templar:
         )
         self.environment.template_class.environment_class = environment_class
 
-        # jinja2 global is inconsistent across versions, this normalizes them
-        self.environment.globals['dict'] = dict
-
         # Custom globals
         self.environment.globals['lookup'] = self._lookup
         self.environment.globals['query'] = self.environment.globals['q'] = self._query_lookup
@@ -592,11 +615,14 @@ class Templar:
         # the current rendering context under which the templar class is working
         self.cur_context = None
 
-        # FIXME this regex should be re-compiled each time variable_start_string and variable_end_string are changed
-        self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))
+        # this regex is re-compiled each time variable_start_string and variable_end_string are possibly changed
+        self._compile_single_var(self.environment)
 
         self.jinja2_native = C.DEFAULT_JINJA2_NATIVE
 
+    def _compile_single_var(self, env):
+        self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (env.variable_start_string, env.variable_end_string))
+
     def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs):
         r"""Creates a new copy of Templar with a new environment.
 
@@ -719,7 +745,7 @@ class Templar:
             variable = self._convert_bare_variable(variable)
 
         if isinstance(variable, string_types):
-            if not self.is_possibly_template(variable):
+            if not self.is_possibly_template(variable, overrides):
                 return variable
 
             # Check to see if the string we are trying to render is just referencing a single
@@ -744,6 +770,7 @@ class Templar:
                 disable_lookups=disable_lookups,
                 convert_data=convert_data,
             )
+            self._compile_single_var(self.environment)
 
             return result
 
@@ -790,8 +817,9 @@ class Templar:
 
     templatable = is_template
 
-    def is_possibly_template(self, data):
-        return is_possibly_template(data, self.environment)
+    def is_possibly_template(self, data, overrides=None):
+        data, env = _create_overlay(data, overrides, self.environment)
+        return is_possibly_template(data, env)
 
     def _convert_bare_variable(self, variable):
         '''
@@ -815,7 +843,7 @@ class Templar:
     def _now_datetime(self, utc=False, fmt=None):
         '''jinja2 global function to return current datetime, potentially formatted via strftime'''
         if utc:
-            now = datetime.datetime.utcnow()
+            now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
         else:
             now = datetime.datetime.now()
 
@@ -824,12 +852,12 @@ class Templar:
 
         return now
 
-    def _query_lookup(self, name, *args, **kwargs):
+    def _query_lookup(self, name, /, *args, **kwargs):
         ''' wrapper for lookup, force wantlist true'''
         kwargs['wantlist'] = True
         return self._lookup(name, *args, **kwargs)
 
-    def _lookup(self, name, *args, **kwargs):
+    def _lookup(self, name, /, *args, **kwargs):
         instance = lookup_loader.get(name, loader=self._loader, templar=self)
 
         if instance is None:
@@ -932,31 +960,12 @@ class Templar:
         if fail_on_undefined is None:
             fail_on_undefined = self._fail_on_undefined_errors
 
-        has_template_overrides = data.startswith(JINJA2_OVERRIDE)
-
         try:
             # NOTE Creating an overlay that lives only inside do_template means that overrides are not applied
             # when templating nested variables in AnsibleJ2Vars where Templar.environment is used, not the overlay.
-            # This is historic behavior that is kept for backwards compatibility.
-            if overrides:
-                myenv = self.environment.overlay(overrides)
-            elif has_template_overrides:
-                myenv = self.environment.overlay()
-            else:
-                myenv = self.environment
-
-            # Get jinja env overrides from template
-            if has_template_overrides:
-                eol = data.find('\n')
-                line = data[len(JINJA2_OVERRIDE):eol]
-                data = data[eol + 1:]
-                for pair in line.split(','):
-                    if ':' not in pair:
-                        raise AnsibleError("failed to parse jinja2 override '%s'."
-                                           " Did you use something different from colon as key-value separator?" % pair.strip())
-                    (key, val) = pair.split(':', 1)
-                    key = key.strip()
-                    setattr(myenv, key, ast.literal_eval(val.strip()))
+            data, myenv = _create_overlay(data, overrides, self.environment)
+            # in case delimiters change
+            self._compile_single_var(myenv)
 
             if escape_backslashes:
                 # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\".
@@ -964,7 +973,7 @@ class Templar:
 
             try:
                 t = myenv.from_string(data)
-            except TemplateSyntaxError as e:
+            except (TemplateSyntaxError, SyntaxError) as e:
                 raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data)), orig_exc=e)
             except Exception as e:
                 if 'recursion' in to_native(e):
diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py
index 3014c74..abe75c0 100644
--- a/lib/ansible/template/native_helpers.py
+++ b/lib/ansible/template/native_helpers.py
@@ -10,7 +10,7 @@ import ast
 from itertools import islice, chain
 from types import GeneratorType
 
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_text
 from ansible.module_utils.six import string_types
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.utils.native_jinja import NativeJinjaText
@@ -67,7 +67,7 @@ def ansible_eval_concat(nodes):
                     )
                 )
             )
-        except (ValueError, SyntaxError, MemoryError):
+        except (TypeError, ValueError, SyntaxError, MemoryError):
             pass
 
     return out
@@ -129,7 +129,7 @@ def ansible_native_concat(nodes):
             # parse the string ourselves without removing leading spaces/tabs.
             ast.parse(out, mode='eval')
         )
-    except (ValueError, SyntaxError, MemoryError):
+    except (TypeError, ValueError, SyntaxError, MemoryError):
         return out
 
     if isinstance(evaled, string_types):
diff --git a/lib/ansible/template/vars.py b/lib/ansible/template/vars.py
index fd1b812..6f40827 100644
--- a/lib/ansible/template/vars.py
+++ b/lib/ansible/template/vars.py
@@ -1,128 +1,76 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-# Make coding more python3-ish
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-from collections.abc import Mapping
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from collections import ChainMap
 
 from jinja2.utils import missing
 
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.text.converters import to_native
 
 
 __all__ = ['AnsibleJ2Vars']
 
 
-class AnsibleJ2Vars(Mapping):
-    '''
-    Helper class to template all variable content before jinja2 sees it. This is
-    done by hijacking the variable storage that jinja2 uses, and overriding __contains__
-    and __getitem__ to look like a dict. Added bonus is avoiding duplicating the large
-    hashes that inject tends to be.
+def _process_locals(_l):
+    if _l is None:
+        return {}
+    return {
+        k: v for k, v in _l.items()
+        if v is not missing
+        and k not in {'context', 'environment', 'template'}  # NOTE is this really needed?
+    }
 
-    To facilitate using builtin jinja2 things like range, globals are also handled here.
-    '''
 
-    def __init__(self, templar, globals, locals=None):
-        '''
-        Initializes this object with a valid Templar() object, as
-        well as several dictionaries of variables representing
-        different scopes (in jinja2 terminology).
-        '''
+class AnsibleJ2Vars(ChainMap):
+    """Helper variable storage class that allows for nested variables templating: `foo: "{{ bar }}"`."""
 
+    def __init__(self, templar, globals, locals=None):
         self._templar = templar
-        self._globals = globals
-        self._locals = dict()
-        if isinstance(locals, dict):
-            for key, val in locals.items():
-                if val is not missing:
-                    if key[:2] == 'l_':
-                        self._locals[key[2:]] = val
-                    elif key not in ('context', 'environment', 'template'):
-                        self._locals[key] = val
-
-    def __contains__(self, k):
-        if k in self._locals:
-            return True
-        if k in self._templar.available_variables:
-            return True
-        if k in self._globals:
-            return True
-        return False
-
-    def __iter__(self):
-        keys = set()
-        keys.update(self._templar.available_variables, self._locals, self._globals)
-        return iter(keys)
-
-    def __len__(self):
-        keys = set()
-        keys.update(self._templar.available_variables, self._locals, self._globals)
-        return len(keys)
+        super().__init__(
+            _process_locals(locals),  # first mapping has the highest precedence
+            self._templar.available_variables,
+            globals,
+        )
 
     def __getitem__(self, varname):
-        if varname in self._locals:
-            return self._locals[varname]
-        if varname in self._templar.available_variables:
-            variable = self._templar.available_variables[varname]
-        elif varname in self._globals:
-            return self._globals[varname]
-        else:
-            raise KeyError("undefined variable: %s" % varname)
-
-        # HostVars is special, return it as-is, as is the special variable
-        # 'vars', which contains the vars structure
+        variable = super().__getitem__(varname)
+
         from ansible.vars.hostvars import HostVars
-        if isinstance(variable, dict) and varname == "vars" or isinstance(variable, HostVars) or hasattr(variable, '__UNSAFE__'):
+        if (varname == "vars" and isinstance(variable, dict)) or isinstance(variable, HostVars) or hasattr(variable, '__UNSAFE__'):
             return variable
-        else:
-            value = None
-            try:
-                value = self._templar.template(variable)
-            except AnsibleUndefinedVariable as e:
-                # Instead of failing here prematurely, return an Undefined
-                # object which fails only after its first usage allowing us to
-                # do lazy evaluation and passing it into filters/tests that
-                # operate on such objects.
-                return self._templar.environment.undefined(
-                    hint=f"{variable}: {e.message}",
-                    name=varname,
-                    exc=AnsibleUndefinedVariable,
-                )
-            except Exception as e:
-                msg = getattr(e, 'message', None) or to_native(e)
-                raise AnsibleError("An unhandled exception occurred while templating '%s'. "
-                                   "Error was a %s, original message: %s" % (to_native(variable), type(e), msg))
-
-            return value
+
+        try:
+            return self._templar.template(variable)
+        except AnsibleUndefinedVariable as e:
+            # Instead of failing here prematurely, return an Undefined
+            # object which fails only after its first usage allowing us to
+            # do lazy evaluation and passing it into filters/tests that
+            # operate on such objects.
+            return self._templar.environment.undefined(
+                hint=f"{variable}: {e.message}",
+                name=varname,
+                exc=AnsibleUndefinedVariable,
+            )
+        except Exception as e:
+            msg = getattr(e, 'message', None) or to_native(e)
+            raise AnsibleError(
+                f"An unhandled exception occurred while templating '{to_native(variable)}'. "
+                f"Error was a {type(e)}, original message: {msg}"
+            )
 
     def add_locals(self, locals):
-        '''
-        If locals are provided, create a copy of self containing those
+        """If locals are provided, create a copy of self containing those
         locals in addition to what is already in this variable proxy.
-        '''
+        """
         if locals is None:
             return self
 
+        current_locals = self.maps[0]
+        current_globals = self.maps[2]
+
         # prior to version 2.9, locals contained all of the vars and not just the current
         # local vars so this was not necessary for locals to propagate down to nested includes
-        new_locals = self._locals | locals
+        new_locals = current_locals | locals
 
-        return AnsibleJ2Vars(self._templar, self._globals, locals=new_locals)
+        return AnsibleJ2Vars(self._templar, current_globals, locals=new_locals)